diff --git a/.cargo/config.toml b/.cargo/config.toml index aea0df2d59..cc11c8e77d 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,20 +26,22 @@ test_details = "test --target aarch64-apple-darwin" # native crate needs no change here. Axum (native), Cloudflare # (wasm32-unknown-unknown), Spin, the CLI (native), and integration-tests # (native) are simply not listed. -build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-device-fastly -p trusted-server-geo-fastly -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. test-fastly-reuse = "test -p trusted-server-adapter-fastly --target wasm32-wasip1 --features reusable-sandbox" # --- Axum adapter (native dev server) --- +# Test and clippy also cover the permission signal crates, whose own tests +# run with the Axum adapter in CI. build-axum = "build -p trusted-server-adapter-axum" check-axum = "check -p trusted-server-adapter-axum" -clippy-axum = "clippy -p trusted-server-adapter-axum --all-targets --all-features -- -D warnings" -test-axum = "test -p trusted-server-adapter-axum" +clippy-axum = "clippy -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb --all-targets --all-features -- -D warnings" +test-axum = "test -p trusted-server-adapter-axum -p trusted-server-permission-signal-gpc -p trusted-server-permission-signal-gpp -p trusted-server-permission-signal-mtm -p trusted-server-permission-signal-tcf -p trusted-server-permission-signal-us-privacy -p trusted-server-cmp-osano -p trusted-server-modules -p trusted-server-identity-lockr -p trusted-server-audience-permutive -p trusted-server-cmp-sourcepoint -p trusted-server-cmp-didomi -p trusted-server-cmp-inmobi -p trusted-server-ads-txt-sellers -p trusted-server-loading-cache-51degrees -p trusted-server-tag-google-tag -p trusted-server-tag-google-tag-manager -p trusted-server-testing-testlight -p trusted-server-framework-nextjs -p trusted-server-ad-tag-google -p trusted-server-bot-protection-datadome -p trusted-server-ad-server-mock -p trusted-server-auction-aps -p trusted-server-auction-prebid -p trusted-server-auction-prebid-server -p trusted-server-auction-protocol-openrtb" # --- Cloudflare adapter (native host + wasm32-unknown-unknown) --- # Build/check target the WASM runtime (requires the `cloudflare` feature); diff --git a/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md b/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md index f0ff876d6c..d8d53ae0e2 100644 --- a/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md +++ b/.claude/skills/planning-prebid-aws/references/configuration-and-secrets.md @@ -12,17 +12,17 @@ Before asking questions the repository can answer: 4. Add observations to the existing deployment decision record with source path/key, candidate requirement, confidence/status, and the remaining question. Report credential identifiers or presence only; keep raw TOML, credential values, and commercially sensitive publisher values out of reports and generated examples. 5. Verify candidate adapters against the selected PBS Go release and confirm partner authorization. Record each host-secret requirement as required, not needed, or unresolved. A bidder name alone proves neither a secret requirement nor permission to use the bidder. -| Trusted Server input | Discovery use | -| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- | -| `auction.enabled` | Determine active versus disabled server-auction intent without expanding defaults | -| `auction.providers.` with `profile = "prebid-server"` | Identify each configured PBS provider; withhold endpoint values | -| Provider `timeout_ms` and `profile_config.test_mode` or `debug` | Record explicit timeout and diagnostic intent, not permission to contact bidders | -| Provider `profile_config.bid_param_override_rules` | Count conditional publisher or placement inputs without reporting their values | -| `auction.bidders..provider` | Group candidate server-side adapters under the referenced PBS provider | -| `integrations.prebid.enabled`, `account_id`, `timeout_ms`, `debug` | Record browser integration intent and compatibility questions separately from server demand | -| `integrations.prebid.client_side_bidders` | Browser-side participation; do not automatically enable these adapters in PBS | -| `integrations.prebid.bundle` | Browser bundle and identity capabilities, not proof of server-side use | -| Relevant privacy, format, and stored-request settings | Identify dependencies that need confirmation from the caller or request path | +| Trusted Server input | Discovery use | +| --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `auction.enabled` | Determine active versus disabled server-auction intent without expanding defaults | +| `demand.` with `implementation = "auction.prebid-server"` | Identify each configured PBS demand source and whether `demand.modules` selects it; withhold endpoint values | +| Demand source `timeout_ms`, `test_mode` and `debug` | Record explicit timeout and diagnostic intent, not permission to contact bidders | +| Demand source `bid_param_override_rules` | Count conditional publisher or placement inputs without reporting their values | +| `auction.bidders..module` | Group candidate server-side adapters under the referenced PBS demand source | +| `prebid` in `auction.modules`, and `auction.prebid` `account_id`, `timeout_ms`, `debug` | Record browser integration intent and compatibility questions separately from server demand | +| `auction.prebid.client_side_bidders` | Browser-side participation; do not automatically enable these adapters in PBS | +| `auction.prebid.bundle` | Browser bundle and identity capabilities, not proof of server-side use | +| Relevant privacy, format, and stored-request settings | Identify dependencies that need confirmation from the caller or request path | Use `ts prebid server inspect --config ` for redacted local discovery. It reports explicit values without expanding defaults or proving host-secret requirements. Resolve schema defaults and adapter metadata separately, recording unsupported details as unresolved rather than fetching credentials or inventing mappings. Read config without invoking commands that publish or rewrite it. Leave `trusted-server.toml` unchanged. Propose caller endpoint/account changes separately after integration approval; discovery must not activate bidders, publish configuration, or change traffic. diff --git a/.env.example b/.env.example index a7f5973cdb..8a874e292c 100644 --- a/.env.example +++ b/.env.example @@ -40,41 +40,36 @@ TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=false # ============================================================================= # Integrations # ============================================================================= +# Which integrations run is set in the TOML file, in the section of each +# one's type, as `[auction] modules = ["prebid"]` and +# `[framework] module = "nextjs"`. There is no per-integration enabled flag. +# An overlay below changes a setting the file already carries, and it cannot +# add one the file leaves out or replace a list. An overlay names its section +# as the file writes it, so a section with a hyphen in its name, such as +# `[ad-tag]`, is set with `env 'NAME=value' command`, because a shell's +# `export` refuses the name. -# Prebid -TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.example.com/openrtb2/auction -# TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"bidder-name":{"param1":12345,"param2":"value"}}' -# Compatibility env shape for bidder -> zone -> params overrides -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_abc"}}}' -# Preferred canonical env shape for future generic rules -# TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_abc"}}]' -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SUPPRESS_NURL_BIDDERS=exampleBidder,anotherBidder -# TRUSTED_SERVER__INTEGRATIONS__PREBID__AUTO_CONFIGURE=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__TEST_MODE=false +# Prebid, the browser side. The Prebid Server endpoint belongs to a demand +# source, so its overlay is TRUSTED_SERVER__DEMAND____ENDPOINT. +# TRUSTED_SERVER__AUCTION__PREBID__ACCOUNT_ID= +# TRUSTED_SERVER__AUCTION__PREBID__TIMEOUT_MS=1000 +# TRUSTED_SERVER__AUCTION__PREBID__DEBUG=false # Next.js -TRUSTED_SERVER__INTEGRATIONS__NEXTJS__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__NEXTJS__REWRITE_ATTRIBUTES=href,link,url +# TRUSTED_SERVER__FRAMEWORK__NEXTJS__MAX_COMBINED_PAYLOAD_BYTES=10485760 # Didomi -TRUSTED_SERVER__INTEGRATIONS__DIDOMI__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__DIDOMI__SDK_ORIGIN=https://sdk.privacy-center.org -# TRUSTED_SERVER__INTEGRATIONS__DIDOMI__API_ORIGIN=https://api.privacy-center.org +# TRUSTED_SERVER__CMP__DIDOMI__SDK_ORIGIN=https://sdk.privacy-center.org +# TRUSTED_SERVER__CMP__DIDOMI__API_ORIGIN=https://api.privacy-center.org # Permutive -TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__ORGANIZATION_ID= -# TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__WORKSPACE_ID= -# TRUSTED_SERVER__INTEGRATIONS__PERMUTIVE__PROJECT_ID= +# TRUSTED_SERVER__AUDIENCE__PERMUTIVE__ORGANIZATION_ID= +# TRUSTED_SERVER__AUDIENCE__PERMUTIVE__WORKSPACE_ID= +# TRUSTED_SERVER__AUDIENCE__PERMUTIVE__PROJECT_ID= # Lockr -TRUSTED_SERVER__INTEGRATIONS__LOCKR__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__LOCKR__APP_ID= -# TRUSTED_SERVER__INTEGRATIONS__LOCKR__CACHE_TTL_SECONDS=3600 +# TRUSTED_SERVER__IDENTITY__LOCKR__APP_ID= +# TRUSTED_SERVER__IDENTITY__LOCKR__CACHE_TTL_SECONDS=3600 # ============================================================================= # Rewrite Configuration (optional) diff --git a/.github/workflows/inspector.yml b/.github/workflows/inspector.yml new file mode 100644 index 0000000000..5d58b3d93e --- /dev/null +++ b/.github/workflows/inspector.yml @@ -0,0 +1,26 @@ +name: "Permissions Inspector" + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + build-inspector-wasm: + name: build inspector wasm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Retrieve Rust version + id: rust-version + run: echo "rust-version=$(grep -oP 'channel = "\K[^"]+' rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - name: Set up Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ steps.rust-version.outputs.rust-version }} + target: wasm32-unknown-unknown + - name: Build the inspector engine + run: ./scripts/build-inspector-wasm.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95654ecddf..37d0d2148f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,12 +68,23 @@ jobs: - name: Run inline control harness run: BID_DELAY=3 ./scripts/template-cache-local-test.sh inline - - name: Run cache purge harness - run: BID_DELAY=3 ./scripts/template-cache-local-test.sh purge + - name: Run config by host harness + run: ./scripts/config-by-host-local-test.sh + + - name: Run config by host harness (reusable-sandbox feature) + run: ./scripts/config-by-host-local-test.sh reuse + + # Last, because it builds core with a key schedule compiled in. + - name: Run attestation harness + run: ./scripts/attestation-local-test.sh test-axum: - name: cargo test (axum native) - runs-on: ubuntu-latest + name: cargo test (axum native, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -122,12 +133,33 @@ jobs: run: | cargo test --package trusted-server-openrtb-codegen --target "$(rustc -vV | sed -n 's/host: //p')" + # The core library's unit tests otherwise run only on the WebAssembly + # targets, which build with panic=abort, so their harness stops at the + # first failing test and reports every later one as never run. A run that + # looks like a single failure can hide many more. This native run reports + # them all at once, which is what makes a red build readable. + - name: Run host-target core library tests + run: | + cargo test --package trusted-server-core --target "$(rustc -vV | sed -n 's/host: //p')" + + # The seam-probe fixture crate's own unit tests only run when invoked on + # the host target, because no adapter alias compiles its test target. Run + # them here so a fault in the seam fixture cannot sit unseen behind a + # green gate set. + - name: Run host-target seam-probe integration tests + run: | + cargo test --package trusted-server-integration-seam-probe --target "$(rustc -vV | sed -n 's/host: //p')" + - name: Verify Fastly WASM release build run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 test-cloudflare: - name: cargo check (cloudflare native + wasm32-unknown-unknown) - runs-on: ubuntu-latest + name: cargo check (cloudflare native + wasm32-unknown-unknown, ${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -322,3 +354,9 @@ jobs: - name: Run unit tests run: npm test -- --run + + # A module that writes a script into pages tests it in its own folder, + # with Node's test runner and no packages. + - name: Run module page script tests + working-directory: . + run: node --test "crates/*/*/tests/*.test.mjs" diff --git a/.gitignore b/.gitignore index 24b9e06aa1..a4a0af36c7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ /spin /spin.sig +# logs +*.log + # EdgeZero local KV store (created by edgezero-adapter-axum framework) .edgezero/ /dist/prebid/ diff --git a/AGENTS.md b/AGENTS.md index dd1dd52337..a2691b7afa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,12 +15,29 @@ JavaScript injection. ``` crates/ - trusted-server-core/ # Core library — shared logic, integrations, HTML processing + trusted-server-core/ # Core library — shared logic, the integration seam, HTML processing trusted-server-adapter-fastly/ # Fastly Compute entry point (wasm32-wasip1 binary) trusted-server-adapter-axum/ # Axum dev server entry point (native binary) trusted-server-adapter-cloudflare/ # Cloudflare Workers entry point (wasm32-unknown-unknown binary) trusted-server-adapter-spin/ # Fermyon Spin entry point (wasm32-wasip1 component) trusted-server-cli/ # Host-target `ts` operator CLI + trusted-server-modules/ # the modules a stock build ships, in the order their hooks run + ad-server/ # ad server implementation crates (mock) + ads-txt/ # ads.txt line module crates (sellers) + ad-tag/ # ad tag module crates (google, which holds its diagnostics module too) + auction/ # auction module crates (aps, prebid, prebid-server) + audience/ # audience module crates (permutive) + bot-protection/ # bot protection module crates (datadome) + cmp/ # consent tool module crates (didomi, inmobi, osano, sourcepoint) + framework/ # framework module crates (nextjs) + identity/ # identity module crates (lockr) + tag/ # tag module crates (google-tag, google-tag-manager) + testing/ # test module crates (seam-probe, testlight) + device/ + fastly/ # trusted-server-device-fastly (opt-in TLS/H2 device module) + edgecookie/ # vendor Edge Cookie module crates (built-in HMAC module is in core) + geo/ # vendor geo module crates (host geo is injected by the adapter) + permission-signal/ # permission signal module crates, one per scheme (gpc, gpp, tcf, us-privacy), and core links none trusted-server-js/ # TypeScript/JS build — per-integration IIFE bundles lib/ # TS source, Vitest tests, esbuild pipeline ``` @@ -61,7 +78,9 @@ fastly compute serve # Deploy to Fastly fastly compute publish -# Run Axum dev server (native — no Viceroy) +# Run Axum dev server (native — no Viceroy). Settings load at runtime from the +# platform config store on every adapter; publish an operator config with +# `ts config push` (see trusted-server.example.toml for the template). cargo run -p trusted-server-adapter-axum # Test Axum adapter only @@ -160,6 +179,24 @@ cd crates/trusted-server-js/lib && node build-all.mjs cargo install viceroy --version 0.17.0 --locked --force ``` +### Windows (use WSL for the Linux-only tests) + +The Rust adapter tests run natively on Windows through the cargo aliases +(`cargo test-fastly` via Viceroy, `cargo test-axum`, `cargo test-cloudflare`), +and CI runs them on `ubuntu-latest`. + +The Docker-based integration suite (`scripts/integration-tests.sh`) and the +Cloudflare worker build (`crates/trusted-server-adapter-cloudflare/build.sh`, +which uses `worker-build` + `wrangler dev`) are Linux tools. On Windows run them +inside WSL (Ubuntu) with Docker Desktop's WSL integration enabled. Provision the +WSL distro with the same toolchain as `.tool-versions` (rustup + the +`wasm32-wasip1` / `wasm32-unknown-unknown` targets, Node, Viceroy, wrangler), then +run the scripts from a clone on the WSL native filesystem for fast builds. +The CLI crate (`trusted-server-cli`) does not build on a Windows host either, +because its dependency `edgezero-adapter-fastly` uses a standard library +feature that is unstable on Windows, so run its tests and the template cache +harness (`scripts/template-cache-local-test.sh`), which builds it, in WSL too. + --- ## Coding Conventions @@ -287,12 +324,48 @@ impl core::error::Error for MyError {} ## Other guidelines +- Use US English spelling everywhere: code, identifiers, comments, + documentation, tests, commit messages, and configuration. For example, write + `color`, `behavior`, and `optimize`, not `colour`, `behaviour`, or `optimise`. + Where a term comes from an external source (for example the IAB TCF purpose + names), match that source's spelling even when it is not US English. - Use only example or fictional information in comments, tests, docs, examples, and similar non-runtime materials. (eg. for urls use: example.com domains only) - Do not write or commit real domains, customer names, credentials, configuration values, or other potentially sensitive real-world information in comments, tests, docs, or examples. +### Permission model terminology + +Permissions are the primitive. A module declares the permissions it requires +(`required_permissions`) and the system decides whether each is _set_. Consent +is only one of many ways a permission may be established. Country or +jurisdiction rules (a `Granted` group baseline), legitimate interest, or +configuration can set a permission with no consent at all. + +- A module that needs nothing **requires no permission**. Never write that it + "runs without any consent". +- A gated module **runs once its required permissions are set**, by whatever + method. + +**Evidence is not rationed, use is.** Every module and every integration sees +all the evidence available for a request, including host signals such as the TLS +JA4 and HTTP/2 signals. The core never decides which vendor may see what, +because withholding a signal from one vendor and not another discriminates +between them, and the core stays neutral. What a vendor may _do_ with the +evidence is governed by the permissions it declares and the system sets. Access +is universal, use is gated. + +The practical consequence: never "fix" a vendor's access to a signal by hiding +the signal. If a use needs controlling, express it as a permission. A change +that removes evidence from a module's reach is working against the +architecture, not protecting it. + +- Reserve "consent" for the consent subsystem (`consent/`, `ConsentContext`, + GDPR and TCF strings) where it genuinely means a consent signal. In the + permission layer prefer "permission", "set" / "unset", and "signal" (consent + is one kind of signal, alongside privacy and opt-out signals). + --- ## Git Commit Conventions @@ -309,6 +382,54 @@ Bad: `"fix: added feature flags"` --- +## Module Architecture + +Each vendor-differentiated capability is pluggable behind its own trait, so a +deployment selects an implementation and the core stays neutral. Every +pluggable thing follows one configuration convention, which is a top-level +table named for the job, a `module` key that selects what runs (`modules` +where several run), and a `[.]` +table for a selected name that has settings. The rules and what +is checked when are in `docs/guide/configuration-rules.md`, which is the page +to read before changing any module configuration. + +| Capability | Trait | Selector | Built-in (core) | Vendor / host crates | +| --------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| Edge Cookie identity | `EdgeCookieModule` (`ec/module.rs`) | `[ec] module` | `hmac`, `host_signals`, `client_fixed` (opt-in, no default) | `crates/edgecookie/` | +| Device detection | `DeviceModule` (`ec/device.rs`) | `[device] module` | `builtin`, User-Agent only (the default) | `crates/device/` | +| Geo / IP intelligence | `PlatformGeo` (`platform/traits.rs`) | `[geo] module` | None, no location (the default), or `platform` | `crates/geo/` | +| Permission signals | `PermissionSignalModule` (`permission_signal/mod.rs`) | `[permission-signal] modules` (an ordered list) | `gpc`, `gpp`, `us-privacy`, `tcf`, all of them with no list | `crates/permission-signal/` | +| Auction demand | `DemandImplementation` (`auction/demand.rs`) | `[demand] modules` (a list) | `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` | an integration builder | +| Ad server | `AdServerImplementation` (`auction/demand.rs`) | `[ad-server] module` | `mock` (`ad-server.mock`) | an integration builder | +| Page integrations | `IntegrationBuilder` (`integrations/mod.rs`) | the section of the module's type: `[cmp] module`, `[tag] modules`, `[ad-tag] modules`, `[bot-protection] module`, `[identity] module`, `[audience] module`, `[framework] module`, `[auction] modules`, `[proxy] modules` | `cmp.didomi`, `cmp.sourcepoint`, `cmp.osano`, `tag.google-tag-manager`, `ad-tag.google`, `ad-tag.google.diagnostics`, `bot-protection.datadome`, `identity.lockr`, `audience.permutive`, `framework.nextjs`, `auction.prebid`, `testing.testlight`, `js_asset_proxy` | an adapter-supplied builder | + +`auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` supply implementations +only. No module section selects them. + +Principles for adding or changing a module: + +- **Core stays neutral.** The trait and the host-neutral default live in + `trusted-server-core`. Host-specific and vendor implementations live in their + own crates and are injected by the adapter (for example `build_device_module` + and `build_geo_module`), so core never depends on a host SDK or a vendor, and + the default request path makes no host-specific calls. +- **Modules read request evidence, not a fixed parameter set.** A module must + be able to see everything about the request it needs (User-Agent, headers, and + host signals such as the TLS JA4 and HTTP/2 signals) through an evidence + abstraction rather than a hard-coded struct of fields. Host signals come from + the host (the Fastly SDK) and are opt-in, so a neutral module triggers no + host signal calls. +- **Modules are separated by capability but composed per request, and one may + need another's output.** Geo resolves the country and region the permission + model uses, and the permission model gates whether the Edge Cookie module + runs. Device signals gate Edge Cookie writes (the browser / bot gate). When + multiple vendor modules share a backend (for example a vendor's Edge Cookie, + geo, and device module on one cloud pipeline) they share a single call per + request rather than calling independently. Give a module the inputs and + upstream results it needs explicitly, rather than having it reach into globals. + +--- + ## Integration System Integrations register in Rust via: @@ -316,13 +437,13 @@ Integrations register in Rust via: ```rust IntegrationRegistration::builder(ID) .with_proxy() - .with_attribute_rewriter() - .with_head_injector() + .with_middleware() .build() ``` - Integration IDs match JS directory names: `prebid` (deferred), `lockr`, `permutive`, `datadome`, `didomi`, `testlight`. -- `creative` is JS-only (no Rust registration); `nextjs`, `aps`, `adserver_mock` are Rust-only. +- An integration runs when the section of its type selects its module. A module with a crate of its own is named by that crate's path under `crates/` (`permission-signal.gpc`, `testing.seam-probe`). Each page integration lives in a crate of its own and holds that name in a `MODULE` constant (`cmp.didomi`, `tag.google-tag-manager`, `ad-tag.google`, `bot-protection.datadome`, `identity.lockr`, `audience.permutive`, `framework.nextjs`, `auction.prebid`, `testing.testlight`), which a test in the crate holds to the crate's folder. A name is written in its section with the type left off. There is no `enabled` flag and no `[integration]` table. +- `creative` is JS-only (no Rust registration), and `nextjs` is Rust-only. `auction-protocol.openrtb`, `auction.prebid-server`, `auction.aps` and `ad-server.mock` register demand or ad server implementations rather than page integrations. - Integrations opt into deferred loading via `.with_deferred_js()` on the registration builder. Deferred modules are served as separate `", - tsjs::tsjs_single_module_script_src(GPT_DIAGNOSTICS_INTEGRATION_ID) + tsjs::tsjs_single_module_script_src(module) ) }) } - /// An active decision, for tests in other modules that need one. - /// - /// The fields are private and built by `prepare_request` from a cookie or query - /// parameter; there is no other way to obtain an active decision across a module - /// boundary. - #[cfg(test)] - pub(crate) fn active_for_tests() -> Self { - Self { - active: true, - clean_browser_path_and_query: None, - cookie_action: GptDiagnosticsCookieAction::None, - } - } } #[cfg(test)] mod head_seam_invariant_tests { use super::*; + /// The compile-time diagnostics module, as the registry would serve it. + fn diagnostics_part() -> JsModulePart { + JsModulePart::compile_time(GPT_DIAGNOSTICS_INTEGRATION_ID) + .expect("should have compiled the diagnostics module in") + } + /// Every combination of the three fields the decision carries. fn all_decisions() -> Vec { let mut out = Vec::new(); @@ -162,9 +180,10 @@ mod head_seam_invariant_tests { // // If a future change makes a script emit without also requiring the stamp, // this fails here rather than silently in a cached template. + let module = diagnostics_part(); for decision in all_decisions() { - let injects = - decision.bootstrap_script().is_some() || decision.module_script_tag().is_some(); + let injects = decision.bootstrap_script().is_some() + || decision.module_script_tag(&module).is_some(); if injects { assert!( decision.requires_private_no_store(), @@ -184,7 +203,7 @@ mod head_seam_invariant_tests { "should not inject a bootstrap for an inert decision" ); assert_eq!( - decision.module_script_tag(), + decision.module_script_tag(&diagnostics_part()), None, "should not inject a module for an inert decision" ); @@ -209,7 +228,20 @@ struct ConsoleCookieState { canonical: bool, } -/// Register GPT diagnostics when explicitly enabled. +/// Validates the GPT diagnostics configuration for deployment and reports whether +/// a section selects the integration's module. +/// +/// # Errors +/// +/// Returns an error when the GPT diagnostics configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) +} + +/// Register GPT diagnostics when a section selects it. /// /// # Errors /// @@ -218,32 +250,59 @@ struct ConsoleCookieState { pub fn register( settings: &Settings, ) -> Result, Report> { - let Some(_config) = - settings.integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID)? - else { + let Some(_config) = settings.module_config::(MODULE)? else { return Ok(None); }; Ok(Some( IntegrationRegistration::builder(GPT_DIAGNOSTICS_INTEGRATION_ID) - .without_js() + .with_standalone_js() + .with_middleware(Arc::new(DiagnosticsHead)) .build(), )) } -/// Whether the diagnostics integration is present and enabled in configuration. +/// Writes what an active decision adds to a reader's copy of a document, on +/// the pages a `[[serve]]` entry names [`MODULE`] for. /// -/// This is the deployment-level switch, not the per-document activation state: -/// callers that only need to know whether diagnostics could consume a value use -/// this, while document behaviour uses [`GptDiagnosticsRequestDecision::active`]. -/// A configuration that cannot be parsed reads as disabled. -#[must_use] -pub fn is_enabled(settings: &Settings) -> bool { - settings - .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID) - .ok() - .flatten() - .is_some() +/// It runs for each reader, and not once for a stored page, because the +/// decision is made for one request. +struct DiagnosticsHead; + +impl DiagnosticsHead { + /// The decision the preparer left for this document, when it left one. + fn decision( + document_state: &IntegrationDocumentState, + ) -> Option> { + document_state.get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + } +} + +impl Middleware for DiagnosticsHead { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + /// The bootstrap goes ahead of the bundle. The module loads synchronously + /// after the bundle, so its listeners are in place before the publisher's + /// scripts in the origin head run. + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let Some(decision) = Self::decision(context.document_state) else { + return MiddlewareAction::pass(); + }; + MiddlewareAction { + head_inserts: decision.bootstrap_script().into_iter().collect(), + after_bundle_inserts: JsModulePart::compile_time(GPT_DIAGNOSTICS_INTEGRATION_ID) + .and_then(|module| decision.module_script_tag(&module)) + .into_iter() + .collect(), + ..MiddlewareAction::pass() + } + } } /// Evaluate activation and sanitize the request before generic cookie handling. @@ -264,15 +323,15 @@ pub fn prepare_request( return Ok(existing.clone()); } - let integration_enabled = settings - .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID)? + let integration_runs = settings + .module_config::(MODULE)? .is_some(); let (directive, clean_path, had_reserved_query) = console_query(request.uri()); let cookie_state = console_cookie_state(request); let eligible_navigation = request.method() == Method::GET && is_navigation_request(request) - && !crate::publisher::is_prefetch_request(request) - && !crate::publisher::is_bot_user_agent(request); + && !trusted_server_core::publisher::is_prefetch_request(request) + && !trusted_server_core::publisher::is_bot_user_agent(request); sanitize_console_cookie(request); if had_reserved_query { @@ -280,7 +339,7 @@ pub fn prepare_request( } let mut decision = GptDiagnosticsRequestDecision::default(); - if integration_enabled && eligible_navigation && had_reserved_query { + if integration_runs && eligible_navigation && had_reserved_query { decision.clean_browser_path_and_query = Some(clean_path); match directive { QueryDirective::Enable => { @@ -292,7 +351,7 @@ pub fn prepare_request( } QueryDirective::Invalid | QueryDirective::Absent => {} } - } else if integration_enabled + } else if integration_runs && directive == QueryDirective::Absent && cookie_state.occurrences == 1 && cookie_state.canonical @@ -302,17 +361,38 @@ pub fn prepare_request( } request.extensions_mut().insert(decision.clone()); + // Only a decision that changes the document or the response is left for + // the page path, so an ordinary request stays on the shared template path. + if decision.requires_private_no_store() { + IntegrationRequestState::insert(request, GPT_DIAGNOSTICS_INTEGRATION_ID, decision.clone()); + } Ok(decision) } -/// Read the request decision, defaulting to inactive when not prepared. -#[must_use] -pub fn request_decision(request: &Request) -> GptDiagnosticsRequestDecision { - request - .extensions() - .get::() - .cloned() - .unwrap_or_default() +/// Builder hook: prepares the request and discards the decision, which stays +/// in the request extensions for the publisher path to read. +/// +/// # Errors +/// +/// Returns the error from [`prepare_request`]. +pub(crate) fn prepare_request_hook( + settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + prepare_request(settings, request).map(|_| ()) +} + +/// Builder hook: finalizes the response from the decision the preparer left +/// for the page path, and leaves the response alone when it left none. +pub(crate) fn finalize_response_hook( + request_state: &IntegrationRequestState, + response: &mut Response, +) { + if let Some(decision) = + request_state.get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + { + finalize_response(&decision, response); + } } /// Apply activation cookie and strict cache privacy to an origin response. @@ -339,7 +419,7 @@ pub fn finalize_response( // `RequestFilterEffects` mutation such as `Cache-Control: public` replaced it, // and the adapter's terminal guard had no marker to re-enforce from, so // request-scoped diagnostics HTML became shared-cacheable. - enforce_synthesized_html_cache_privacy(response); + enforce_terminal_private_cache_privacy(response); } } @@ -436,19 +516,20 @@ fn replace_path_and_query( #[cfg(test)] mod tests { use super::*; - use crate::integrations::IntegrationRegistry; - use crate::test_support::tests::create_test_settings; use serde_json::json; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::test_support::tests::create_test_settings; - fn settings(enabled: bool) -> Settings { + /// Settings that select the module and run its middleware on every + /// reader's copy of a page. + fn settings() -> Settings { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - GPT_DIAGNOSTICS_INTEGRATION_ID, - &json!({ "enabled": enabled }), - ) - .expect("should insert diagnostics config"); + settings.select_module("ad-tag", "ad-tag.google.diagnostics"); + trusted_server_core::html_processor::test_support::place_on_every_page( + &mut settings, + MiddlewarePhase::Serve, + &[MODULE], + ); settings } @@ -467,14 +548,11 @@ mod tests { #[test] fn register_excludes_diagnostics_from_unified_and_deferred_bundles() { - let settings = settings(true); - let plan = std::sync::Arc::new( - crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"), - ); - let registry = - IntegrationRegistry::with_plan(&settings, plan).expect("should build registry"); + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); - assert!(registry.integration_enabled(GPT_DIAGNOSTICS_INTEGRATION_ID)); + assert!(registry.integration_runs(GPT_DIAGNOSTICS_INTEGRATION_ID)); assert!( !registry .js_module_ids_immediate() @@ -485,6 +563,10 @@ mod tests { .js_module_ids_deferred() .contains(&GPT_DIAGNOSTICS_INTEGRATION_ID) ); + assert!( + registry.js_part(GPT_DIAGNOSTICS_INTEGRATION_ID).is_some(), + "should serve the diagnostics module standalone" + ); } #[test] @@ -494,7 +576,7 @@ mod tests { Some("other=value; __Host-ts-console=1"), ); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(decision.active()); assert_eq!( @@ -518,7 +600,7 @@ mod tests { .headers_mut() .insert("purpose", HeaderValue::from_static("prefetch")); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!( !decision.active(), @@ -534,7 +616,7 @@ mod tests { "https://publisher.example/page", Some("__Host-ts-console=1; other=value"), ); - let decision = prepare_request(&settings(true), &mut active).expect("should prepare"); + let decision = prepare_request(&settings(), &mut active).expect("should prepare"); assert!(decision.active()); assert_eq!(active.headers()[header::COOKIE], "other=value"); @@ -542,7 +624,7 @@ mod tests { "https://publisher.example/page", Some("__Host-ts-console=1; __Host-ts-console=1; other=value"), ); - let decision = prepare_request(&settings(true), &mut duplicate).expect("should prepare"); + let decision = prepare_request(&settings(), &mut duplicate).expect("should prepare"); assert!(!decision.active()); assert_eq!(duplicate.headers()[header::COOKIE], "other=value"); } @@ -558,7 +640,7 @@ mod tests { &format!("https://publisher.example/page?{query}&keep=1"), Some("__Host-ts-console=1"), ); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(!decision.active(), "{query} should fail closed"); assert_eq!(decision.cookie_action, GptDiagnosticsCookieAction::None); assert_eq!(request.uri().query(), Some("keep=1")); @@ -568,7 +650,7 @@ mod tests { "https://publisher.example/page?ts_console=false&keep=1", Some("__Host-ts-console=1"), ); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(!decision.active()); assert_eq!( decision.cookie_action, @@ -579,7 +661,7 @@ mod tests { #[test] fn finalization_sets_cookie_and_strips_shared_cache_headers() { let mut request = navigation("https://publisher.example/?ts_console=1", None); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); let mut response = Response::builder() .header(header::CACHE_CONTROL, "public, max-age=60") .header(header::ETAG, "\"origin\"") @@ -622,7 +704,7 @@ mod tests { // because the adapter's terminal guard keys on the marker, not on the stamp, and // the `Set-Cookie` privacy net never sees a response that sets no cookie. let mut request = navigation("https://publisher.example/", Some("__Host-ts-console=1")); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(decision.active(), "the session cookie should activate"); assert_eq!( decision.cookie_action, @@ -642,7 +724,7 @@ mod tests { assert!( response .extensions() - .get::() + .get::() .is_some(), "should mark request-scoped diagnostics HTML for terminal re-enforcement" ); @@ -651,7 +733,7 @@ mod tests { #[test] fn an_inactive_decision_leaves_the_origin_cache_policy_alone() { let mut request = navigation("https://publisher.example/", None); - let decision = prepare_request(&settings(true), &mut request).expect("should prepare"); + let decision = prepare_request(&settings(), &mut request).expect("should prepare"); assert!(!decision.requires_private_no_store()); let mut response = Response::builder() .header(header::CACHE_CONTROL, "public, max-age=60") @@ -669,7 +751,7 @@ mod tests { assert!( response .extensions() - .get::() + .get::() .is_none(), "should not mark an untouched response terminal-private" ); @@ -679,17 +761,188 @@ mod tests { fn config_rejects_unknown_fields() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - GPT_DIAGNOSTICS_INTEGRATION_ID, - &json!({ "enabled": true, "typo": true }), + .insert_module_config( + "ad-tag", + "ad-tag.google.diagnostics", + &json!({"typo": true }), ) .expect("should insert diagnostics config"); let error = settings - .integration_config::(GPT_DIAGNOSTICS_INTEGRATION_ID) + .module_config::(MODULE) .expect_err("should reject unknown diagnostics config fields"); let error_text = format!("{error:?}"); assert!(error_text.contains("typo") || error_text.contains("unknown field")); } + + #[test] + fn module_constant_names_the_crate_folder_and_the_module_within_it() { + assert_eq!( + MODULE, + format!("{}.diagnostics", trusted_server_core::module_name!()), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn only_a_decision_that_acts_is_left_for_the_page_path() { + let mut ordinary = navigation("https://publisher.example/page", None); + prepare_request(&settings(), &mut ordinary).expect("should prepare"); + assert!( + IntegrationRequestState::of(&ordinary).is_empty(), + "an ordinary request should stay on the shared template path" + ); + + let mut activating = navigation("https://publisher.example/page?ts_console=1", None); + let decision = prepare_request(&settings(), &mut activating).expect("should prepare"); + assert_eq!( + IntegrationRequestState::of(&activating) + .get::(GPT_DIAGNOSTICS_INTEGRATION_ID) + .as_deref(), + Some(&decision), + "should leave the decision that acts for the page path" + ); + } + + #[test] + fn the_finalizer_acts_only_on_a_decision_left_for_the_page_path() { + let mut untouched = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + finalize_response_hook(&IntegrationRequestState::default(), &mut untouched); + assert_eq!( + untouched.headers()[header::CACHE_CONTROL], + "public, max-age=60", + "should leave a response alone when nothing was left for it" + ); + assert!( + !untouched.headers().contains_key(header::SET_COOKIE), + "should set no cookie when nothing was left for it" + ); + + let mut request = navigation("https://publisher.example/?ts_console=1", None); + prepare_request(&settings(), &mut request).expect("should prepare"); + let mut response = Response::builder() + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + finalize_response_hook(&IntegrationRequestState::of(&request), &mut response); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "no-store, private", + "should make the activating response private" + ); + assert_eq!( + response.headers()[header::SET_COOKIE], + SET_CONSOLE_COOKIE, + "should establish the session" + ); + } + + #[test] + fn an_active_document_loads_the_module_after_the_unified_bundle_once() { + use trusted_server_core::html_processor::HtmlProcessorConfig; + use trusted_server_core::html_processor::test_support::create_page_processor; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); + let mut request = navigation("https://publisher.example/page?ts_console=1", None); + let decision = prepare_request(&settings, &mut request).expect("should prepare"); + assert!( + decision.active(), + "should activate diagnostics for the query" + ); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "publisher.example", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_page_processor(&settings, ®istry, config); + + let output = processor + .process_chunk( + b"Test", + true, + ) + .expect("should process HTML"); + let processed = String::from_utf8(output).expect("should produce valid UTF-8"); + let bootstrap_marker = "__tsjs_gpt_diagnostics_active"; + let bundle_marker = "id=\"trustedserver-js\""; + let diagnostics_marker = "tsjs-gpt_diagnostics.min.js"; + + assert_eq!( + processed.matches(bootstrap_marker).count(), + 1, + "should inject the diagnostics bootstrap once" + ); + assert_eq!( + processed.matches(bundle_marker).count(), + 1, + "should inject the immediate TSJS bundle once" + ); + assert_eq!( + processed.matches(diagnostics_marker).count(), + 1, + "should inject one standalone diagnostics module" + ); + let bootstrap_index = processed + .find(bootstrap_marker) + .expect("should include diagnostics bootstrap"); + let bundle_index = processed + .find(bundle_marker) + .expect("should include immediate TSJS bundle"); + let diagnostics_index = processed + .find(diagnostics_marker) + .expect("should include standalone diagnostics module"); + assert!( + bootstrap_index < bundle_index, + "should activate before core executes" + ); + assert!( + bundle_index < diagnostics_index, + "should load diagnostics after core" + ); + } + + #[test] + fn an_inactive_document_gets_neither_script() { + use trusted_server_core::html_processor::HtmlProcessorConfig; + use trusted_server_core::html_processor::test_support::create_page_processor; + use trusted_server_core::streaming_processor::StreamProcessor as _; + + let settings = settings(); + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should build registry"); + let mut request = navigation("https://publisher.example/page", None); + prepare_request(&settings, &mut request).expect("should prepare"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "publisher.example", + "https", + ) + .with_request_state(IntegrationRequestState::of(&request)); + let mut processor = create_page_processor(&settings, ®istry, config); + + let output = processor + .process_chunk( + b"Test", + true, + ) + .expect("should process HTML"); + let processed = String::from_utf8(output).expect("should produce valid UTF-8"); + + assert!( + !processed.contains("__tsjs_gpt_diagnostics_active") + && !processed.contains("tsjs-gpt_diagnostics.min.js"), + "should write nothing into a document diagnostics is not active for: {processed}" + ); + } } diff --git a/crates/ad-tag/google/src/fixtures/page-change.input.html b/crates/ad-tag/google/src/fixtures/page-change.input.html new file mode 100644 index 0000000000..6a9f667691 --- /dev/null +++ b/crates/ad-tag/google/src/fixtures/page-change.input.html @@ -0,0 +1,17 @@ + + + + +Google Publisher Tag + + + + + + + +An article on the origin + +
+ + diff --git a/crates/ad-tag/google/src/fixtures/page-change.recorded.html b/crates/ad-tag/google/src/fixtures/page-change.recorded.html new file mode 100644 index 0000000000..a6f6a55c05 --- /dev/null +++ b/crates/ad-tag/google/src/fixtures/page-change.recorded.html @@ -0,0 +1,963 @@ + + + + +Google Publisher Tag + + + + + + + +An article on the origin + +
+ + diff --git a/crates/ad-tag/google/src/fixtures/page-change.settings.toml b/crates/ad-tag/google/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..50ef4c9659 --- /dev/null +++ b/crates/ad-tag/google/src/fixtures/page-change.settings.toml @@ -0,0 +1,13 @@ +[ad-tag] +modules = ["google", "google.diagnostics"] + +[ad-tag.google] +gam_attribution_enabled = false + +[[fetch]] +media_type = "text/html" +middleware = ["ad-tag.google"] + +[[serve]] +media_type = "text/html" +middleware = ["ad-tag.google.diagnostics"] diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/ad-tag/google/src/gpt_bootstrap.js similarity index 97% rename from crates/trusted-server-core/src/integrations/gpt_bootstrap.js rename to crates/ad-tag/google/src/gpt_bootstrap.js index afae775350..374f7bbb5c 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/ad-tag/google/src/gpt_bootstrap.js @@ -358,6 +358,30 @@ ); } + // Defines the slot and applies its viewport bands as a size mapping. + function defineWithViewportSizes(api, slot, elementId) { + var definedSlot = api.defineSlot( + slot.gam_unit_path, + slot.formats, + elementId, + ); + var bands = slot.viewport_sizes; + if ( + definedSlot && + bands && + bands.length && + api.sizeMapping && + definedSlot.defineSizeMapping + ) { + var builder = api.sizeMapping(); + bands.forEach(function (band) { + builder.addSize(band[0], band[1]); + }); + definedSlot.defineSizeMapping(builder.build()); + } + return definedSlot; + } + function normalizedGptFormats(formats) { return formats.length === 2 && formats.every(function (format) { @@ -680,11 +704,7 @@ var tsOwned = false; if (!gptSlot) { gptSlot = runHandoffInternal(function () { - return window.googletag.defineSlot( - slot.gam_unit_path, - slot.formats, - element.id, - ); + return defineWithViewportSizes(window.googletag, slot, element.id); }); if (!gptSlot) { releaseTrustedServerFirstImpressionClaim(element, fallbackClaim); @@ -832,11 +852,7 @@ // Define TS's fallback on the publisher's actual div. The scoped // handoff wrapper returns this slot if the publisher defines it later. s = runHandoffInternal(function () { - return googletag.defineSlot( - slot.gam_unit_path, - slot.formats, - actualDivId, - ); + return defineWithViewportSizes(googletag, slot, actualDivId); }); if (!s) { releaseTrustedServerFirstImpressionClaim(el, tsClaim); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/ad-tag/google/src/lib.rs similarity index 79% rename from crates/trusted-server-core/src/integrations/gpt.rs rename to crates/ad-tag/google/src/lib.rs index 3292cc5b3a..c0af46e057 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/ad-tag/google/src/lib.rs @@ -18,7 +18,7 @@ //! //! ## How It Works //! -//! 1. **HTML rewriting** – The [`IntegrationAttributeRewriter`] swaps `src`/`href` +//! 1. **HTML rewriting** – The module's middleware swaps `src`/`href` //! attributes pointing at Google's GPT script with a first-party URL //! (`/integrations/gpt/script`). //! 2. **Script proxy** – [`IntegrationProxy`] endpoints serve `gpt.js` @@ -31,7 +31,25 @@ //! `" + )); + } + scripts } - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - if self.config.gam_attribution_enabled { - vec![("data-ts-gam-attribution", "true")] - } else { - Vec::new() - } + /// The attribute on the script bundle's tag that lets the bundle apply + /// GAM attribution, when `gam_attribution_enabled` is set. + fn bundle_tag_attribute(&self) -> Option<(&'static str, &'static str)> { + self.config + .gam_attribution_enabled + .then_some(("data-ts-gam-attribution", "true")) } } @@ -541,10 +621,6 @@ const GPT_BOOTSTRAP_JS: &str = include_str!("gpt_bootstrap.js"); // Default value functions -fn default_enabled() -> bool { - true -} - fn default_script_url() -> String { "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string() } @@ -560,30 +636,30 @@ fn default_rewrite_script() -> bool { #[cfg(test)] mod tests { use super::*; - use crate::constants::HEADER_X_FORWARDED_FOR; - use crate::integrations::IntegrationDocumentState; - use crate::test_support::tests::create_test_settings; use http::Method; + use trusted_server_core::constants::HEADER_X_FORWARDED_FOR; + use trusted_server_core::integrations::IntegrationDocumentState; + use trusted_server_core::test_support::tests::create_test_settings; fn test_config() -> GptConfig { GptConfig { - enabled: true, gam_attribution_enabled: false, script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, slim_prebid_url: None, + write_loader: false, } } - fn test_context() -> IntegrationAttributeContext<'static> { - IntegrationAttributeContext { - attribute_name: "src", - element_name: "script", - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - } + /// What the module's middleware decides to do with a page. + fn page_action(integration: Arc) -> MiddlewareAction { + let document_state = IntegrationDocumentState::default(); + let context = trusted_server_core::middleware::test_support::context( + MiddlewarePhase::Fetch, + &document_state, + ); + PageChange(integration).create(&context) } fn build_http_request(method: Method, uri: &str) -> http::Request { @@ -667,17 +743,12 @@ mod tests { ); } - // -- Attribute rewriter -- + // -- Element handler -- #[test] - fn attribute_rewriter_rewrites_gpt_urls() { - let integration = GptIntegration::new(test_config()); - let ctx = test_context(); - - let result = integration.rewrite( - "src", + fn element_handler_rewrites_gpt_urls() { + let result = GptIntegration::rewrite_script_address( "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - &ctx, ); match result { @@ -692,11 +763,8 @@ mod tests { } #[test] - fn attribute_rewriter_keeps_non_gpt_urls() { - let integration = GptIntegration::new(test_config()); - let ctx = test_context(); - - let result = integration.rewrite("src", "https://cdn.example.com/analytics.js", &ctx); + fn element_handler_keeps_non_gpt_urls() { + let result = GptIntegration::rewrite_script_address("https://cdn.example.com/analytics.js"); assert_eq!( result, @@ -706,50 +774,45 @@ mod tests { } #[test] - fn attribute_rewriter_noop_when_disabled() { + fn element_handler_noop_when_disabled() { let config = GptConfig { rewrite_script: false, ..test_config() }; - let integration = GptIntegration::new(config); - let ctx = test_context(); + let action = page_action(GptIntegration::new(config)); - let result = integration.rewrite( - "src", - "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - &ctx, + assert!( + action.element_handlers.is_empty(), + "should judge no address when rewrite_script is disabled" ); - assert_eq!( - result, - AttributeRewriteAction::Keep, - "should keep GPT URLs when rewrite_script is disabled" + action.head_inserts.len(), + 2, + "should still write the bootstrap into the head" ); } #[test] fn handles_attribute_respects_config() { - let enabled = GptIntegration::new(test_config()); - assert!( - enabled.handles_attribute("src"), - "should handle src when rewrite_script is true" - ); - assert!( - enabled.handles_attribute("href"), - "should handle href when rewrite_script is true" - ); - assert!( - !enabled.handles_attribute("action"), - "should not handle action attribute" + let enabled = page_action(GptIntegration::new(test_config())); + let judged: Vec<&str> = enabled + .element_handlers + .iter() + .map(|handler| handler.attribute()) + .collect(); + assert_eq!( + judged, + ["src", "href"], + "should judge src and href, and no other attribute, when rewrite_script is true" ); - let disabled = GptIntegration::new(GptConfig { + let disabled = page_action(GptIntegration::new(GptConfig { rewrite_script: false, ..test_config() - }); + })); assert!( - !disabled.handles_attribute("src"), - "should not handle src when rewrite_script is false" + disabled.element_handlers.is_empty(), + "should judge no attribute when rewrite_script is false" ); } @@ -1066,11 +1129,10 @@ mod tests { fn build_with_valid_config() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - GPT_INTEGRATION_ID, + .insert_module_config( + "ad-tag", + "ad-tag.google", &serde_json::json!({ - "enabled": true, "script_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", "cache_ttl_seconds": 3600, "rewrite_script": true @@ -1087,23 +1149,14 @@ mod tests { } #[test] - fn build_disabled_returns_none() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - GPT_INTEGRATION_ID, - &serde_json::json!({ - "enabled": false - }), - ) - .expect("should insert GPT config"); + fn build_without_the_integration_named_returns_none() { + let settings = create_test_settings(); assert!( build(&settings) .expect("should evaluate integration build") .is_none(), - "should not build when integration is disabled" + "should not build an integration no section selects" ); } @@ -1169,20 +1222,12 @@ mod tests { ); } - // -- Head injector -- + // -- Head markup -- #[test] - fn head_injector_emits_enable_flag() { + fn head_markup_emits_enable_flag() { let integration = GptIntegration::new(test_config()); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - - let inserts = integration.head_inserts(&ctx); + let inserts = integration.head_markup(); assert_eq!(inserts.len(), 2, "should emit exactly two head inserts"); assert_eq!( @@ -1191,7 +1236,7 @@ mod tests { "should set the enable flag and call the GPT shim activation function" ); assert!( - integration.tsjs_script_tag_attributes().is_empty(), + integration.bundle_tag_attribute().is_none(), "should not authorize GAM attribution metadata by default" ); } @@ -1202,15 +1247,7 @@ mod tests { gam_attribution_enabled: true, ..test_config() }); - let document_state = IntegrationDocumentState::default(); - let context = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - document_state: &document_state, - }; - - let inserts = integration.head_inserts(&context); + let inserts = integration.head_markup(); assert_eq!(inserts.len(), 2, "should not add another head insert"); assert!( @@ -1218,8 +1255,8 @@ mod tests { "should activate the early bootstrap marker" ); assert_eq!( - integration.tsjs_script_tag_attributes(), - vec![("data-ts-gam-attribution", "true")], + integration.bundle_tag_attribute(), + Some(("data-ts-gam-attribution", "true")), "should authorize the bundle fallback on the publisher tag" ); } @@ -1228,14 +1265,7 @@ mod tests { fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { let config = test_config(); let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - let inserts = integration.head_inserts(&ctx); + let inserts = integration.head_markup(); let combined = inserts.join(""); assert!(combined.contains("ts.adInit"), "should define tsjs.adInit"); assert!( @@ -1286,14 +1316,7 @@ mod tests { // fallback lives in the Vitest suite (gpt_bootstrap.test.ts). let config = test_config(); let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - let combined = integration.head_inserts(&ctx).join(""); + let combined = integration.head_markup().join(""); assert!( combined.contains("ts.scheduleInitialAdInit"), "should install the fallback scheduler for bundle-load failures" @@ -1320,14 +1343,7 @@ mod tests { fn head_inserts_bootstrap_uses_css_safe_div_prefix_lookup() { let config = test_config(); let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - let combined = integration.head_inserts(&ctx).join(""); + let combined = integration.head_markup().join(""); assert!( combined.contains("querySelectorAll(\"[id]\")"), "bootstrap should scan ID-bearing elements instead of interpolating div_id into CSS" @@ -1345,14 +1361,7 @@ mod tests { #[test] fn head_inserts_bootstrap_installs_inner_div_slot_handoff() { let integration = GptIntegration::new(test_config()); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - let combined = integration.head_inserts(&ctx).join(""); + let combined = integration.head_markup().join(""); assert!( combined.contains("gptSlotHandoffs"), "bootstrap should keep late publisher slot handoff state on window.tsjs" @@ -1362,7 +1371,7 @@ mod tests { "bootstrap should install idempotent GPT handoff wrappers" ); assert!( - combined.contains("return googletag.defineSlot") && combined.contains("actualDivId"), + combined.contains("return defineWithViewportSizes(googletag, slot, actualDivId)"), "bootstrap should define the TS fallback on the actual inner div" ); assert!( @@ -1375,14 +1384,7 @@ mod tests { fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { let config = test_config(); let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - let combined = integration.head_inserts(&ctx).join(""); + let combined = integration.head_markup().join(""); assert!( combined.contains("ts.servicesEnabled"), "should guard enableServices/enableSingleRequest with the tsjs.servicesEnabled flag" @@ -1402,14 +1404,7 @@ mod tests { // slots or they render blank. let config = test_config(); let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - let combined = integration.head_inserts(&ctx).join(""); + let combined = integration.head_markup().join(""); assert!( combined.contains("gpt.setConfig"), "bootstrap should wrap googletag.setConfig() to detect the disabled state" @@ -1462,11 +1457,36 @@ mod tests { } #[test] - fn head_injector_integration_id() { + fn the_middleware_is_named_as_the_module() { let integration = GptIntegration::new(test_config()); + assert_eq!(PageChange(integration).middleware_id(), MODULE); + } + + #[test] + fn a_registration_states_the_attribution_attribute_only_when_it_is_enabled() { + let attributes_with = |gam_attribution_enabled: bool| { + let mut settings = create_test_settings(); + settings + .insert_module_config( + "ad-tag", + MODULE, + &serde_json::json!({ "gam_attribution_enabled": gam_attribution_enabled }), + ) + .expect("should select GPT"); + register(&settings) + .expect("should register GPT") + .expect("should run GPT") + .bundle_tag_attributes + }; + assert_eq!( - IntegrationHeadInjector::integration_id(integration.as_ref()), - "gpt" + attributes_with(true), + [("data-ts-gam-attribution", "true")], + "should put the attribute on the bundle's tag" + ); + assert!( + attributes_with(false).is_empty(), + "should put nothing on the bundle's tag by default" ); } @@ -1477,15 +1497,7 @@ mod tests { ..test_config() }; let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - - let inserts = integration.head_inserts(&ctx); + let inserts = integration.head_markup(); assert_eq!( inserts.len(), @@ -1507,15 +1519,7 @@ mod tests { ..test_config() }; let integration = GptIntegration::new(config); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - - let inserts = integration.head_inserts(&ctx); + let inserts = integration.head_markup(); // The injected `` must be neutralised: the only // `` left is the tag's own legitimate closer. @@ -1540,15 +1544,7 @@ mod tests { #[test] fn head_inserts_omits_slim_prebid_url_when_not_configured() { let integration = GptIntegration::new(test_config()); - let doc_state = IntegrationDocumentState::default(); - let ctx = IntegrationHtmlContext { - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "example.com", - document_state: &doc_state, - }; - - let inserts = integration.head_inserts(&ctx); + let inserts = integration.head_markup(); assert_eq!( inserts.len(), @@ -1562,4 +1558,82 @@ mod tests { "should not emit slim-Prebid URL tag when not configured" ); } + + #[test] + fn head_inserts_write_the_loader_tag_when_asked() { + let integration = GptIntegration::new(GptConfig { + write_loader: true, + ..test_config() + }); + let inserts = integration.head_markup(); + + assert_eq!( + inserts.len(), + 3, + "should add one head insert for the loader" + ); + assert_eq!( + inserts[2], r#""#, + "should ask for the loader on this module's own route" + ); + } + + #[test] + fn head_inserts_write_the_loader_tag_last() { + let integration = GptIntegration::new(GptConfig { + slim_prebid_url: Some("https://cdn.example.com/tsjs-prebid.min.js".to_string()), + write_loader: true, + ..test_config() + }); + let inserts = integration.head_markup(); + + assert_eq!( + inserts.len(), + 4, + "should add the loader after every other insert" + ); + assert!( + inserts[3].starts_with(" + + +An article on the origin + +
+ + diff --git a/crates/auction/aps/src/fixtures/page-change.recorded.html b/crates/auction/aps/src/fixtures/page-change.recorded.html new file mode 100644 index 0000000000..c33ea015d7 --- /dev/null +++ b/crates/auction/aps/src/fixtures/page-change.recorded.html @@ -0,0 +1,13 @@ + + + + +APS + + + +An article on the origin + +
+ + diff --git a/crates/auction/aps/src/fixtures/page-change.settings.toml b/crates/auction/aps/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..0bc023ecf3 --- /dev/null +++ b/crates/auction/aps/src/fixtures/page-change.settings.toml @@ -0,0 +1,9 @@ +[demand] +modules = ["aps_main"] + +[demand.aps_main] +implementation = "auction.aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +account_id = "example-aps-account-id" +rendering_mode = "publisher_native" diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/auction/aps/src/lib.rs similarity index 60% rename from crates/trusted-server-core/src/integrations/aps.rs rename to crates/auction/aps/src/lib.rs index 4f45fbeb9c..3d7b8b11f1 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/auction/aps/src/lib.rs @@ -1,5 +1,17 @@ //! Amazon Publisher Services (APS/TAM) `OpenRTB` integration. +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; #[cfg(test)] @@ -15,48 +27,76 @@ use serde::de::{self, Visitor}; use serde::{Deserialize, Serialize}; use serde_json::{Value as Json, json}; use url::Url; -use validator::{Validate, ValidationError}; - -use crate::auction::openrtb::ignored_bidder_params_count; -use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::plan::{AuctionPlanConfig, NotificationConfig, ProviderConfig, RoutingMode}; -use crate::auction::profile::ApsProfilePlan; +use validator::Validate; +use validator::ValidationError; + +use trusted_server_core::auction::demand::{ + CONSERVATIVE_LANGUAGE_MAX_BYTES, CompiledDemand, DemandFieldPolicy, DemandImplementation, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, +}; +use trusted_server_core::auction::openrtb::ignored_bidder_params_count; +use trusted_server_core::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::routing::ProviderAuctionInput; +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; #[cfg(test)] -use crate::auction::types::{AdSlot, AuctionContext, AuctionRequest}; -use crate::auction::types::{ - ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, MediaType, -}; -use crate::error::TrustedServerError; -use crate::integrations::{ - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, - IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +use trusted_server_core::auction::types::{AdSlot, AuctionContext, AuctionRequest}; +use trusted_server_core::auction::types::{AuctionResponse, Bid, BidRenderer, MediaType}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, + UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ +use trusted_server_core::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; #[cfg(test)] -use crate::openrtb::ToExt; +use trusted_server_core::openrtb::ToExt; #[cfg(test)] -use crate::openrtb::{ +use trusted_server_core::openrtb::{ Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, User, UserExt, to_openrtb_i32, }; +use trusted_server_core::platform::PlatformResponse; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::platform::{PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::settings::Settings; + +pub(crate) const APS_INTEGRATION_ID: &str = "aps"; + +/// The builder a deployment hands to an adapter. It offers APS to `[demand]` +/// and registers the page support its renderer needs when the auction plan +/// selects an APS source. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + APS_INTEGRATION_ID, + env!("CARGO_PKG_NAME"), + ) + .with_demand(&DEMAND) + .with_plan_registration(register_for_plan) +} + +/// The name an `implementation` line gives this implementation, its module +/// path. +pub const MODULE: &str = "auction.aps"; +/// Renderer type tag carried on the wire by an APS bid, read by the browser to +/// select the APS renderer. +pub const APS_RENDERER_TYPE: &str = "aps"; -const APS_INTEGRATION_ID: &str = "aps"; +/// Wire key carrying [`ApsRendererV1::bid_id`], for callers that read the bid +/// identifier out of a renderer payload without deserializing the rest. +/// +/// `ApsRendererV1` renames its fields to camelCase, so this is `bidId` rather +/// than the Rust field name. `renderer_bid_id_key_matches_the_serialized_form` +/// pins the two together. +pub const APS_RENDERER_BID_ID_KEY: &str = "bidId"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; const DEFAULT_CURRENCY: &str = "USD"; -#[cfg(test)] +/// The SDK source APS expects from a Prebid-shaped caller. const APS_SDK_SOURCE: &str = "prebid"; -#[cfg(test)] +/// The SDK version APS expects from a Prebid-shaped caller. const APS_SDK_VERSION: &str = "2.2.0"; const MAX_ACCOUNT_ID_BYTES: usize = 1024; const MAX_CREATIVE_ID_BYTES: usize = 1024; @@ -69,6 +109,45 @@ const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; +/// APS creative tag type accepted by the Trusted Server renderer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ApsTagType { + /// APS loads the creative URL in a nested iframe. + Iframe, + /// APS fetches creative HTML and executes it in its nested renderer frame. + Script, +} + +/// Version 1 APS renderer descriptor shared with browser clients. +/// +/// Carried by a bid as the payload of a [`BidRenderer`] tagged +/// [`APS_RENDERER_TYPE`], so it travels with the APS integration rather than +/// with the neutral auction types. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ApsRendererV1 { + /// Renderer contract version. + pub version: u8, + /// APS account identifier used to initialize the fixed runner. + pub account_id: String, + /// Selected `OpenRTB` bid identifier. + pub bid_id: String, + /// Optional `OpenRTB` creative identifier. + #[serde(skip_serializing_if = "Option::is_none")] + pub creative_id: Option, + /// APS creative delivery mode. + pub tag_type: ApsTagType, + /// HTTPS creative URL consumed by the fixed APS runner. + pub creative_url: String, + /// Base64-encoded exact one-bid APS response envelope. + pub aax_response: String, + /// Creative width. + pub width: u32, + /// Creative height. + pub height: u32, +} + const APS_RENDERER_DOCUMENT: &str = r#" @@ -374,80 +453,217 @@ impl Default for LegacyApsProviderConfig { } } -/// Browser integration toggle retained independently from APS server providers. -#[derive(Debug, Clone, Default, Deserialize, Serialize, Validate)] +/// The APS demand implementation. +pub static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Fixed(800), + allows_all_eligible: true, + serves_stored_requests: false, + canonicalize_endpoint: check_aps_endpoint, + compile: compile_demand, +}; + +/// One compiled APS demand source, from its `[demand.]` table. +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ApsConfig { - /// Whether browser-side APS integration behavior is enabled. +pub struct ApsDemand { + /// APS account identifier. + #[serde(deserialize_with = "deserialize_account_id")] + pub account_id: String, + /// Include APS request and response diagnostics. + /// + /// This default-off metadata is unredacted and client-visible, so it can + /// carry identity, consent, page, account, bid and creative data. Set it + /// only on a controlled test site and never in production. #[serde(default)] - pub enabled: bool, + pub debug: bool, + /// Permit APS script creatives. + #[serde(default)] + pub allow_script_creatives: bool, + /// APS-authorized inventory domain used instead of the deployment hostname. + #[serde(default)] + pub inventory_domain: Option, + /// Canonical inventory page origin, keeping the sanitized path. + #[serde(default)] + pub inventory_page_origin: Option, /// Rendering owner for selected APS bids. #[serde(default)] pub rendering_mode: ApsRenderingMode, } -#[cfg(test)] -impl IntegrationConfig for LegacyApsProviderConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} - -impl IntegrationConfig for ApsConfig { - fn is_enabled(&self) -> bool { - self.enabled +/// Refuse the legacy APS bid path, which this implementation does not speak. +fn check_aps_endpoint(endpoint: &mut Url) -> Result<(), String> { + if endpoint + .path() + .trim_end_matches('/') + .ends_with("/e/dtb/bid") + { + return Err("names the unsupported legacy APS path `/e/dtb/bid`".to_string()); } + Ok(()) } -/// Typed server-side APS profile configuration used by the auction compiler. -#[derive(Debug, Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct ApsProfileConfig { - #[serde(deserialize_with = "deserialize_account_id")] - pub(crate) account_id: String, - #[serde(default)] - pub(crate) debug: bool, - #[serde(default)] - pub(crate) allow_script_creatives: bool, - #[serde(default)] - pub(crate) inventory_domain: Option, - #[serde(default)] - pub(crate) inventory_page_origin: Option, +fn compile_demand( + settings: &serde_json::Map, +) -> Result, Report> { + Ok(Arc::new(compile_aps_settings(Json::Object( + settings.clone(), + ))?)) } -/// Parse and validate server-owned APS profile fields without browser enablement. -pub(crate) fn compile_profile_config( - value: serde_json::Value, -) -> Result> { - let profile: ApsProfileConfig = serde_json::from_value(value).map_err(|error| { +/// Parse and validate the settings of one APS demand source. +/// +/// # Errors +/// +/// Returns a configuration error when a setting is missing, unknown or invalid. +pub(crate) fn compile_aps_settings(value: Json) -> Result> { + let demand: ApsDemand = serde_json::from_value(value).map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config: {error}"), + message: format!("invalid `aps` settings: {error}"), }) })?; - if let Some(domain) = profile.inventory_domain.as_deref() { + if let Some(domain) = demand.inventory_domain.as_deref() { validate_inventory_domain(domain).map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config inventory_domain: {error}"), + message: format!("invalid `aps` inventory_domain: {error}"), }) })?; } - if let Some(origin) = profile.inventory_page_origin.as_deref() { + if let Some(origin) = demand.inventory_page_origin.as_deref() { validate_inventory_page_origin(origin).map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config inventory_page_origin: {error}"), + message: format!("invalid `aps` inventory_page_origin: {error}"), }) })?; } validate_inventory_identity_override_values( - profile.inventory_domain.as_deref(), - profile.inventory_page_origin.as_deref(), + demand.inventory_domain.as_deref(), + demand.inventory_page_origin.as_deref(), ) .map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!("invalid `aps` profile_config inventory identity: {error}"), + message: format!("invalid `aps` inventory identity: {error}"), }) })?; - Ok(profile) + if demand.rendering_mode == ApsRenderingMode::PublisherNative && demand.allow_script_creatives { + log::warn!( + "APS publisher-native rendering with script creatives is ON; selected bidder scripts execute with publisher-origin privileges" + ); + } + if demand.debug { + log::warn!( + "APS debug mode is ON. Raw request and response data, including creative markup, is included in client-visible /auction responses" + ); + } + Ok(demand) +} + +#[async_trait(?Send)] +impl CompiledDemand for ApsDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + primary_banner_size: true, + language_max_bytes: Some(CONSERVATIVE_LANGUAGE_MAX_BYTES), + regs: RegsPolicy::ApplicabilityBit, + ..DemandFieldPolicy::default() + } + } + + fn site_domain(&self, publisher_domain: &str) -> String { + self.inventory_domain + .clone() + .unwrap_or_else(|| publisher_domain.to_owned()) + } + + fn site_page(&self, publisher_page: Option<&str>, site_domain: &str) -> Option { + let fallback = publisher_page + .and_then(valid_aps_page_url) + .unwrap_or_else(|| format!("https://{site_domain}")); + let Some(origin) = self.inventory_page_origin.as_deref() else { + return Some(fallback); + }; + let (Ok(mut canonical), Ok(current)) = (Url::parse(origin), Url::parse(&fallback)) else { + return Some(fallback); + }; + canonical.set_path(current.path()); + canonical.set_query(current.query()); + canonical.set_fragment(None); + Some(canonical.to_string()) + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = Some(serde_json::Map::from_iter([ + ("account".to_string(), Json::String(self.account_id.clone())), + ( + "sdk".to_string(), + json!({"source": APS_SDK_SOURCE, "version": APS_SDK_VERSION}), + ), + ])); + Ok(()) + } + + fn capture_request( + &self, + body: &[u8], + headers: &HeaderMap, + ) -> Option> { + self.debug + .then(|| Box::new(ApsDebugRequest::capture(body, headers)) as Box<_>) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + let provider_id = context.provider_id; + let response_time_ms = context.response_time_ms; + let debug_request = context + .captured + .and_then(|captured| captured.downcast_ref::()) + .cloned(); + match parse_planned_aps_response( + provider_id, + self, + context.endpoint, + context.input, + response, + response_time_ms, + debug_request, + ) + .await + { + Ok(parsed) => Ok(parsed), + Err(error) => { + log::warn!("Provider '{provider_id}' APS response parse failed: {error:?}"); + Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response"))) + } + } + } + + fn as_any(&self) -> &dyn core::any::Any { + self + } +} + +/// Accept only a page URL APS can be given. +fn valid_aps_page_url(value: &str) -> Option { + const MAX_APS_PAGE_URL_BYTES: usize = 8192; + + if value.len() > MAX_APS_PAGE_URL_BYTES { + return None; + } + let parsed = Url::parse(value).ok()?; + (matches!(parsed.scheme(), "http" | "https") + && parsed.host_str().is_some() + && parsed.username().is_empty() + && parsed.password().is_none()) + .then(|| parsed.to_string()) } #[cfg(test)] @@ -603,7 +819,7 @@ fn planned_aps_renderer( if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { return None; } - Some(BidRenderer::Aps(ApsRendererV1 { + let descriptor = ApsRendererV1 { version: 1, account_id: policy.account_id.to_string(), bid_id: input.bid_id.to_string(), @@ -613,7 +829,10 @@ fn planned_aps_renderer( aax_response: BASE64_STANDARD.encode(serialized), width: input.width, height: input.height, - })) + }; + BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) + .ok() + .map(|renderer| renderer.picking_bid_by(APS_RENDERER_BID_ID_KEY)) } fn planned_aps_valid_creative_url(value: &str, publisher_domain: &str) -> bool { @@ -866,7 +1085,7 @@ fn parse_planned_aps_value( /// Parse one APS-profile response using only provider-local routed state. pub(crate) async fn parse_planned_aps_response( provider_id: &str, - profile: &ApsProfilePlan, + demand: &ApsDemand, endpoint: &str, input: &ProviderAuctionInput, response: PlatformResponse, @@ -876,9 +1095,9 @@ pub(crate) async fn parse_planned_aps_response( let policy = PlannedApsResponsePolicy { provider_id, endpoint, - account_id: &profile.account_id, - debug: profile.debug, - allow_script_creatives: profile.allow_script_creatives, + account_id: &demand.account_id, + debug: demand.debug, + allow_script_creatives: demand.allow_script_creatives, publisher_domain: &input.common_request().publisher.domain, }; let response = response.response; @@ -984,7 +1203,7 @@ impl ApsAuctionProvider { Self { config } } - fn build_regs(consent: Option<&crate::consent::ConsentContext>) -> Option { + fn build_regs(consent: Option<&trusted_server_core::consent::ConsentContext>) -> Option { let consent = consent?; let ext = RegsExt { gdpr: Some(u8::from(consent.gdpr_applies)), @@ -1318,7 +1537,7 @@ impl ApsAuctionProvider { if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { return None; } - Some(BidRenderer::Aps(ApsRendererV1 { + let descriptor = ApsRendererV1 { version: 1, account_id: self.config.account_id.clone(), bid_id: input.bid_id.to_string(), @@ -1328,7 +1547,17 @@ impl ApsAuctionProvider { aax_response: BASE64_STANDARD.encode(serialized), width: input.width, height: input.height, - })) + }; + match BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) { + Ok(renderer) => Some(renderer.picking_bid_by(APS_RENDERER_BID_ID_KEY)), + Err(error) => { + log::warn!( + "Dropping APS bid '{}': its renderer descriptor could not be built: {error:?}", + input.bid_id + ); + None + } + } } fn increment_reason(reasons: &mut BTreeMap, reason: &'static str) { @@ -1819,10 +2048,11 @@ impl IntegrationProxy for ApsRendererIntegration { .collect() } + // The renderer page is the same for every request, so the route takes + // nothing from its module call. async fn handle( &self, - _settings: &Settings, - _services: &RuntimeServices, + _call: trusted_server_core::module_context::ModuleCall<'_>, request: http::Request, ) -> Result, Report> { if request.method() != Method::GET || request.uri().path() != APS_RENDERER_ROUTE { @@ -1848,137 +2078,73 @@ impl IntegrationProxy for ApsRendererIntegration { } } -impl IntegrationHeadInjector for ApsRendererIntegration { - fn integration_id(&self) -> &'static str { - APS_INTEGRATION_ID - } - - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - Vec::new() - } - - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - (self.rendering_mode == ApsRenderingMode::PublisherNative) - .then_some(("data-ts-aps-rendering-mode", "publisher_native")) - .into_iter() - .collect() - } -} +/// The attribute on the script bundle's tag that lets the browser render an +/// APS creative itself. It is written by the server alone, so a page cannot +/// claim the mode for itself. +const NATIVE_RENDERING_ATTRIBUTE: (&str, &str) = ("data-ts-aps-rendering-mode", "publisher_native"); -/// Register renderer support when the auction plan contains an APS provider. +/// Register renderer support when the plan selects an APS demand source. /// -/// Browser integration enablement does not control server-side APS rendering. -/// An absent or disabled browser block uses trusted-server rendering. An enabled -/// browser block may select publisher-native rendering. +/// The rendering owner comes from the demand table itself, so a deployment +/// sets it where it sets the rest of that source's settings. Two APS sources +/// that disagree is a configuration error, because one page can only be +/// rendered one way. /// /// # Errors /// -/// Returns an error when APS browser configuration is invalid. +/// Returns an error when two selected APS sources set different rendering +/// modes. pub fn register_for_plan( - settings: &Settings, - plan: &crate::auction::AuctionPlan, + _settings: &Settings, + plan: &trusted_server_core::auction::AuctionPlan, ) -> Result, Report> { - if !plan.has_profile(APS_INTEGRATION_ID) { - return Ok(None); + let mut selected: Option<(&str, ApsRenderingMode)> = None; + for provider in plan.providers() { + if provider.implementation.id != MODULE { + continue; + } + let Some(demand) = provider.demand.as_any().downcast_ref::() else { + continue; + }; + match selected { + Some((first, mode)) if mode != demand.rendering_mode => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[demand.{}] and [demand.{first}] set different rendering_mode values, and one page can be rendered only one way", + provider.id.as_str() + ), + })); + } + Some(_) => {} + None => selected = Some((provider.id.as_str(), demand.rendering_mode)), + } } - let rendering_mode = settings - .integration_config::(APS_INTEGRATION_ID)? - .map(|config| config.rendering_mode) - .unwrap_or_default(); - let integration = Arc::new(ApsRendererIntegration { rendering_mode }); - let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) - .without_js() - .with_head_injector(integration.clone()); - let registration = if rendering_mode == ApsRenderingMode::TrustedServer { - registration.with_proxy(integration) - } else { - registration - }; - Ok(Some(registration.build())) -} - -/// Register the APS auction provider when enabled. -/// -/// # Errors -/// -/// Returns an error when enabled APS configuration is invalid. -#[cfg(test)] -#[allow(clippy::missing_panics_doc)] -pub fn register( - settings: &Settings, -) -> Result, Report> { - let Some(config) = - settings.integration_config::(APS_INTEGRATION_ID)? - else { + let Some((_, rendering_mode)) = selected else { return Ok(None); }; - let mut browser_settings = settings.clone(); - browser_settings.integrations.insert_config( - APS_INTEGRATION_ID, - &ApsConfig { - enabled: true, - rendering_mode: config.rendering_mode, - }, - )?; - register_for_plan( - &browser_settings, - &crate::auction::AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 1000, - providers: BTreeMap::from([( - "aps".parse().expect("should parse APS provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: default_endpoint(), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, - )]), - ..AuctionPlanConfig::default() - }) - .expect("should compile APS renderer test plan"), - ) -} - -#[cfg(test)] -#[allow(clippy::missing_errors_doc)] -pub fn register_providers( - settings: &Settings, -) -> Result>, Report> { - let Some(config) = - settings.integration_config::(APS_INTEGRATION_ID)? - else { - return Ok(Vec::new()); + let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID).without_js(); + let registration = match rendering_mode { + ApsRenderingMode::TrustedServer => { + registration.with_proxy(Arc::new(ApsRendererIntegration { rendering_mode })) + } + ApsRenderingMode::PublisherNative => registration + .with_bundle_tag_attribute(NATIVE_RENDERING_ATTRIBUTE.0, NATIVE_RENDERING_ATTRIBUTE.1), }; - log::info!("Registering APS OpenRTB provider"); - if config.debug { - log::warn!( - "APS debug mode is ON — raw request and response data, including creative markup, will be included in client-visible /auction responses" - ); - } - if config.rendering_mode == ApsRenderingMode::PublisherNative && config.allow_script_creatives { - log::warn!( - "APS publisher-native rendering with script creatives is ON; selected bidder scripts execute with publisher-origin privileges" - ); - } - Ok(vec![Arc::new(ApsAuctionProvider::new(config))]) + Ok(Some(registration.build())) } #[cfg(test)] mod tests { use super::*; - use crate::auction::test_support::canonical_parity_auction_request; - use crate::auction::types::{ + use serde_json::json; + use trusted_server_core::auction::test_support::canonical_parity_auction_request; + use trusted_server_core::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; - use crate::integrations::IntegrationDocumentState; - use crate::platform::test_support::{ + use trusted_server_core::platform::test_support::{ StubHttpClient, build_services_with_http_client, noop_services, }; - use crate::test_support::tests::create_test_settings; - use serde_json::json; + use trusted_server_core::test_support::tests::create_test_settings; fn config() -> LegacyApsProviderConfig { LegacyApsProviderConfig { @@ -2185,8 +2351,8 @@ mod tests { .is_err() ); assert!( - serde_json::from_value::(json!({ - "enabled": true, + compile_aps_settings(json!({ + "account_id": "example-account", "rendering_mode": "unsupported" })) .is_err(), @@ -2455,7 +2621,7 @@ mod tests { .renderer .as_ref() .expect("should include renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer"); let decoded = BASE64_STANDARD .decode(&renderer.aax_response) @@ -2509,7 +2675,7 @@ mod tests { bid.renderer .as_ref() .expect("should retain renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer") .creative_id .is_none() @@ -2591,7 +2757,7 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"seatbid":[]}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let downstream = http::Request::builder() @@ -2922,7 +3088,7 @@ mod tests { .renderer .as_ref() .expect("should keep script renderer") - .as_aps() + .payload_as::(APS_RENDERER_TYPE) .expect("should be APS renderer"); assert_eq!(renderer.tag_type, ApsTagType::Script); } @@ -3027,16 +3193,16 @@ mod tests { assert_eq!(routes[0].method, Method::GET); assert_eq!(routes[0].path, APS_RENDERER_ROUTE); - let settings = create_test_settings(); - let services = noop_services(); let request = http::Request::builder() .method(Method::GET) .uri(APS_RENDERER_ROUTE) .body(EdgeBody::empty()) .expect("should build renderer request"); - let response = - futures::executor::block_on(integration.handle(&settings, &services, request)) - .expect("should serve renderer"); + let response = futures::executor::block_on(integration.handle( + trusted_server_core::module_context::ModuleCall::empty(), + request, + )) + .expect("should serve renderer"); assert_eq!(response.status(), StatusCode::OK); assert_eq!( response.headers()[header::CONTENT_TYPE], @@ -3054,129 +3220,96 @@ mod tests { .uri(APS_RENDERER_ROUTE) .body(EdgeBody::empty()) .expect("should build method rejection request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, post)) - .expect("should reject unsupported method"); + let response = futures::executor::block_on(integration.handle( + trusted_server_core::module_context::ModuleCall::empty(), + post, + )) + .expect("should reject unsupported method"); assert_eq!(response.status(), StatusCode::NOT_FOUND); } - fn plan_with_aps_profile() -> crate::auction::AuctionPlan { - crate::auction::AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 1_000, - providers: BTreeMap::from([( - "aps".parse().expect("should parse APS provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: default_endpoint(), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, - )]), - ..AuctionPlanConfig::default() - }) + fn aps_plan(rendering_modes: &[Option<&str>]) -> trusted_server_core::auction::AuctionPlan { + let tables = rendering_modes + .iter() + .enumerate() + .map(|(index, mode)| { + let mut table = trusted_server_core::auction::test_support::demand_table( + MODULE, + &default_endpoint(), + ); + table.insert("account_id".to_string(), json!("example-account")); + table.insert("routing".to_string(), json!("all_eligible")); + if let Some(mode) = mode { + table.insert("rendering_mode".to_string(), json!(mode)); + } + (if index == 0 { "aps_main" } else { "aps_second" }, table) + }) + .collect::>(); + trusted_server_core::auction::AuctionPlan::compile( + trusted_server_core::auction::test_support::plan_config_with(tables, &[builder()]), + ) .expect("should compile APS plan") } #[test] - fn aps_plan_registers_trusted_server_renderer_without_enabled_browser_config() { - for disabled_browser_config in [false, true] { - let mut settings = create_test_settings(); - if disabled_browser_config { - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": false, - "rendering_mode": "publisher_native" - }), - ) - .expect("should insert disabled APS browser config"); - } - - let registration = register_for_plan(&settings, &plan_with_aps_profile()) - .expect("should register APS renderer support") - .expect("should return APS renderer registration"); - - assert_eq!( - registration.proxies.len(), - 1, - "APS plan should register trusted-server renderer route" - ); - assert!( - registration.head_injectors[0] - .tsjs_script_tag_attributes() - .is_empty(), - "disabled or absent browser config should not select publisher-native rendering" - ); - } - } + fn a_selected_aps_source_registers_the_trusted_server_renderer() { + let registration = register_for_plan(&create_test_settings(), &aps_plan(&[None])) + .expect("should register APS renderer support") + .expect("should return APS renderer registration"); - #[test] - fn enabled_config_registers_renderer_proxy() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({"enabled": true, "account_id": "example-account"}), - ) - .expect("should insert APS config"); - - let registration = register(&settings) - .expect("should register APS") - .expect("should return enabled registration"); - - assert_eq!(registration.integration_id, APS_INTEGRATION_ID); - assert_eq!(registration.proxies.len(), 1); - assert_eq!(registration.head_injectors.len(), 1); - let document_state = IntegrationDocumentState::default(); - let context = IntegrationHtmlContext { - request_host: "publisher.example", - request_scheme: "https", - origin_host: "origin.example", - document_state: &document_state, - }; - assert!( - registration.head_injectors[0] - .head_inserts(&context) - .is_empty(), - "should not inject a native-mode head marker by default" + assert_eq!( + registration.integration_id, APS_INTEGRATION_ID, + "the renderer registers under the APS id" + ); + assert_eq!( + registration.proxies.len(), + 1, + "a selected APS source should register the trusted-server renderer route" ); assert!( - registration.head_injectors[0] - .tsjs_script_tag_attributes() - .is_empty(), - "should not authorize native rendering by default" + registration.bundle_tag_attributes.is_empty(), + "the default rendering mode should not authorize publisher-native rendering" ); assert!(registration.js_disabled); } #[test] - fn publisher_native_config_registers_runner_mode_without_renderer_route() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": true, - "account_id": "example-account", - "rendering_mode": "publisher_native" - }), - ) - .expect("should insert native APS config"); + fn no_aps_source_registers_nothing() { + let plan = trusted_server_core::auction::AuctionPlan::compile( + trusted_server_core::auction::test_support::plan_config(Vec::new()), + ) + .expect("should compile an empty plan"); + + assert!( + register_for_plan(&create_test_settings(), &plan) + .expect("should evaluate renderer registration") + .is_none(), + "a plan with no APS source should register no renderer" + ); + } - let registration = register(&settings) - .expect("should register APS") - .expect("should return enabled registration"); + #[test] + fn publisher_native_rendering_drops_the_renderer_route_and_marks_the_bundle_tag() { + let registration = register_for_plan( + &create_test_settings(), + &aps_plan(&[Some("publisher_native")]), + ) + .expect("should register APS renderer support") + .expect("should return APS renderer registration"); assert!( registration.proxies.is_empty(), - "should not register the static renderer" + "publisher-native rendering should not register the static renderer" + ); + assert!( + registration.middleware.is_empty(), + "should change no page, so a page cannot be made to carry a forged native-mode \ + marker" + ); + assert_eq!( + registration.bundle_tag_attributes, + [("data-ts-aps-rendering-mode", "publisher_native")], + "should authorize native mode on the publisher bundle tag" ); - assert_eq!(registration.head_injectors.len(), 1); let integration = ApsRendererIntegration { rendering_mode: ApsRenderingMode::PublisherNative, @@ -3185,94 +3318,45 @@ mod tests { integration.routes().is_empty(), "should expose no renderer route" ); - let document_state = IntegrationDocumentState::default(); - let context = IntegrationHtmlContext { - request_host: "publisher.example", - request_scheme: "https", - origin_host: "origin.example", - document_state: &document_state, + } + + #[test] + fn two_aps_sources_that_disagree_on_rendering_are_refused() { + let plan = aps_plan(&[None, Some("publisher_native")]); + let error = match register_for_plan(&create_test_settings(), &plan) { + Ok(_) => panic!("should refuse two rendering modes"), + Err(error) => error, }; + let message = error.to_string(); assert!( - integration.head_inserts(&context).is_empty(), - "should not inject a forgeable native-mode marker" - ); - assert_eq!( - integration.tsjs_script_tag_attributes(), - vec![("data-ts-aps-rendering-mode", "publisher_native")], - "should authorize native mode on the publisher bundle tag" + message.contains("rendering_mode") && message.contains("aps_second"), + "should name the setting and the source that disagrees: {error:?}" ); } #[test] fn publisher_native_script_creatives_remain_available_for_controlled_validation() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": true, - "account_id": "example-account", - "allow_script_creatives": true, - "rendering_mode": "publisher_native" - }), - ) - .expect("should insert native APS script config"); - - let providers = register_providers(&settings).expect("should register APS provider"); + let demand = compile_aps_settings(json!({ + "account_id": "example-account", + "allow_script_creatives": true, + "rendering_mode": "publisher_native" + })) + .expect("should compile the controlled experiment"); - assert_eq!( - providers.len(), - 1, - "should retain the controlled experiment" - ); + assert!(demand.allow_script_creatives); + assert_eq!(demand.rendering_mode, ApsRenderingMode::PublisherNative); } #[test] - fn config_without_enabled_does_not_register_provider_or_renderer() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({"account_id": "example-account"}), - ) - .expect("should insert default-disabled APS config"); - + fn an_aps_source_needs_an_account() { + let error = compile_aps_settings(json!({"debug": true})) + .expect_err("should refuse an APS source with no account"); assert!( - register(&settings) - .expect("should evaluate renderer registration") - .is_none(), - "omitted enabled should not register the renderer route" - ); - assert!( - register_providers(&settings) - .expect("should evaluate provider registration") - .is_empty(), - "omitted enabled should not register the auction provider" + format!("{error:?}").contains("account_id"), + "should name the missing setting: {error:?}" ); } - #[test] - fn enabled_invalid_config_fails_provider_registration() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - APS_INTEGRATION_ID, - &json!({ - "enabled": true, - "account_id": "example-account", - "endpoint": "http://insecure.example/openrtb" - }), - ) - .expect("should insert invalid APS config for startup validation"); - - let _error = register_providers(&settings) - .err() - .expect("should reject invalid enabled APS configuration"); - } - #[test] fn renderer_document_is_static_and_nonce_bound() { assert!(APS_RENDERER_DOCUMENT.contains("^#tsaps=")); @@ -3308,4 +3392,1244 @@ mod tests { assert!(APS_RENDERER_CSP.contains("sandbox allow-forms")); assert!(!APS_RENDERER_CSP.contains("allow-same-origin")); } + + #[test] + fn renderer_bid_id_key_matches_the_serialized_form() { + let descriptor = ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }; + let renderer = BidRenderer::from_typed(APS_RENDERER_TYPE, &descriptor) + .expect("should build APS renderer descriptor"); + + assert_eq!( + renderer + .payload_field(APS_RENDERER_TYPE, APS_RENDERER_BID_ID_KEY) + .and_then(serde_json::Value::as_str), + Some(descriptor.bid_id.as_str()), + "should name the wire key `ApsRendererV1` serializes `bid_id` to" + ); + assert_eq!( + renderer + .payload_field(APS_RENDERER_TYPE, APS_RENDERER_BID_ID_KEY) + .and_then(serde_json::Value::as_str), + renderer + .payload_as::(APS_RENDERER_TYPE) + .as_ref() + .map(|full| full.bid_id.as_str()), + "should read the same value as deserializing the whole descriptor" + ); + } + + #[test] + fn aps_renderer_serializes_to_versioned_camel_case_contract() { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert_eq!( + serialized, + json!({ + "type": "aps", + "version": 1, + "accountId": "example-account-id", + "bidId": "fictional-bid-id", + "creativeId": "fictional-creative-id", + "tagType": "iframe", + "creativeUrl": "https://creative.example/render", + "aaxResponse": "base64-data", + "width": 300, + "height": 250 + }), + "should match renderer wire contract" + ); + } + + #[test] + fn aps_renderer_omits_absent_creative_id() { + let renderer = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + + assert!( + serialized.get("creativeId").is_none(), + "should omit absent creative ID" + ); + } + + /// Rewrites every object in `value` with its keys in sorted order, so + /// serializing the result gives one fixed key order. + /// + /// `serde_json::Map` is a `BTreeMap`, which serializes keys in sorted + /// order, only while the crate's `preserve_order` feature is off. With the + /// feature on it is an `IndexMap` and the order follows insertion instead. + /// Nothing in this crate asks for the feature, but Cargo unifies features + /// across everything built for one target, and `trusted-server-cli` pulls + /// it in through `edgezero-cli` and then `handlebars`. A maintainer + /// running `cargo test --workspace --target ` therefore builds this + /// crate with `preserve_order` on, and a test that pinned insertion order + /// would fail there for no reason. Sorting both sides removes the + /// dependence on which map `serde_json` was built with. + fn with_sorted_keys(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let mut keys = map.keys().collect::>(); + keys.sort_unstable(); + let mut sorted = serde_json::Map::with_capacity(keys.len()); + for key in keys { + let child = map.get(key).expect("should find a key the map just listed"); + sorted.insert(key.clone(), with_sorted_keys(child)); + } + serde_json::Value::Object(sorted) + } + serde_json::Value::Array(items) => { + serde_json::Value::Array(items.iter().map(with_sorted_keys).collect()) + } + scalar => scalar.clone(), + } + } + + #[test] + fn the_open_renderer_serializes_to_the_same_bytes_as_the_aps_variant_did() { + // Literal strings captured from the closed-enum form before this + // change, through the same `serde_json::to_value` path production + // uses: `BidExt::to_ext` for the OpenRTB response extension, and + // `build_bid_map` for `window.tsjs.bids`. + // + // Both sides go through `with_sorted_keys` first, because the key + // order `serde_json` emits is not ours to pin, and that function + // explains why. Sorting settles the order without weakening what is + // pinned, since two objects serialize to the same sorted bytes only + // when they carry exactly the same keys with exactly the same values. + let full = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: Some("fictional-creative-id".to_string()), + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + let absent = BidRenderer::from_typed( + APS_RENDERER_TYPE, + &ApsRendererV1 { + version: 1, + account_id: "example-account-id".to_string(), + bid_id: "fictional-bid-id".to_string(), + creative_id: None, + tag_type: ApsTagType::Script, + creative_url: "https://creative.example/render".to_string(), + aax_response: "base64-data".to_string(), + width: 300, + height: 250, + }, + ) + .expect("should build APS renderer descriptor"); + + let full_bytes = serde_json::to_string(&with_sorted_keys( + &serde_json::to_value(&full).expect("should convert renderer to a JSON value"), + )) + .expect("should serialize renderer"); + let absent_bytes = serde_json::to_string(&with_sorted_keys( + &serde_json::to_value(&absent).expect("should convert renderer to a JSON value"), + )) + .expect("should serialize renderer"); + + assert_eq!( + full_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeId\":\"fictional-creative-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"iframe\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced" + ); + assert_eq!( + absent_bytes, + "{\"aaxResponse\":\"base64-data\",\"accountId\":\"example-account-id\",\"bidId\":\"fictional-bid-id\",\"creativeUrl\":\"https://creative.example/render\",\"height\":250,\"tagType\":\"script\",\"type\":\"aps\",\"version\":1,\"width\":300}", + "should serialize to the bytes the closed enum produced with no creative ID" + ); + } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } +} + +/// APS run through core's auction engine: the orchestrator, and the reading +/// of an APS response as the driver hands it over. +#[cfg(test)] +mod engine_tests { + use std::collections::HashMap; + use std::sync::Arc; + + use base64::Engine as _; + + use trusted_server_core::auction::orchestrator::AuctionOrchestrator; + use trusted_server_core::auction::plan::{AuctionPlan, AuctionPlanConfig, NotificationConfig}; + use trusted_server_core::auction::test_support::{ + build_for_first_source, canonical_parity_auction_request, demand_table, + deterministic_signer, golden_inbound_request, golden_plan_config, parse_as_first_source, + plan_config_with, + }; + use trusted_server_core::auction::types::{ + AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, MediaType, PublisherInfo, + UserInfo, + }; + use trusted_server_core::platform::test_support::{ + NamingBackend, StubHttpClient, build_services_with_backend_and_http_client, + }; + use trusted_server_core::platform::{BackendNamingPolicy, PlatformResponse}; + use trusted_server_core::test_support::tests::create_test_settings; + + use super::{APS_RENDERER_TYPE, ApsRendererV1, MODULE, builder}; + + /// A plan configuration of APS sources, each with the settings and the + /// notification policy given. + fn aps_instances_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + let tables = providers + .iter() + .map(|(id, settings, notifications)| { + let mut entry = demand_table(MODULE, "https://aps.example/e/pb/bid"); + entry.insert("timeout_ms".to_string(), serde_json::json!(1_000)); + entry.insert("routing".to_string(), serde_json::json!("all_eligible")); + entry.insert( + "notifications".to_string(), + serde_json::to_value(notifications).expect("should serialize notifications"), + ); + if let serde_json::Value::Object(settings) = settings { + entry.extend(settings.clone()); + } + (*id, entry) + }) + .collect::>(); + let mut config = plan_config_with(tables, &[builder()]); + config.timeout_ms = 777; + config + } + + fn aps_config() -> AuctionPlanConfig { + aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id": "example-account"}), + NotificationConfig::default(), + )]) + } + + fn planned_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::new(), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + /// One APS source as a `[demand.]` table, with nothing set that APS + /// defaults. + fn bare_aps_table() -> serde_json::Map { + let mut table = demand_table(MODULE, "https://aps.example/e/pb/bid"); + table.insert( + "account_id".to_string(), + serde_json::json!("example-account"), + ); + table + } + + fn compile_one( + table: serde_json::Map, + ) -> Result> + { + AuctionPlan::compile(plan_config_with(vec![("aps_one", table)], &[builder()])) + } + + #[test] + fn the_plan_compiles_an_aps_source_with_its_own_defaults() { + let plan = compile_one(bare_aps_table()).expect("should compile an APS source"); + + assert!( + plan.has_implementation(MODULE), + "a compiled plan should say it selected APS" + ); + let source = &plan.providers()[0]; + assert!( + source.demand.as_any().is::(), + "the APS source should compile its own settings" + ); + assert_eq!( + source.timeout_ms, 800, + "should default to the timeout APS declares" + ); + assert_eq!( + source.endpoint.as_str(), + "https://aps.example/e/pb/bid", + "should keep the endpoint as written" + ); + } + + #[test] + fn the_plan_refuses_what_aps_does_not_accept() { + let mut legacy_path = bare_aps_table(); + legacy_path.insert( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/dtb/bid"), + ); + assert!( + compile_one(legacy_path).is_err(), + "should refuse the legacy APS path" + ); + + let mut domain_alone = bare_aps_table(); + domain_alone.insert( + "inventory_domain".to_string(), + serde_json::json!("publisher.example"), + ); + assert!( + compile_one(domain_alone).is_err(), + "should refuse an APS inventory domain without its page origin" + ); + + let mut without_account = bare_aps_table(); + without_account.remove("account_id"); + assert!( + compile_one(without_account).is_err(), + "should refuse an APS source with no account" + ); + } + + #[test] + fn an_aps_demand_source_registers_the_renderer_with_no_integration_table() { + use trusted_server_core::auction::test_support::demand_selection; + use trusted_server_core::integrations::IntegrationRegistry; + + let mut settings = create_test_settings(); + let mut table = bare_aps_table(); + table.insert("routing".to_string(), serde_json::json!("all_eligible")); + settings.demand = demand_selection(vec![("aps_main", table)]); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan_with(&settings, &[builder()]) + .expect("should compile APS plan"), + ); + let registry = + IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder()]) + .expect("should build APS renderer registry"); + + assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); + } + + #[test] + fn two_aps_sources_that_disagree_on_rendering_are_refused() { + use trusted_server_core::auction::test_support::demand_selection; + use trusted_server_core::integrations::IntegrationRegistry; + + let mut settings = create_test_settings(); + let mut publisher_native = bare_aps_table(); + publisher_native.insert( + "rendering_mode".to_string(), + serde_json::json!("publisher_native"), + ); + settings.demand = demand_selection(vec![ + ("aps_one", bare_aps_table()), + ("aps_two", publisher_native), + ]); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan_with(&settings, &[builder()]) + .expect("should compile APS plan"), + ); + let error = + match IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder()]) { + Ok(_) => panic!("should refuse two rendering modes"), + Err(error) => error, + }; + assert!( + error.to_string().contains("rendering_mode"), + "should name the setting that disagrees: {error:?}" + ); + } + + /// The request the driver builds for one APS source over the canonical + /// request, the way the goldens were captured. + fn golden_request( + settings: serde_json::Value, + signer: Option<&trusted_server_core::request_signing::RequestSigner>, + ) -> trusted_server_core::openrtb::OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + build_for_first_source( + &plan, + canonical_parity_auction_request(), + &golden_inbound_request(), + 321, + signer, + ) + .expect("should build request") + .expect("should retain impression") + } + + /// The request the driver builds for one APS source over `request`. + fn request_for( + settings: serde_json::Value, + request: AuctionRequest, + accept_language: Option<&str>, + ) -> trusted_server_core::openrtb::OpenRtbRequest { + let plan = AuctionPlan::compile(golden_plan_config(MODULE, settings, true, &[builder()])) + .expect("should compile plan"); + let mut inbound = http::Request::builder().uri("https://publisher.example/auction"); + if let Some(language) = accept_language { + inbound = inbound.header(http::header::ACCEPT_LANGUAGE, language); + } + let inbound = inbound + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + build_for_first_source(&plan, request, &inbound, 321, None) + .expect("should build request") + .expect("should retain impression") + } + + #[test] + fn consent_fields_keep_an_empty_admitted_context() { + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::consent::jurisdiction::Jurisdiction; + + let cases = [ + ("empty", ConsentContext::default()), + ( + "gdpr", + ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("tc-string".to_string()), + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ( + "unknown-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "nonregulated-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ( + "usp-gpp", + ConsentContext { + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("gpp-string".to_string()), + gpp_section_ids: Some(vec![7, 8]), + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ]; + for (name, consent) in cases { + let mut canonical = canonical_parity_auction_request(); + canonical.user.consent = Some(consent.clone()); + let value = serde_json::to_value(request_for( + serde_json::json!({"account_id": "example-account-id"}), + canonical, + None, + )) + .expect("should serialize request"); + let regs = value + .get("regs") + .expect("APS should preserve empty admitted context"); + assert_eq!( + regs["gdpr"], + serde_json::json!(u8::from(consent.gdpr_applies)), + "{name}" + ); + assert!( + !value.to_string().contains("1YYY"), + "must never synthesize USP from GPC" + ); + if name == "usp-gpp" { + assert_eq!(regs["us_privacy"], "1YNN"); + assert_eq!(regs["gpp"], "gpp-string"); + assert_eq!(regs["gpp_sid"], serde_json::json!([7, 8])); + assert_eq!(regs["ext"]["us_privacy"], "1YNN"); + assert_eq!(regs["ext"]["gpp"], "gpp-string"); + assert_eq!(regs["ext"]["gpp_sid"], serde_json::json!([7, 8])); + } + } + } + + #[test] + fn language_is_kept_only_within_the_conservative_limit() { + let settings = serde_json::json!({"account_id": "example-account-id"}); + + let long = request_for( + settings.clone(), + canonical_parity_auction_request(), + Some("abcdefghijk"), + ); + assert_eq!( + long.device.and_then(|device| device.language).as_deref(), + None, + "should drop a language tag over the limit" + ); + + let ordinary = request_for( + settings, + canonical_parity_auction_request(), + Some("en-US,en;q=0.9"), + ); + assert_eq!( + ordinary + .device + .and_then(|device| device.language) + .as_deref(), + Some("en") + ); + } + + #[test] + fn inventory_identity_and_page_fallback_preserve_legacy_policy() { + let mut request = canonical_parity_auction_request(); + request.publisher.domain = "deployment.example".to_string(); + request.publisher.page_url = + Some("https://deployment.example/news/story?edition=fictional#section".to_string()); + let built = request_for( + serde_json::json!({ + "account_id": "example-account-id", + "inventory_domain": "publisher.example", + "inventory_page_origin": "https://www.publisher.example" + }), + request, + None, + ); + let site = built.site.expect("should include APS site"); + assert_eq!(site.domain.as_deref(), Some("publisher.example")); + assert_eq!( + site.page.as_deref(), + Some("https://www.publisher.example/news/story?edition=fictional") + ); + assert_eq!( + site.publisher + .and_then(|publisher| publisher.domain) + .as_deref(), + Some("publisher.example") + ); + + for unsafe_page in [ + "https://user:password@publisher.example/private", + "data:text/html,fictional", + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(unsafe_page.to_string()); + let built = request_for( + serde_json::json!({"account_id":"example-account-id"}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some("https://publisher.example"), + "unsafe page should fall back to publisher domain" + ); + } + } + + #[test] + fn the_driver_request_matches_the_exact_golden() { + let request = golden_request( + serde_json::json!({"account_id": "example-account-id"}), + None, + ); + assert_eq!( + serde_json::to_string(&request).expect("should serialize APS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve APS parity differences" + ); + } + + #[test] + fn signing_is_applied_after_the_request_is_built_and_sets_every_owned_key() { + let settings = serde_json::json!({"account_id": "example-account-id"}); + let unsigned = serde_json::to_value(golden_request(settings.clone(), None)) + .expect("should serialize unsigned request"); + assert!( + unsigned["ext"].get("trusted_server").is_none(), + "should omit the extension when unsigned" + ); + + let signer = deterministic_signer(); + let signed = serde_json::to_value(golden_request(settings, Some(&signer))) + .expect("should serialize signed request"); + let extension = &signed["ext"]["trusted_server"]; + assert_eq!(extension["version"], "1.1", "should set signing version"); + assert_eq!(extension["kid"], "fictional-kid", "should set key ID"); + assert_eq!( + extension["request_host"], "publisher.example", + "should set host" + ); + assert_eq!(extension["request_scheme"], "https", "should set scheme"); + assert_eq!( + extension["ts"], 1_706_900_000_u64, + "should set explicit time" + ); + assert!( + extension["signature"] + .as_str() + .is_some_and(|value| !value.is_empty()), + "should set signature" + ); + } + + #[test] + fn the_signed_driver_request_matches_the_exact_golden() { + let signer = deterministic_signer(); + let request = golden_request( + serde_json::json!({"account_id": "example-account-id"}), + Some(&signer), + ); + assert_eq!( + serde_json::to_string(&request).expect("should serialize signed request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + "signed wire fixture should stay exact" + ); + } + + /// Settings whose `[demand]` selects one APS source with `account_id` + /// and the settings `extra` adds. + fn settings_with_aps_source( + account_id: &str, + extra: &[(&str, serde_json::Value)], + ) -> trusted_server_core::settings::Settings { + let mut settings = create_test_settings(); + let mut table = demand_table(MODULE, "https://aps.example.com/e/pb/bid"); + table.insert("account_id".to_string(), serde_json::json!(account_id)); + for (key, value) in extra { + table.insert((*key).to_string(), value.clone()); + } + settings.demand = + trusted_server_core::auction::test_support::demand_selection(vec![("aps_main", table)]); + settings + } + + fn validate_for_deploy( + settings: &trusted_server_core::settings::Settings, + ) -> Result<(), error_stack::Report> { + trusted_server_core::config::validate_settings_for_deploy_with(settings, &[builder()]) + } + + #[test] + fn deploy_validation_rejects_a_blank_account_id() { + for (label, account_id) in [("empty", ""), ("whitespace-only", " ")] { + let err = validate_for_deploy(&settings_with_aps_source(account_id, &[])) + .expect_err("should reject blank APS account_id"); + + assert!( + format!("{err:?}").contains("account_id"), + "should mention the APS account_id for {label}: {err:?}" + ); + } + } + + #[test] + fn deploy_validation_normalizes_a_padded_account_id() { + validate_for_deploy(&settings_with_aps_source(" example-account ", &[])) + .expect("should accept a padded APS account_id after trimming it"); + } + + #[test] + fn deploy_validation_rejects_a_setting_aps_does_not_know() { + let settings = + settings_with_aps_source("example-account", &[("enabled", serde_json::json!(false))]); + + let error = validate_for_deploy(&settings) + .expect_err("should reject a setting the APS implementation does not know"); + let rendered = format!("{error:?}"); + assert!( + rendered.contains("aps_main"), + "should identify the demand source: {rendered}" + ); + assert!( + rendered.contains("enabled"), + "should identify the setting it does not know: {rendered}" + ); + } + + #[tokio::test] + async fn planned_aps_transport_omits_accept_header() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(400, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_config()).expect("should compile planned APS auction"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS auction"); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 1); + assert!( + headers[0].iter().all(|(name, _)| name != "accept"), + "planned APS transport must not add Accept beyond legacy headers" + ); + } + + #[tokio::test] + async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + serde_json::to_vec(&serde_json::json!({ + "cur": "USD", + "seatbid": [ + {"seat": "returned-seat", "bid": [ + {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", + "crid": "fictional-creative", "adomain": ["advertiser.example"], + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, + {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, + {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, + {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "crid": "x".repeat(1025), + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} + ]}, + {"seat": 7, "bid": "bad-shape"} + ] + })) + .expect("should serialize APS profile response"), + vec![ + ("content-type", "application/json"), + ("authorization", "fictional-secret"), + ], + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id": "example-account", "debug": true}), + NotificationConfig { + suppress_all: false, + suppress_seats: vec!["returned-seat".to_string()], + }, + )])) + .expect("should compile planned APS profile"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS profile"); + + let response = &result.provider_responses[0]; + assert_eq!(response.provider, "aps_instance"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!( + response.bids.len(), + 1, + "should retain one bid per impression" + ); + let bid = &response.bids[0]; + assert_eq!(bid.bidder, "aps"); + assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); + assert_eq!( + bid.bid_id.as_deref(), + Some("a-high"), + "lexical ID should break equal-price tie" + ); + assert!(bid.creative.is_none()); + assert!( + bid.nurl.is_none() && bid.burl.is_none(), + "APS must discard notification URLs" + ); + let renderer = bid + .renderer + .as_ref() + .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) + .expect("should construct typed APS renderer"); + assert_eq!(renderer.account_id, "example-account"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode minimized APS response"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse minimized APS response"), + serde_json::json!({"seatbid":[{"bid":[{ + "id":"a-high","price":2.0,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]}) + ); + assert_eq!(response.metadata["seatbid_count"], 2); + assert_eq!(response.metadata["accepted_bid_count"], 1); + assert_eq!(response.metadata["dropped_bid_count"], 12); + for reason in [ + "lost_to_higher_bid", + "script_rendering_disabled", + "unknown_impid", + "invalid_dimensions", + "invalid_price", + "unsupported_media_type", + "unsupported_tagtype", + "creative_id_too_large", + "missing_render_source", + "empty_seatbid_bids", + ] { + assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); + } + assert_eq!( + response.metadata["drop_reasons"]["invalid_creative_url"], 2, + "same-publisher and credentialed URLs should both be rejected" + ); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; + assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); + assert_eq!( + debug["responseheaders"], + serde_json::json!({"content-type": ["application/json"]}), + "async stub should preserve allowlisted response headers" + ); + assert!( + debug["requestbody"] + .as_str() + .is_some_and(|body| body.contains("example-account")) + ); + assert!(debug["requestheaders"].get("authorization").is_none()); + assert!(debug["responseheaders"].get("authorization").is_none()); + } + + #[tokio::test] + async fn two_planned_aps_instances_correlate_independently() { + let http = Arc::new(StubHttpClient::new()); + for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize APS instance response"), + ); + } + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(aps_instances_config(&[ + ( + "aps_a", + serde_json::json!({"account_id":"account-a"}), + NotificationConfig::default(), + ), + ( + "aps_b", + serde_json::json!({"account_id":"account-b"}), + NotificationConfig::default(), + ), + ])) + .expect("should compile two APS instances"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute two APS instances"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "aps_a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!(result.provider_responses[1].provider, "aps_b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!(http.recorded_request_bodies().len(), 2); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b"), + "global ranking should remain orchestrator-owned" + ); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { + let plan = AuctionPlan::compile(aps_config()).expect("should compile APS plan"); + for (seat, expected) in [ + (serde_json::Value::Null, None), + (serde_json::json!(7), None), + (serde_json::json!(""), None), + (serde_json::json!("exact-seat"), Some("exact-seat")), + ] { + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, + "nurl":"https://notice.example/win","burl":"https://notice.example/bill", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize seat identity response"), + )) + .expect("should build seat identity response"), + ); + let parsed = parse_as_first_source(&plan, planned_request(), response, 4) + .await + .expect("should parse seat identity response"); + assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); + assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); + } + } + + #[tokio::test] + async fn planned_aps_response_status_shape_and_currency_matrix() { + let plan = AuctionPlan::compile(aps_config()).expect("should compile APS plan"); + let cases = [ + (204, Vec::new(), BidStatus::NoBid, None, None), + (400, Vec::new(), BidStatus::Error, None, Some("http_status")), + ( + 200, + b"not-json".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + b"[]".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"contextual":true}"#.to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + Some("unsupported_currency"), + None, + ), + ]; + for (status, body, expected, reason, error_type) in cases { + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build APS matrix response"), + ); + let parsed = parse_as_first_source(&plan, planned_request(), response, 4) + .await + .expect("should classify APS matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if let Some(reason) = reason { + assert_eq!( + parsed.metadata["drop_reasons"][reason], 1, + "status {status}" + ); + } + if let Some(error_type) = error_type { + assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); + } + } + } + + #[tokio::test] + async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({ + "account_id":"example-account-id", + "allow_script_creatives":true + }), + NotificationConfig::default(), + )])) + .expect("should compile script-enabled APS plan"); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ + "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, + "w":300,"h":250,"crid":"fictional-creative", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + },{ + "id":"script-bid","impid":"fictional-slot","price":1.0, + "w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} + }]}]})) + .expect("should serialize APS renderer fixture response"), + )) + .expect("should build APS renderer fixture response"), + ); + + let parsed = parse_as_first_source(&plan, planned_request(), response, 3) + .await + .expect("should parse APS renderer fixture response"); + + assert_eq!(parsed.status, BidStatus::Success); + assert_eq!( + parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, + "enabled script creative should be eligible before reduction" + ); + let renderer = parsed.bids[0] + .renderer + .as_ref() + .and_then(|renderer| renderer.payload_as::(APS_RENDERER_TYPE)) + .expect("should construct APS renderer"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode APS fixture envelope"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" + )) + .expect("should parse shared APS renderer fixture"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse decoded APS renderer"), + fixture + ); + } + + #[tokio::test] + async fn planned_aps_debug_response_headers_are_allowlisted() { + let plan = AuctionPlan::compile(aps_instances_config(&[( + "aps_instance", + serde_json::json!({"account_id":"example-account","debug":true}), + NotificationConfig::default(), + )])) + .expect("should compile debug APS plan"); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .header("content-type", "application/json") + .header("authorization", "fictional-secret") + .body(edgezero_core::body::Body::from("{}")) + .expect("should build debug APS response"), + ); + + let parsed = parse_as_first_source(&plan, planned_request(), response, 3) + .await + .expect("should parse debug APS response"); + + let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; + assert_eq!( + headers, + &serde_json::json!({"content-type":["application/json"]}) + ); + assert!(headers.get("authorization").is_none()); + } + + #[tokio::test] + async fn planned_routing_count_survives_a_bounded_body_failure() { + let (profile, provider_id) = ("aps", "aps_instance"); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = aps_config(); + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + trusted_server_core::auction::plan::BidderRouteConfig { + module: provider_id + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); + } + + /// The page a reader receives with this module running, kept as a file + /// so that changing how the page change is made can be shown to leave + /// the page as it was. + #[test] + fn the_page_a_reader_receives_is_the_recorded_one() { + trusted_server_core::html_processor::test_support::assert_page_is_recorded( + include_str!("fixtures/page-change.settings.toml"), + &[super::builder()], + include_str!("fixtures/page-change.input.html"), + include_str!("fixtures/page-change.recorded.html"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/fixtures/page-change.recorded.html" + ), + ); + } } diff --git a/crates/auction/prebid-server/Cargo.toml b/crates/auction/prebid-server/Cargo.toml new file mode 100644 index 0000000000..c1bfa1cefd --- /dev/null +++ b/crates/auction/prebid-server/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "trusted-server-auction-prebid-server" +description = "Prebid Server as a demand implementation: its request extensions, bidder parameter overrides, transport headers and response reader." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +base64 = { workspace = true } +bytes = { workspace = true } +edgezero-core = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +validator = { workspace = true } +web-time = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this vendor crate, the way Prebid.js requires a named +# maintainer of every adapter. Prebid.org has not yet adopted this crate, so the +# Trusted Server maintainers own it until a vendor owner steps up. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "seeking vendor owner" diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/auction/prebid-server/src/lib.rs similarity index 67% rename from crates/trusted-server-core/src/integrations/prebid.rs rename to crates/auction/prebid-server/src/lib.rs index f1d1d0b509..092117af6a 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/auction/prebid-server/src/lib.rs @@ -1,78 +1,324 @@ -use std::collections::HashMap; +//! The Prebid Server demand implementation, `auction.prebid-server`. +//! +//! Prebid Server takes the shared `OpenRTB` baseline with its own impression +//! and request extensions, forwards the raw headers its auction needs, and +//! answers with Prebid's own response shape. + +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] + +use core::any::Any; +#[cfg(test)] use std::collections::HashSet; -use std::sync::{Arc, LazyLock}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; #[cfg(test)] use std::time::Duration; use async_trait::async_trait; -use base64::{ - Engine as _, - engine::general_purpose::{ - STANDARD as BASE64_STANDARD, STANDARD_NO_PAD as BASE64_STANDARD_NO_PAD, - }, -}; +#[cfg(test)] +use base64::Engine as _; +#[cfg(test)] use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; +#[cfg(test)] +use http::StatusCode; use http::header::HeaderValue; -use http::{Method, StatusCode, header}; -use regex::Regex; +use http::{HeaderMap, header}; use serde::{Deserialize, Serialize}; -use serde_json::Value as Json; +use serde_json::{Map, Value, Value as Json, json}; use url::{Url, Url as ParsedUrl}; -use validator::{Validate, ValidationError}; +#[cfg(test)] +use validator::Validate; -use crate::auction::openrtb::{ +use trusted_server_core::auction::demand::{ + CompiledDemand, DemandFieldPolicy, DemandImplementation, DemandResponse, DemandTimeoutDefault, + DemandTransport, ProviderAuctionInput, RegsPolicy, RequestExtensions, TransportHeaders, +}; +use trusted_server_core::auction::openrtb::{ BidDimensionIndex, BidRejectionReason, ResponseAdmissionDiagnostics, build_bid_dimension_index, parse_optional_bid_dimension, resolve_bid_dimensions, }; -use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; -use crate::auction::plan::AuctionPlan; -use crate::auction::profile::PrebidProfilePlan; +use trusted_server_core::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; #[cfg(test)] -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::routing::{PrebidTransportHeaders, ProviderAuctionInput}; +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; #[cfg(test)] -use crate::auction::types::{AuctionContext, AuctionRequest, MediaType}; -use crate::auction::types::{AuctionResponse, Bid as AuctionBid}; -use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; -use crate::consent_config::ConsentForwardingMode; -use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; -use crate::error::TrustedServerError; +use trusted_server_core::auction::types::{AuctionContext, AuctionRequest, MediaType}; +use trusted_server_core::auction::types::{AuctionResponse, Bid as AuctionBid}; +use trusted_server_core::consent::{ConsentContext, ConsentSource}; +use trusted_server_core::consent_config::ConsentForwardingMode; +use trusted_server_core::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; +use trusted_server_core::error::TrustedServerError; #[cfg(test)] -use crate::http_util::RequestInfo; -use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, - IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +use trusted_server_core::http_util::RequestInfo; +use trusted_server_core::integrations::{ + UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ +use trusted_server_core::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; #[cfg(test)] -use crate::openrtb::{ - Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, ImpStoredRequest, - OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, - TrustedServerExt, User, UserExt, to_openrtb_i32, +use trusted_server_core::openrtb::{ + Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, + RegsExt, Site, ToExt, TrustedServerExt, User, UserExt, to_openrtb_i32, }; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; -use crate::proxy::{ProxyRequestConfig, is_host_allowed, proxy_request}; +use trusted_server_core::platform::PlatformHttpRequest; +use trusted_server_core::platform::PlatformResponse; +#[cfg(test)] +use trusted_server_core::platform::RuntimeServices; +#[cfg(test)] +use trusted_server_core::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; #[cfg(test)] -use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; -use crate::settings::{IntegrationConfig, Settings}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; + +/// The name an `implementation` line gives this implementation, its module +/// path, which also serves as its builder's id. +pub const MODULE: &str = "auction.prebid-server"; + +/// The canonical Prebid Server auction path. +const AUCTION_PATH: &str = "/openrtb2/auction"; + +/// The Prebid Server demand implementation. +pub static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Fixed(1000), + allows_all_eligible: false, + serves_stored_requests: true, + canonicalize_endpoint, + compile, +}; + +/// The builder a deployment hands to an adapter, which offers Prebid Server +/// to `[demand]`. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::implementations( + MODULE, + env!("CARGO_PKG_NAME"), + ) + .with_demand(&DEMAND) +} + +/// One compiled Prebid Server demand source. +#[derive(Debug, Clone)] +pub struct PrebidServerDemand { + /// Include Prebid HTTP exchange diagnostics. + pub debug: bool, + /// Set `OpenRTB` test mode. + pub test_mode: bool, + /// Query fragment appended to the page URL for a test deployment. + pub debug_query_params: Option, + /// Compiled override matching and merge index. + pub(crate) override_engine: BidParamOverrideEngine, + /// Consent transport policy. + pub consent_forwarding: ConsentForwardingMode, +} + +/// The operator settings `[demand.]` holds for this implementation. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct PrebidServerSettings { + #[serde(default)] + debug: bool, + #[serde(default)] + test_mode: bool, + #[serde(default)] + debug_query_params: Option, + #[serde(default)] + bid_param_zone_overrides: BTreeMap>>, + #[serde(default)] + bid_param_overrides: BTreeMap>, + #[serde(default)] + bid_param_override_rules: Vec, + #[serde(default)] + consent_forwarding: ConsentForwardingMode, +} + +fn compile( + settings: &Map, +) -> Result, Report> { + let settings = + PrebidServerSettings::deserialize(Value::Object(settings.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + let override_engine = compile_profile_override_rules( + &settings.bid_param_zone_overrides, + &settings.bid_param_overrides, + &settings.bid_param_override_rules, + )?; + Ok(Arc::new(PrebidServerDemand { + debug: settings.debug, + test_mode: settings.test_mode, + debug_query_params: settings.debug_query_params, + override_engine, + consent_forwarding: settings.consent_forwarding, + })) +} + +/// Complete an endpoint that names the host alone with the auction path. +fn canonicalize_endpoint(endpoint: &mut Url) -> Result<(), String> { + match endpoint.path() { + "" | "/" | "/openrtb2/auction/" => endpoint.set_path(AUCTION_PATH), + _ => {} + } + Ok(()) +} + +#[async_trait(?Send)] +impl CompiledDemand for PrebidServerDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + imp_tagid: true, + site_ref: true, + precise_geo: true, + additional_consent: true, + test: self.test_mode, + unsigned_request_identity: true, + consumes_bidder_params: true, + regs: RegsPolicy::Jurisdiction, + ..DemandFieldPolicy::default() + } + } + + fn site_page(&self, publisher_page: Option<&str>, site_domain: &str) -> Option { + let _ = site_domain; + publisher_page.map(|page| { + self.debug_query_params.as_deref().map_or_else( + || page.to_owned(), + |query| append_query_fragment(page, query), + ) + }) + } + + fn body_consent<'c>(&self, consent: Option<&'c ConsentContext>) -> Option<&'c ConsentContext> { + consent.filter(|value| { + self.consent_forwarding.includes_body_consent() + || !matches!(value.source, ConsentSource::Cookie) + }) + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + for impression in &mut extensions.impressions { + let slot = impression.slot; + let bidder = slot + .bidder_params() + .iter() + .filter_map(|(bidder, params)| { + let mut params = params.clone(); + self.override_engine + .apply_routed(bidder.as_str(), slot.zone(), &mut params); + params + .as_object() + .is_some_and(|params| !params.is_empty()) + .then(|| (bidder.as_str().to_string(), params)) + }) + .collect::>(); + // Usable inline parameters win. Otherwise only a stored request the + // slot's intent permits is sent, and a slot left with neither has + // no demand for Prebid Server and is left out of the request. + let mut prebid = Map::new(); + if !bidder.is_empty() { + prebid.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.allows_stored_fallback() { + prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); + } + if prebid.is_empty() { + impression.omitted = true; + continue; + } + *impression.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid), + )])); + } + let mut prebid_request = Map::new(); + if self.debug { + prebid_request.insert("debug".to_string(), Value::Bool(true)); + prebid_request.insert("returnallbidstatus".to_string(), Value::Bool(true)); + } + *extensions.request = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid_request), + )])); + Ok(()) + } + + fn prepare_outbound(&self, headers: &mut HeaderMap, transport: DemandTransport<'_>) { + apply_prebid_transport_headers( + transport.headers, + headers, + self.consent_forwarding, + transport.attested_client_ip, + ); + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + let provider_id = context.provider_id; + let response_time_ms = context.response_time_ms; + let auction_id = context.input.common_request().id.clone(); + match parse_planned_prebid_response( + provider_id, + self, + context.input, + response, + response_time_ms, + &auction_id, + ) + .await + { + Ok(parsed) => Ok(parsed), + Err(error) => { + log::warn!("Provider '{provider_id}' PBS response parse failed: {error:?}"); + Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response"))) + } + } + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +/// Append a query fragment the deployment set, without repeating one the page +/// already carries. +fn append_query_fragment(url: &str, query: &str) -> String { + if query.is_empty() || url.contains(query) { + return url.to_string(); + } + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}{query}") +} +/// The provider id [`PrebidAuctionProvider`] reports. +#[cfg(test)] const PREBID_INTEGRATION_ID: &str = "prebid"; -const PREBID_BUNDLE_ROUTE: &str = "/integrations/prebid/bundle.js"; -const PREBID_BUNDLE_CONTENT_TYPE: &str = "application/javascript; charset=utf-8"; -const PREBID_BUNDLE_IMMUTABLE_CACHE_CONTROL: &str = "public, max-age=31536000, immutable"; -const PREBID_BUNDLE_REVALIDATION_CACHE_CONTROL: &str = - "public, max-age=300, s-maxage=300, stale-while-revalidate=60, stale-if-error=86400"; -const PREBID_BUNDLE_ERROR_CACHE_CONTROL: &str = "no-store"; -const PREBID_BUNDLE_ERROR_CONTENT_TYPE: &str = "text/plain; charset=utf-8"; -const PREBID_BUNDLE_NOSNIFF_HEADER: &str = "x-content-type-options"; -const PREBID_BUNDLE_NOSNIFF_VALUE: &str = "nosniff"; + +/// The settings table [`LegacyPrebidServerConfig`] is read from. +#[cfg(test)] +const LEGACY_TABLE: &str = "auction.prebid"; + #[cfg(test)] const TRUSTED_SERVER_BIDDER: &str = "trustedServer"; #[cfg(test)] @@ -221,143 +467,19 @@ fn extract_prebid_error_message( #[cfg(test)] const GPC_US_PRIVACY: &str = "1YYN"; -/// Rejects a Prebid User ID identifier that Prebid.js could not address. -/// -/// Applies only the constraints Prebid itself imposes on a `userSync.userIds` -/// entry name and on a storage key: a non-empty ASCII token with no -/// surrounding whitespace. Anything narrower would encode one vendor's rules -/// into core. -fn validate_prebid_user_id_token(value: &str) -> Result<(), ValidationError> { - let is_valid = !value.is_empty() - && value.trim() == value - && value - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')); - if is_valid { - return Ok(()); - } - - let mut error = ValidationError::new("invalid_prebid_user_id_token"); - error.message = Some( - "must be a non-empty ASCII token of letters, digits, `_`, `-`, or `.` without surrounding whitespace" - .into(), - ); - Err(error) -} - -/// Browser storage mechanism for an operator-managed Prebid User ID module. -#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum PrebidUserIdStorageType { - /// Store the module's value in a browser cookie. - #[default] - Cookie, - /// Store the module's value in browser local storage. - Html5, -} - -/// Browser storage settings forwarded verbatim to a Prebid User ID module. -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct PrebidManagedUserIdStorage { - /// Browser storage mechanism. - #[serde(default, rename = "type")] - pub storage_type: PrebidUserIdStorageType, - /// Cookie or local-storage key the module reads and writes. - #[validate(custom(function = "validate_prebid_user_id_token"))] - pub name: String, - /// Number of days the browser retains the stored value. - /// - /// Omitted leaves Prebid's own default in place. Core applies no upper - /// bound: the ceiling is a property of the selected module, not of Trusted - /// Server. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub expires: Option, - /// Number of seconds before the module may refresh the stored value. - /// - /// Omitted leaves Prebid's own default in place. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[validate(range(min = 1))] - pub refresh_in_seconds: Option, -} - -/// Rejects a managed User ID list whose names address one module twice. -/// -/// Prebid matches `userSync.userIds` entry names to submodules -/// case-insensitively and takes the first matching entry, so two entries whose -/// names differ only by case give one submodule two conflicting configurations -/// with no defined winner. -fn validate_unique_managed_user_id_names( - entries: &[PrebidManagedUserIdConfig], -) -> Result<(), ValidationError> { - let mut seen = HashSet::with_capacity(entries.len()); - let Some(duplicate) = entries - .iter() - .find(|entry| !seen.insert(entry.name.to_ascii_lowercase())) - else { - return Ok(()); - }; - - let mut error = ValidationError::new("duplicate_managed_user_id_name"); - // Name the matching rule: for a collision that differs only by case, the - // printed name alone does not look repeated in the operator's config. - error.message = Some( - format!( - "managed Prebid User ID module `{}` is configured more than once (names are matched case-insensitively)", - duplicate.name - ) - .into(), - ); - Err(error) -} - -/// Operator-owned Prebid User ID module entry that Trusted Server manages. -/// -/// Core treats every entry as opaque: it validates only what Prebid.js needs to -/// address the module, then forwards the entry to the browser unchanged. Which -/// identity vendor an entry selects is an operator configuration choice, not a -/// property of core. -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct PrebidManagedUserIdConfig { - /// Prebid `userSync.userIds` entry name, for example `sharedId`. - #[validate(custom(function = "validate_prebid_user_id_token"))] - pub name: String, - /// Module-specific parameters, forwarded to Prebid without inspection. - #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] - pub params: serde_json::Map, - /// Optional browser storage settings for the module. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[validate(nested)] - pub storage: Option, -} - +/// The configuration of [`PrebidAuctionProvider`], the provider the tests +/// compare this implementation with. #[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] pub struct LegacyPrebidServerConfig { - #[serde(default = "default_enabled")] - pub enabled: bool, #[validate(url)] pub server_url: String, - /// Prebid Server account ID, injected into the client-side bundle via - /// `window.__tsjs_prebid.accountId` so publishers don't need to configure - /// it in JavaScript. - #[serde(default)] - pub account_id: Option, - /// Prebid User ID modules that Trusted Server installs and keeps installed. - /// - /// Each entry is forwarded to Prebid.js verbatim; publisher-configured - /// entries with other names are preserved. Names must be unique. - #[serde(default)] - #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] - pub managed_user_ids: Vec, #[serde(default = "default_timeout_ms")] #[validate(range(min = 1, max = 60000))] pub timeout_ms: u32, #[serde( default = "default_bidders", - deserialize_with = "crate::settings::vec_from_seq_or_map" + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" )] pub bidders: Vec, #[serde(default)] @@ -370,29 +492,6 @@ pub struct LegacyPrebidServerConfig { pub test_mode: bool, #[serde(default)] pub debug_query_params: Option, - /// Patterns to match Prebid script URLs for serving empty JS. - /// Supports suffix matching (e.g., "/prebid.min.js" matches any path ending with that) - /// and wildcard patterns (e.g., "/static/prebid/*" matches paths under that prefix). - #[serde( - default = "default_script_patterns", - deserialize_with = "crate::settings::vec_from_seq_or_map" - )] - pub script_patterns: Vec, - /// Absolute HTTPS URL of the generated external Prebid bundle. - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_url"))] - pub external_bundle_url: Option, - /// Optional hex SHA-256 of the exact external bundle bytes. - #[serde(default)] - #[validate(regex( - path = *EXTERNAL_BUNDLE_SHA256_PATTERN, - message = "external_bundle_sha256 must be a 64-character hex SHA-256" - ))] - pub external_bundle_sha256: Option, - /// Optional browser Subresource Integrity value for the first-party script. - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_sri"))] - pub external_bundle_sri: Option, /// Bidders that should run client-side in the browser via native Prebid.js /// adapters instead of being routed through the server-side auction. /// @@ -403,15 +502,11 @@ pub struct LegacyPrebidServerConfig { /// /// This list is independent of [`bidders`](Self::bidders) — the operator /// manages both lists explicitly. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[serde( + default, + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" + )] pub client_side_bidders: Vec, - /// GAM ad-unit-path suffixes excluded from Trusted Server refresh auctions. - /// - /// Matching is exact and case-sensitive. Excluded slots still refresh through - /// GAM, but are not included in synthetic Prebid refresh ad units. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] - pub excluded_gam_ad_unit_path_suffixes: Vec, /// Compatibility sugar for per-bidder, per-zone param overrides. /// /// This preserves the natural `bidder -> zone -> params` config shape for @@ -421,7 +516,7 @@ pub struct LegacyPrebidServerConfig { /// /// Example in TOML: /// ```toml - /// [integrations.prebid.bid_param_zone_overrides.kargo] + /// [auction.prebid.bid_param_zone_overrides.kargo] /// header = {placementId = "_s2sHeaderId"} /// in_content = {placementId = "_s2sContentId"} /// fixed_bottom = {placementId = "_s2sBottomId"} @@ -437,7 +532,7 @@ pub struct LegacyPrebidServerConfig { /// /// Example in TOML: /// ```toml - /// [integrations.prebid.bid_param_overrides.bidder-name] + /// [auction.prebid.bid_param_overrides.bidder-name] /// param1 = 12345 /// param2 = "value" /// ``` @@ -453,7 +548,7 @@ pub struct LegacyPrebidServerConfig { /// /// Example in TOML: /// ```toml - /// [[integrations.prebid.bid_param_override_rules]] + /// [[auction.prebid.bid_param_override_rules]] /// when.bidder = "kargo" /// when.zone = "header" /// set = { placementId = "_abc" } @@ -480,135 +575,15 @@ pub struct LegacyPrebidServerConfig { /// Use this when only specific PBS seats fire win/billing notifications /// internally. The global [`suppress_nurl`](Self::suppress_nurl) switch still /// suppresses every bidder when set. - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub suppress_nurl_bidders: Vec, -} - -#[cfg(test)] -impl IntegrationConfig for LegacyPrebidServerConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} - -/// CLI build inputs retained in app config but ignored safely by the runtime. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct PrebidBundleBuildConfig { - /// Typed Prebid.js module selections consumed by `ts prebid client`. - #[serde(default)] - pub modules: PrebidBundleModulesConfig, -} - -/// Exact Prebid.js module stems selected by `ts prebid client`. -/// -/// The CLI validates these values. The runtime only parses them so app config -/// carrying build inputs remains loadable. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct PrebidBundleModulesConfig { - /// Bidder adapter module stems. - #[serde(default)] - pub bidder: Vec, - /// User ID module stems; omission selects the generator's curated preset. - #[serde(default)] - pub user_id: Option>, - /// Analytics adapter module stems; omission selects no analytics adapters. - #[serde(default)] - pub analytics: Option>, -} - -/// Browser-only Prebid integration settings. -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct PrebidIntegrationConfig { - #[serde(default = "default_enabled")] - pub enabled: bool, - #[serde(default)] - pub account_id: Option, - #[serde(default = "default_timeout_ms")] - pub timeout_ms: u32, - #[serde(default)] - pub debug: bool, #[serde( - default = "default_script_patterns", - deserialize_with = "crate::settings::vec_from_seq_or_map" + default, + deserialize_with = "trusted_server_core::settings::vec_from_seq_or_map" )] - pub script_patterns: Vec, - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_url"))] - pub external_bundle_url: Option, - #[serde(default)] - #[validate(regex( - path = *EXTERNAL_BUNDLE_SHA256_PATTERN, - message = "external_bundle_sha256 must be a 64-character hex SHA-256" - ))] - pub external_bundle_sha256: Option, - #[serde(default)] - #[validate(custom(function = "validate_external_bundle_sri"))] - pub external_bundle_sri: Option, - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub client_side_bidders: Vec, - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] - pub excluded_gam_ad_unit_path_suffixes: Vec, - /// Prebid User ID modules that Trusted Server installs and keeps installed. - /// - /// Each entry is forwarded to Prebid.js verbatim; publisher-configured - /// entries with other names are preserved. Names must be unique, and no two - /// names may resolve to the same Prebid User ID submodule. - #[serde(default)] - #[validate(nested, custom(function = "validate_unique_managed_user_id_names"))] - pub managed_user_ids: Vec, - /// CLI-only external bundle build inputs; runtime registration ignores these fields. - #[serde(default)] - pub bundle: PrebidBundleBuildConfig, -} - -impl Default for PrebidIntegrationConfig { - fn default() -> Self { - Self { - enabled: default_enabled(), - account_id: None, - timeout_ms: default_timeout_ms(), - debug: false, - script_patterns: default_script_patterns(), - external_bundle_url: None, - external_bundle_sha256: None, - external_bundle_sri: None, - client_side_bidders: Vec::new(), - excluded_gam_ad_unit_path_suffixes: Vec::new(), - managed_user_ids: Vec::new(), - bundle: PrebidBundleBuildConfig::default(), - } - } -} - -impl IntegrationConfig for PrebidIntegrationConfig { - fn is_enabled(&self) -> bool { - self.enabled - } + pub suppress_nurl_bidders: Vec, } #[cfg(test)] -impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { - fn from(config: &LegacyPrebidServerConfig) -> Self { - Self { - enabled: config.enabled, - account_id: config.account_id.clone(), - timeout_ms: config.timeout_ms, - debug: config.debug, - script_patterns: config.script_patterns.clone(), - external_bundle_url: config.external_bundle_url.clone(), - external_bundle_sha256: config.external_bundle_sha256.clone(), - external_bundle_sri: config.external_bundle_sri.clone(), - client_side_bidders: config.client_side_bidders.clone(), - excluded_gam_ad_unit_path_suffixes: config.excluded_gam_ad_unit_path_suffixes.clone(), - managed_user_ids: config.managed_user_ids.clone(), - bundle: PrebidBundleBuildConfig::default(), - } - } -} +impl IntegrationConfig for LegacyPrebidServerConfig {} #[cfg(test)] fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { @@ -620,97 +595,23 @@ fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { bidders.retain(|bidder| !bidder.eq_ignore_ascii_case("aps")); if bidders.len() != original_len { log::warn!( - "prebid: ignoring APS in integrations.prebid.{field}; configure APS under [integrations.aps]" + "prebid: ignoring APS in auction.prebid.{field}; configure APS as a [demand] source with implementation = \"auction.aps\"" ); } } } -fn excluded_gam_ad_unit_path_suffix_validation_error(message: &'static str) -> ValidationError { - let mut error = ValidationError::new("invalid_gam_ad_unit_path_suffix"); - error.message = Some(message.into()); - error -} - -fn validate_excluded_gam_ad_unit_path_suffix(value: &str) -> Result<(), ValidationError> { - if value.trim() != value { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must not have surrounding whitespace", - )); - } - - if value.is_empty() { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must not be empty", - )); - } - - if !value.starts_with('/') { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must start with '/'", - )); - } - - if value == "/" { - return Err(excluded_gam_ad_unit_path_suffix_validation_error( - "excluded_gam_ad_unit_path_suffixes entries must identify a non-root path suffix", - )); - } - - Ok(()) -} - -fn validate_excluded_gam_ad_unit_path_suffixes(values: &[String]) -> Result<(), ValidationError> { - for value in values { - validate_excluded_gam_ad_unit_path_suffix(value)?; - } - - Ok(()) -} - -#[cfg(test)] -fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidServerConfig) { - let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); - for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { - if !canonical.contains(&suffix) { - canonical.push(suffix); - } - } - config.excluded_gam_ad_unit_path_suffixes = canonical; -} - #[cfg(test)] fn load_config( settings: &Settings, ) -> Result, Report> { - let Some(mut config) = - settings.integration_config::(PREBID_INTEGRATION_ID)? - else { + let Some(mut config) = settings.module_config::(LEGACY_TABLE)? else { return Ok(None); }; - canonicalize_excluded_gam_ad_unit_path_suffixes(&mut config); remove_aps_bidders(&mut config); Ok(Some(config)) } -/// Validate enabled Prebid config using the same startup-only checks as runtime registration. -/// -/// # Errors -/// -/// Returns a configuration error if enabled Prebid settings fail typed parsing, -/// schema validation, or bidder-param override compilation. -#[cfg(test)] -pub fn validate_config_for_startup( - settings: &Settings, -) -> Result, Report> { - let Some(config) = load_config(settings)? else { - return Ok(None); - }; - BidParamOverrideEngine::try_from_config(&config)?; - validate_external_bundle_config(&config, &settings.proxy.allowed_domains)?; - Ok(Some(config)) -} - /// Canonical bidder-param override rule. /// /// A rule matches against the request-time facts in [`BidParamOverrideWhen`] @@ -740,6 +641,7 @@ pub struct BidParamOverrideWhen { pub zone: Option, } +#[cfg(test)] fn default_timeout_ms() -> u32 { 1000 } @@ -749,5012 +651,2436 @@ fn default_bidders() -> Vec { vec!["mocktioneer".to_string()] } -fn default_enabled() -> bool { - true +/// Returns `true` when `params` is not a usable PBS bidder-params object — an +/// empty object `{}` or any non-object value such as `null`. +/// +/// PBS rejects an imp whose `bidder.` carries such a value, and it cannot +/// tell a fabricated empty from an explicitly supplied one — they are identical +/// bytes on the wire. The merge uses this to stop an unusable value from +/// clobbering real params, and the final pass uses it to drop whatever remains. +#[cfg(test)] +fn is_unusable_bidder_params(params: &Json) -> bool { + // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. + params.as_object().is_none_or(serde_json::Map::is_empty) } -/// Default suffixes that identify Prebid scripts -const PREBID_SCRIPT_SUFFIXES: &[&str] = &[ - "/prebid.js", - "/prebid.min.js", - "/prebidjs.js", - "/prebidjs.min.js", -]; +#[cfg(test)] +fn expand_trusted_server_bidders( + configured_bidders: &[String], + params: &Json, +) -> HashMap { + let per_bidder = params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object); + + if configured_bidders.is_empty() { + return per_bidder + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + .unwrap_or_default(); + } -fn default_script_patterns() -> Vec { - PREBID_SCRIPT_SUFFIXES + configured_bidders .iter() - .map(|&s| s.to_owned()) + .map(|bidder| { + let value = per_bidder + .and_then(|m| m.get(bidder).cloned()) + .unwrap_or_else(|| { + // No per-bidder map → use entire params as shared config + if per_bidder.is_some() { + Json::Object(Default::default()) + } else { + params.clone() + } + }); + (bidder.clone(), value) + }) .collect() } -fn validate_external_bundle_url(value: &str) -> Result<(), ValidationError> { - let url = Url::parse(value).map_err(|_| { - let mut err = ValidationError::new("invalid_external_bundle_url"); - err.message = Some("external_bundle_url must be a valid absolute URL".into()); - err - })?; - - if url.scheme() != "https" { - let mut err = ValidationError::new("invalid_external_bundle_scheme"); - err.message = Some("external_bundle_url must use https".into()); - return Err(err); - } - - if url.host_str().is_none() { - let mut err = ValidationError::new("missing_external_bundle_host"); - err.message = Some("external_bundle_url must include a host".into()); - return Err(err); - } - - Ok(()) -} - -/// Exact hex SHA-256: 64 hex digits. Used by the built-in `regex` validator on -/// [`PrebidIntegrationConfig::external_bundle_sha256`]. -static EXTERNAL_BUNDLE_SHA256_PATTERN: LazyLock = - LazyLock::new(|| Regex::new(r"^[0-9a-fA-F]{64}$").expect("SHA-256 hex regex should compile")); - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum ExternalBundleSriAlgorithm { - Sha256, - Sha384, - Sha512, -} - -impl ExternalBundleSriAlgorithm { - fn parse(value: &str) -> Option { - match value { - "sha256" => Some(Self::Sha256), - "sha384" => Some(Self::Sha384), - "sha512" => Some(Self::Sha512), - _ => None, +/// Shallow-merges `override_obj` into `params`. +/// +/// When `params` is a JSON object, each top-level key in `override_obj` is +/// inserted or replaced in `params`. Nested objects are replaced wholesale — +/// they are not recursed into. When `params` is not an object, it is left +/// untouched to preserve pre-existing behavior. +/// +/// Returns whether `params` was an object and the merge path ran. +fn merge_bidder_param_object( + params: &mut Json, + override_obj: &serde_json::Map, +) -> bool { + if let Json::Object(base) = params { + for (k, v) in override_obj { + base.insert(k.clone(), v.clone()); } - } - fn expected_digest_len(self) -> usize { - match self { - Self::Sha256 => 32, - Self::Sha384 => 48, - Self::Sha512 => 64, - } + true + } else { + false } } -fn external_bundle_sri_validation_error(message: &'static str) -> ValidationError { - let mut err = ValidationError::new("invalid_external_bundle_sri"); - err.message = Some(message.into()); - err -} - -fn parse_external_bundle_sri(value: &str) -> Result<(), ValidationError> { - let trimmed = value.trim(); - if trimmed.is_empty() || trimmed != value { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri must be non-empty with no surrounding whitespace", - )); - } - - for token in trimmed.split_ascii_whitespace() { - let Some((algorithm_raw, digest_raw)) = token.split_once('-') else { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri entries must use algorithm-digest format", - )); - }; - - let Some(algorithm) = ExternalBundleSriAlgorithm::parse(algorithm_raw) else { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri must use sha256, sha384, or sha512", - )); - }; - - if digest_raw.is_empty() { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri digest must be non-empty", - )); - } - - let digest = BASE64_STANDARD - .decode(digest_raw) - .or_else(|_| BASE64_STANDARD_NO_PAD.decode(digest_raw)) - .map_err(|_| { - external_bundle_sri_validation_error("external_bundle_sri digest must be base64") - })?; +// ============================================================================ +// Generic bid-parameter override engine +// ============================================================================ - if digest.len() != algorithm.expected_digest_len() { - return Err(external_bundle_sri_validation_error( - "external_bundle_sri digest length does not match its algorithm", - )); +#[cfg(test)] +fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { + if !config.bidders.iter().any(|b| b == bidder) { + if config.client_side_bidders.iter().any(|b| b == bidder) { + log::warn!( + "prebid: {field} entry targets client-side-only bidder \ + '{bidder}' — server-side override will never apply" + ); + } else { + log::warn!( + "prebid: {field} entry references unconfigured bidder \ + '{bidder}' — rule will never fire" + ); } } - - Ok(()) -} - -fn validate_external_bundle_sri(value: &str) -> Result<(), ValidationError> { - parse_external_bundle_sri(value) -} - -fn validate_external_bundle_url_allowed( - external_bundle_url: Option<&str>, - allowed_domains: &[String], -) -> Result<(), Report> { - let url = external_bundle_url.ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url is required when prebid is enabled" - .to_string(), - }) - })?; - - let parsed = Url::parse(url).map_err(|_| { - Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url must be a valid absolute URL" - .to_string(), - }) - })?; - - if parsed.scheme() != "https" { - return Err(Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url must use https".to_string(), - })); - } - - let host = parsed.host_str().ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: "integrations.prebid.external_bundle_url must include a host".to_string(), - }) - })?; - - if allowed_domains.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: - "proxy.allowed_domains must include the external Prebid bundle host when integrations.prebid.external_bundle_url is configured" - .to_string(), - })); - } - - if !allowed_domains - .iter() - .any(|pattern| is_host_allowed(host, pattern)) - { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "integrations.prebid.external_bundle_url host `{host}` is not permitted by proxy.allowed_domains" - ), - })); - } - - Ok(()) -} - -pub(crate) fn validate_browser_config_for_startup( - config: &PrebidIntegrationConfig, - allowed_domains: &[String], -) -> Result<(), Report> { - validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) } -pub(crate) fn validate_browser_bidder_ownership( - config: &PrebidIntegrationConfig, - plan: &AuctionPlan, -) -> Result<(), Report> { - if !plan.enabled() { - return Ok(()); - } - - let server_side = plan - .browser_bidder_codes() - .collect::>(); - let conflicts = config - .client_side_bidders - .iter() - .filter(|bidder| server_side.contains(bidder.as_str())) - .cloned() - .collect::>(); - if conflicts.is_empty() { - return Ok(()); - } - - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Prebid bidders must have exactly one browser owner; configured as both client-side and server-side: {}", - conflicts.into_iter().collect::>().join(", ") - ), - })) +#[derive(Debug, Default, Clone)] +pub(crate) struct BidParamOverrideEngine { + rules: Vec, + // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. + // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. + // Both slices are in declaration order; `apply` merges them without allocation. + bidder_index: HashMap>, + wildcard_indices: Vec, } -#[cfg(test)] -fn validate_external_bundle_config( - config: &LegacyPrebidServerConfig, - allowed_domains: &[String], -) -> Result<(), Report> { - validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) +#[derive(Debug, Clone, PartialEq, Eq)] +struct CompiledBidParamOverrideRule { + bidder: Option, + zone: Option, + set: serde_json::Map, } -pub struct PrebidIntegration { - config: PrebidIntegrationConfig, - planned_head_inserts: Option>, - #[cfg(test)] - legacy_config: Option, - #[cfg(test)] - engine: Arc, +#[derive(Debug, Copy, Clone)] +struct BidParamOverrideFacts<'a> { + bidder: &'a str, + zone: Option<&'a str>, } -impl PrebidIntegration { - #[cfg(test)] - fn try_new(config: LegacyPrebidServerConfig) -> Result, Report> { - let engine = Arc::new(BidParamOverrideEngine::try_from_config(&config)?); - Ok(Arc::new(Self { - config: PrebidIntegrationConfig::from(&config), - planned_head_inserts: None, - legacy_config: Some(config), - engine, - })) - } - +impl BidParamOverrideEngine { #[cfg(test)] - fn new(config: LegacyPrebidServerConfig) -> Arc { - Self::try_new(config).expect("should compile prebid bid param overrides") - } + fn try_from_config( + config: &LegacyPrebidServerConfig, + ) -> Result> { + let mut rules = Vec::new(); - fn for_browser_plan(config: &PrebidIntegrationConfig, plan: &AuctionPlan) -> Arc { - let mut integration = Self { - config: config.clone(), - planned_head_inserts: None, - #[cfg(test)] - legacy_config: None, - #[cfg(test)] - engine: Arc::new(BidParamOverrideEngine::default()), - }; - integration.planned_head_inserts = Some(integration.head_inserts_for_plan(config, plan)); - Arc::new(integration) - } + let mut bidder_overrides = config.bid_param_overrides.iter().collect::>(); + bidder_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - #[cfg(test)] - fn auction_provider(&self) -> PrebidAuctionProvider { - PrebidAuctionProvider { - config: self - .legacy_config - .clone() - .expect("should retain legacy config for legacy provider tests"), - bid_param_override_engine: Arc::clone(&self.engine), + for (bidder, set) in bidder_overrides { + warn_unconfigured_bidder(config, bidder, "bid_param_overrides"); + rules.push(CompiledBidParamOverrideRule::from_bidder_override( + bidder.as_str(), + set, + )?); } - } - fn matches_script_url(&self, attr_value: &str) -> bool { - let trimmed = attr_value.trim(); - let without_query = trimmed.split(['?', '#']).next().unwrap_or(trimmed); + let mut zone_overrides = config.bid_param_zone_overrides.iter().collect::>(); + zone_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); + + for (bidder, zone_override_sets) in zone_overrides { + warn_unconfigured_bidder(config, bidder, "bid_param_zone_overrides"); + let mut sorted_zone_overrides = zone_override_sets.iter().collect::>(); + sorted_zone_overrides + .sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); - if self.matches_script_pattern(without_query) { - return true; + for (zone, set) in sorted_zone_overrides { + rules.push(CompiledBidParamOverrideRule::from_zone_override( + bidder.as_str(), + zone.as_str(), + set, + )?); + } } - if !without_query.starts_with('/') - && !without_query.starts_with("//") - && !without_query.contains("://") - { - let with_slash = format!("/{without_query}"); - if self.matches_script_pattern(&with_slash) { - return true; + for rule in &config.bid_param_override_rules { + if let Some(bidder) = &rule.when.bidder { + warn_unconfigured_bidder(config, bidder, "bid_param_override_rules"); } + rules.push(CompiledBidParamOverrideRule::try_from(rule)?); } - let parsed = if without_query.starts_with("//") { - ParsedUrl::parse(&format!("https:{without_query}")) - } else { - ParsedUrl::parse(without_query) - }; + let mut bidder_index: HashMap> = HashMap::new(); + let mut wildcard_indices: Vec = Vec::new(); + for (idx, rule) in rules.iter().enumerate() { + match &rule.bidder { + Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(idx), + None => wildcard_indices.push(idx), + } + } - parsed - .ok() - .is_some_and(|url| self.matches_script_pattern(url.path())) + Ok(Self { + rules, + bidder_index, + wildcard_indices, + }) } - fn matches_script_pattern(&self, path: &str) -> bool { - // Normalize path to lowercase for case-insensitive matching - let path_lower = path.to_ascii_lowercase(); - - // Check if path matches any configured pattern - for pattern in &self.config.script_patterns { - let pattern_lower = pattern.to_ascii_lowercase(); + fn try_from_profile_config( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], + ) -> Result> { + let mut rules = Vec::new(); + for (bidder, set) in bid_param_overrides { + rules.push(CompiledBidParamOverrideRule::from_bidder_override( + bidder, set, + )?); + } + for (bidder, zone_override_sets) in bid_param_zone_overrides { + for (zone, set) in zone_override_sets { + rules.push(CompiledBidParamOverrideRule::from_zone_override( + bidder, zone, set, + )?); + } + } + for rule in bid_param_override_rules { + rules.push(CompiledBidParamOverrideRule::try_from(rule)?); + } + let mut bidder_index: HashMap> = HashMap::new(); + let mut wildcard_indices = Vec::new(); + for (index, rule) in rules.iter().enumerate() { + match &rule.bidder { + Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(index), + None => wildcard_indices.push(index), + } + } + Ok(Self { + rules, + bidder_index, + wildcard_indices, + }) + } - // Check for wildcard patterns: /* or {*name} - if pattern_lower.ends_with("/*") || pattern_lower.contains("{*") { - // Extract prefix before the wildcard - let prefix = if pattern_lower.ends_with("/*") { - &pattern_lower[..pattern_lower.len() - 1] // Remove trailing * + fn apply(&self, facts: BidParamOverrideFacts<'_>, params: &mut Json) { + let bidder_indices = self.bidder_index.get(facts.bidder).map(Vec::as_slice); + for idx in merged_rule_indices(&self.wildcard_indices, bidder_indices) { + let rule = &self.rules[idx]; + if rule.matches(facts) { + if merge_bidder_param_object(params, &rule.set) { + log::debug!( + "prebid: applying bidder param override for bidder '{}' zone {:?}: keys {:?}", + facts.bidder, + facts.zone, + rule.set.keys().collect::>() + ); } else { - // Find {* and extract prefix before it - pattern_lower.split("{*").next().unwrap_or("") - }; - - if path_lower.starts_with(prefix) { - // Check if it ends with a known Prebid script suffix - if PREBID_SCRIPT_SUFFIXES - .iter() - .any(|suffix| path_lower.ends_with(suffix)) - { - return true; - } - } - } else { - // Exact match or suffix match - if path_lower.ends_with(&pattern_lower) { - return true; + log::debug!( + "prebid: skipping bidder param override for bidder '{}' zone {:?}: params is not a JSON object", + facts.bidder, + facts.zone + ); } } } - false - } - - fn handle_script_handler( - &self, - ) -> Result, Report> { - let body = "// Script overridden by Trusted Server\n"; - - let mut response = http::Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, PREBID_BUNDLE_CONTENT_TYPE) - .body(EdgeBody::from(body)) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid script handler response".to_string(), - })?; - CacheControlPolicy::NoStorePrivate - .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); - Ok(response) - } - - fn external_bundle_script_tag(&self) -> String { - external_bundle_script_tag( - self.config.external_bundle_sha256.as_deref(), - self.config.external_bundle_sri.as_deref(), - ) - } - - /// Build the prepared browser injection from browser settings and validated routes. - pub(crate) fn head_inserts_for_plan( - &self, - browser_config: &PrebidIntegrationConfig, - plan: &AuctionPlan, - ) -> Vec { - let payload = InjectedPrebidClientConfig { - server_side_bidders: Some(if plan.enabled() { - plan.browser_bidder_codes().collect() - } else { - Vec::new() - }), - ..InjectedPrebidClientConfig::from(browser_config) - }; - let config_json = serialize_injected_prebid_config(&payload); - - vec![ - injected_prebid_config_script(&config_json), - external_bundle_script_tag( - browser_config.external_bundle_sha256.as_deref(), - browser_config.external_bundle_sri.as_deref(), - ), - ] } - fn is_managed_external(&self) -> bool { - self.config.external_bundle_url.is_some() + /// Apply compiled profile rules to already centrally routed bidder params. + pub(crate) fn apply_routed(&self, bidder: &str, zone: Option<&str>, params: &mut Json) { + self.apply(BidParamOverrideFacts { bidder, zone }, params); } +} - fn external_bundle_request_cache_mode( - &self, - req: &http::Request, - ) -> Result, Report> { - let versions = req - .uri() - .query() - .map(|query| { - url::form_urlencoded::parse(query.as_bytes()) - .filter(|(key, _)| key == "v") - .map(|(_, value)| value.into_owned()) - .collect::>() - }) - .unwrap_or_default(); +/// Validate and compile server-side Prebid profile override fields. +/// +/// This narrow hook shares the existing override compiler without coupling +/// auction-profile availability to whether the browser integration runs. +pub(crate) fn compile_profile_override_rules( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], +) -> Result> { + BidParamOverrideEngine::try_from_profile_config( + bid_param_zone_overrides, + bid_param_overrides, + bid_param_override_rules, + ) +} - if versions.len() > 1 { - return Ok(None); - } +fn merged_rule_indices<'a>( + wildcard_indices: &'a [usize], + bidder_indices: Option<&'a [usize]>, +) -> impl Iterator + 'a { + let mut wildcard = wildcard_indices.iter().copied().peekable(); + let mut bidder = bidder_indices.unwrap_or(&[]).iter().copied().peekable(); - let requested_version = versions.first().map(String::as_str); - match ( - self.config.external_bundle_sha256.as_deref(), - requested_version, - ) { - (None, Some(_)) => Ok(None), - (Some(expected), Some(actual)) if expected != actual => Ok(None), - (Some(_), Some(_)) => Ok(Some(ExternalBundleCacheMode::Immutable)), - _ => Ok(Some(ExternalBundleCacheMode::Revalidate)), - } - } + std::iter::from_fn(move || match (wildcard.peek(), bidder.peek()) { + (Some(wildcard_idx), Some(bidder_idx)) if wildcard_idx <= bidder_idx => wildcard.next(), + (Some(_), Some(_)) => bidder.next(), + (Some(_), None) => wildcard.next(), + (None, Some(_)) => bidder.next(), + (None, None) => None, + }) +} - fn apply_external_bundle_headers( - &self, - response: &mut http::Response, - mode: ExternalBundleCacheMode, - ) { - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static(PREBID_BUNDLE_CONTENT_TYPE), - ); - response.headers_mut().insert( - header::HeaderName::from_static(PREBID_BUNDLE_NOSNIFF_HEADER), - HeaderValue::from_static(PREBID_BUNDLE_NOSNIFF_VALUE), - ); +impl CompiledBidParamOverrideRule { + fn from_bidder_override( + bidder: &str, + set: &serde_json::Map, + ) -> Result> { + Self::new( + Some(bidder), + None, + set, + &format!("auction.prebid.bid_param_overrides.{bidder}"), + false, + ) + } - match mode { - ExternalBundleCacheMode::Immutable => { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(PREBID_BUNDLE_IMMUTABLE_CACHE_CONTROL), - ); - if let Some(sha256) = self.config.external_bundle_sha256.as_deref() { - response.headers_mut().insert( - header::ETAG, - HeaderValue::from_str(&format!("\"sha256:{sha256}\"")) - .expect("should build etag header"), - ); - } - } - ExternalBundleCacheMode::Revalidate => { - response.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(PREBID_BUNDLE_REVALIDATION_CACHE_CONTROL), - ); - if let Some(sha256) = self.config.external_bundle_sha256.as_deref() { - response.headers_mut().insert( - header::ETAG, - HeaderValue::from_str(&format!("\"sha256:{sha256}\"")) - .expect("should build etag header"), - ); - } - } - } + fn from_zone_override( + bidder: &str, + zone: &str, + set: &serde_json::Map, + ) -> Result> { + Self::new( + Some(bidder), + Some(zone), + set, + &format!("auction.prebid.bid_param_zone_overrides.{bidder}.{zone}"), + false, + ) } - fn sanitize_external_bundle_response( - &self, - response: http::Response, - mode: ExternalBundleCacheMode, - ) -> http::Response { - let status = response.status(); - let content_encoding = response.headers().get(header::CONTENT_ENCODING).cloned(); - let body = response.into_body(); - - let mut sanitized = http::Response::builder() - .status(status) - .body(body) - .expect("should build sanitized response"); - - if let Some(content_encoding) = content_encoding { - sanitized - .headers_mut() - .insert(header::CONTENT_ENCODING, content_encoding); - } + fn new( + bidder: Option<&str>, + zone: Option<&str>, + set: &serde_json::Map, + source: &str, + is_canonical: bool, + ) -> Result> { + let bidder = bidder + .map(|value| validate_override_matcher_string(value, "when.bidder", source)) + .transpose()?; + let zone = zone + .map(|value| validate_override_matcher_string(value, "when.zone", source)) + .transpose()?; - if status == StatusCode::OK { - self.apply_external_bundle_headers(&mut sanitized, mode); - } else { - sanitized.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static(PREBID_BUNDLE_ERROR_CONTENT_TYPE), - ); - sanitized.headers_mut().insert( - header::CACHE_CONTROL, - HeaderValue::from_static(PREBID_BUNDLE_ERROR_CACHE_CONTROL), - ); - sanitized.headers_mut().insert( - header::HeaderName::from_static(PREBID_BUNDLE_NOSNIFF_HEADER), - HeaderValue::from_static(PREBID_BUNDLE_NOSNIFF_VALUE), - ); + if bidder.is_none() && zone.is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("{source} must include at least one matcher"), + })); } - sanitized + Ok(Self { + bidder, + zone, + set: non_empty_override_object(set, source, is_canonical)?, + }) } - async fn handle_external_bundle( - &self, - settings: &Settings, - services: &RuntimeServices, - req: http::Request, - ) -> Result, Report> { - let Some(cache_mode) = self.external_bundle_request_cache_mode(&req)? else { - return Ok(http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(EdgeBody::from("Not Found")) - .expect("should build not found response")); - }; - - let target_url = self.config.external_bundle_url.as_deref().ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: - "integrations.prebid.external_bundle_url is required when prebid is enabled" - .to_string(), - }) - })?; + fn matches(&self, facts: BidParamOverrideFacts<'_>) -> bool { + if let Some(expected_bidder) = self.bidder.as_deref() + && expected_bidder != facts.bidder + { + return false; + } - let proxy_config = ProxyRequestConfig::new(target_url) - .without_ec_id() - .without_forward_headers() - .with_streaming() - .with_allowed_domains(&settings.proxy.allowed_domains) - .with_https_only(); + if let Some(expected_zone) = self.zone.as_deref() + && facts.zone != Some(expected_zone) + { + return false; + } - let response = proxy_request(settings, req, proxy_config, services).await?; - Ok(self.sanitize_external_bundle_response(response, cache_mode)) + true } } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum ExternalBundleCacheMode { - Immutable, - Revalidate, -} - -fn escape_html_attr(value: &str) -> String { - value - .replace('&', "&") - .replace('"', """) - .replace('<', "<") - .replace('>', ">") -} +impl TryFrom for CompiledBidParamOverrideRule { + type Error = Report; -fn external_bundle_script_src(sha256: Option<&str>) -> String { - match sha256 { - Some(sha256) => format!("{PREBID_BUNDLE_ROUTE}?v={sha256}"), - None => PREBID_BUNDLE_ROUTE.to_string(), + fn try_from(rule: BidParamOverrideRule) -> Result { + Self::try_from(&rule) } } -fn external_bundle_script_tag(sha256: Option<&str>, sri: Option<&str>) -> String { - let src = external_bundle_script_src(sha256); - let integrity = sri - .map(|value| format!(" integrity=\"{}\"", escape_html_attr(value))) - .unwrap_or_default(); +impl TryFrom<&BidParamOverrideRule> for CompiledBidParamOverrideRule { + type Error = Report; - format!("") + fn try_from(rule: &BidParamOverrideRule) -> Result { + Self::new( + rule.when.bidder.as_deref(), + rule.when.zone.as_deref(), + &rule.set, + "auction.prebid.bid_param_override_rules[*]", + true, + ) + } } -#[cfg(test)] -fn build( - settings: &Settings, -) -> Result>, Report> { - let Some(config) = load_config(settings)? else { - return Ok(None); - }; - - validate_external_bundle_config(&config, &settings.proxy.allowed_domains)?; +// Trims leading/trailing whitespace from config-side matcher strings so that +// accidental padding in TOML/JSON does not produce a silently non-matching rule. +// Runtime facts (bidder name, zone) are not trimmed — they come from controlled +// internal sources (bidder-map keys, trustedServer.zone) where whitespace is +// never expected. Matching is exact and case-sensitive after this normalisation. +fn validate_override_matcher_string( + value: &str, + field: &str, + source: &str, +) -> Result> { + let trimmed = value.trim(); - // Warn about bidders that appear in both lists — this is likely a config - // mistake. A bidder should be in either `bidders` (server-side) or - // `client_side_bidders` (browser-side), not both. - for bidder in &config.client_side_bidders { - if config.bidders.iter().any(|b| b == bidder) { - log::warn!( - "prebid: bidder \"{}\" is in both bidders and client_side_bidders — \ - it will run server-side AND be left for client-side, which is likely unintended", - bidder - ); - } + if trimmed.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("{source}.{field} must not be empty"), + })); } - Ok(Some(PrebidIntegration::try_new(config)?)) + Ok(trimmed.to_string()) } -/// Register the Prebid integration when enabled. -/// -/// # Errors -/// -/// Returns an error when the Prebid integration is enabled with invalid -/// configuration. -pub fn register_for_plan( - settings: &Settings, - plan: &AuctionPlan, -) -> Result, Report> { - let Some(mut config) = - settings.integration_config::(PREBID_INTEGRATION_ID)? - else { - return Ok(None); - }; - let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); - for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { - if !canonical.contains(&suffix) { - canonical.push(suffix); - } +fn non_empty_override_object( + value: &serde_json::Map, + source: &str, + is_canonical: bool, +) -> Result, Report> { + if value.is_empty() { + let message = if is_canonical { + format!("{source}.set must not be empty") + } else { + format!("{source} entry must not be empty") + }; + return Err(Report::new(TrustedServerError::Configuration { message })); } - config.excluded_gam_ad_unit_path_suffixes = canonical; - validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; - validate_browser_bidder_ownership(&config, plan)?; - let integration = PrebidIntegration::for_browser_plan(&config, plan); - Ok(Some( - IntegrationRegistration::builder(PREBID_INTEGRATION_ID) - .with_proxy(integration.clone()) - .with_attribute_rewriter(integration.clone()) - .with_head_injector(integration) - .with_deferred_js() - .build(), - )) + + Ok(value.clone()) } +/// Copies browser headers to the outgoing Prebid Server request. +/// +/// The inbound `X-Forwarded-For` header is never copied: on the server-side +/// publisher auction path `from` is the browser navigation request, so its +/// XFF value is client-supplied and spoofable. Instead the header is +/// synthesized from `client_ip` — the platform-attested client address — +/// matching the trusted IP already sent in `OpenRTB` `device.ip`. +/// +/// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are +/// stripped from the `Cookie` header since consent travels exclusively +/// through the `OpenRTB` body. #[cfg(test)] -#[allow(clippy::missing_errors_doc)] -pub fn register( - settings: &Settings, -) -> Result, Report> { - let Some(integration) = build(settings)? else { - return Ok(None); - }; - - Ok(Some( - IntegrationRegistration::builder(PREBID_INTEGRATION_ID) - .with_proxy(integration.clone()) - .with_attribute_rewriter(integration.clone()) - .with_head_injector(integration) - .with_deferred_js() - .build(), - )) +fn copy_request_headers( + from: &http::Request, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + apply_prebid_header_values( + from.headers().get(header::COOKIE), + from.headers().get(header::USER_AGENT), + from.headers().get(header::REFERER), + from.headers().get(header::ACCEPT_LANGUAGE), + to.headers_mut(), + consent_forwarding, + client_ip, + ); } -#[async_trait(?Send)] -impl IntegrationProxy for PrebidIntegration { - fn integration_name(&self) -> &'static str { - PREBID_INTEGRATION_ID - } - - fn routes(&self) -> Vec { - let mut routes = vec![]; - - routes.push(self.get("/bundle.js")); +/// Apply the common raw-header transport policy to a planned PBS request's +/// outbound headers. +pub(crate) fn apply_prebid_transport_headers( + from: &TransportHeaders, + to: &mut http::HeaderMap, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + apply_prebid_header_values( + from.cookie(), + from.user_agent(), + from.referer(), + from.accept_language(), + to, + consent_forwarding, + client_ip, + ); +} - // Register routes for script removal patterns - // Patterns can be exact paths (e.g., "/prebid.min.js") or use matchit wildcards - // (e.g., "/static/prebid/{*rest}") - for pattern in &self.config.script_patterns { - // Intentional leak: runs once at startup and patterns are small. - // `IntegrationEndpoint` requires `&'static str`. - let static_path: &'static str = Box::leak(pattern.clone().into_boxed_str()); - routes.push(IntegrationEndpoint::get(static_path)); +#[allow( + clippy::too_many_arguments, + reason = "the helper preserves four independently optional raw headers plus transport policy" +)] +fn apply_prebid_header_values( + cookie: Option<&HeaderValue>, + user_agent: Option<&HeaderValue>, + referer: Option<&HeaderValue>, + accept_language: Option<&HeaderValue>, + to: &mut http::HeaderMap, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + for (name, value) in [ + (header::USER_AGENT, user_agent), + (header::REFERER, referer), + (header::ACCEPT_LANGUAGE, accept_language), + ] { + if let Some(value) = value { + to.insert(name, value.clone()); } + } - routes + if let Some(ip) = client_ip + && let Ok(value) = HeaderValue::from_str(&ip.to_string()) + { + to.insert(header::HeaderName::from_static("x-forwarded-for"), value); } - async fn handle( - &self, - settings: &Settings, - services: &RuntimeServices, - req: http::Request, - ) -> Result, Report> { - let path = req.uri().path().to_string(); - let method = req.method().clone(); - - match method { - Method::GET if self.is_managed_external() && path == PREBID_BUNDLE_ROUTE => { - self.handle_external_bundle(settings, services, req).await + let Some(cookie_value) = cookie else { + return; + }; + if !consent_forwarding.strips_consent_cookies() { + to.insert(header::COOKIE, cookie_value.clone()); + return; + } + match cookie_value.to_str() { + Ok(value) => { + let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); + if !stripped.is_empty() + && let Ok(cookie_header) = HeaderValue::from_str(&stripped) + { + to.insert(header::COOKIE, cookie_header); } - // Serve empty JS for matching script patterns - Method::GET if self.matches_script_pattern(&path) => self.handle_script_handler(), - _ => http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(EdgeBody::from("Not Found")) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid not found response".to_string(), - }), + } + Err(_) => { + to.insert(header::COOKIE, cookie_value.clone()); } } } -impl IntegrationAttributeRewriter for PrebidIntegration { - fn integration_id(&self) -> &'static str { - PREBID_INTEGRATION_ID - } - - fn handles_attribute(&self, attribute: &str) -> bool { - matches!(attribute, "src" | "href") +/// Appends query parameters to a URL, handling both URLs with and without existing query strings. +/// Returns the original URL unchanged if params are empty or already present. +#[cfg(test)] +fn append_query_params(url: &str, params: &str) -> String { + if params.is_empty() || url.contains(params) { + return url.to_string(); } - - fn rewrite( - &self, - _attr_name: &str, - attr_value: &str, - _ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if self.matches_script_url(attr_value) { - AttributeRewriteAction::remove_element() - } else { - AttributeRewriteAction::keep() - } + if url.contains('?') { + format!("{}&{}", url, params) + } else { + format!("{}?{}", url, params) } } -fn serialize_injected_prebid_config(payload: &impl Serialize) -> String { - // JSON appears in script raw-text, where every less-than sign must be escaped. - serde_json::to_string(payload) - .unwrap_or_else(|error| { - log::warn!("Prebid: failed to serialize client config: {error}"); - "{}".to_string() - }) - .replace('<', "\\u003c") -} - -fn injected_prebid_config_script(config_json: &str) -> String { - format!( - r#""# +/// Parse a planned PBS response with the configured profile behavior. +/// +/// This preserves the legacy PBS status, body, bid, cache, and metadata +/// semantics while allowing each planned provider to retain its own identity. +pub(crate) async fn parse_planned_prebid_response( + provider_id: &str, + demand: &PrebidServerDemand, + input: &ProviderAuctionInput, + response: PlatformResponse, + response_time_ms: u64, + auction_id: &str, +) -> Result> { + let response = response.response; + let status = response.status(); + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body_bytes = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + "prebid", ) -} - -impl IntegrationHeadInjector for PrebidIntegration { - fn integration_id(&self) -> &'static str { - PREBID_INTEGRATION_ID - } + .await + .change_context(TrustedServerError::Integration { + integration: MODULE.to_string(), + message: "Failed to read Prebid response body".to_string(), + }); - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - if let Some(inserts) = &self.planned_head_inserts { - return inserts.clone(); - } - let payload = InjectedPrebidClientConfig { - bidders: Some({ - #[cfg(test)] - { - self.legacy_config - .as_ref() - .map_or(&[][..], |config| config.bidders.as_slice()) - } - #[cfg(not(test))] - { - &[] - } - }), - ..InjectedPrebidClientConfig::from(&self.config) + if !status.is_success() { + log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); + let body_bytes = match body_bytes { + Ok(body_bytes) => Some(body_bytes), + Err(error) => { + log::warn!( + "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" + ); + None + } }; + if demand.debug + && let Some(body_bytes) = body_bytes.as_deref() + { + match prebid_body_preview(body_bytes) { + Some(preview) => { + let truncation = if preview.truncated { + " (truncated)" + } else { + "" + }; + log::warn!( + "Prebid auction {auction_id:?} error response body preview{truncation}: {}", + preview.text + ); + } + None => log::warn!( + "Prebid auction {auction_id:?} returned an empty error response body" + ), + } + } - let config_json = serialize_injected_prebid_config(&payload); - let mut inserts = vec![injected_prebid_config_script(&config_json)]; - - inserts.push(self.external_bundle_script_tag()); - - inserts + let status_code = status.as_u16(); + let mut parsed = AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) + .with_metadata("http_status", serde_json::json!(status_code)) + .with_metadata( + "message", + serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), + ); + if demand.debug + && let Some(message) = body_bytes + .as_deref() + .and_then(|body| extract_prebid_error_message(body, content_type.as_deref())) + { + parsed.metadata.insert( + "upstream_message".to_string(), + serde_json::json!(message.text), + ); + parsed.metadata.insert( + "upstream_message_truncated".to_string(), + serde_json::json!(message.truncated), + ); + } + return Ok(parsed); } -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct InjectedManagedUserIdStorage<'a> { - #[serde(rename = "type")] - storage_type: PrebidUserIdStorageType, - name: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - expires: Option, - #[serde(skip_serializing_if = "Option::is_none")] - refresh_in_seconds: Option, -} -impl<'a> From<&'a PrebidManagedUserIdStorage> for InjectedManagedUserIdStorage<'a> { - fn from(storage: &'a PrebidManagedUserIdStorage) -> Self { - Self { - storage_type: storage.storage_type, - name: &storage.name, - expires: storage.expires, - refresh_in_seconds: storage.refresh_in_seconds, + let body_bytes = body_bytes?; + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Integration { + integration: MODULE.to_string(), + message: "Failed to parse Prebid response".to_string(), + })?; + if demand.debug && log::log_enabled!(log::Level::Trace) { + match serde_json::to_string_pretty(&response_json) { + Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), + Err(error) => log::warn!("Prebid: failed to serialize response for logging: {error}"), } } -} -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct InjectedManagedUserId<'a> { - name: &'a str, - #[serde(skip_serializing_if = "serde_json::Map::is_empty")] - params: &'a serde_json::Map, - #[serde(skip_serializing_if = "Option::is_none")] - storage: Option>, + let mut parsed = + parse_planned_prebid_openrtb(provider_id, input, &response_json, response_time_ms); + enrich_planned_prebid_metadata(demand, &response_json, &mut parsed); + log::info!( + "Prebid provider {provider_id} returned {} bids in {}ms", + parsed.bids.len(), + response_time_ms + ); + Ok(parsed) } -impl<'a> From<&'a PrebidManagedUserIdConfig> for InjectedManagedUserId<'a> { - fn from(config: &'a PrebidManagedUserIdConfig) -> Self { - Self { - name: &config.name, - params: &config.params, - storage: config - .storage - .as_ref() - .map(InjectedManagedUserIdStorage::from), +fn parse_planned_prebid_openrtb( + provider_id: &str, + input: &ProviderAuctionInput, + response_json: &Json, + response_time_ms: u64, +) -> AuctionResponse { + let Some(response) = response_json.as_object() else { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); + }; + match response.get("cur") { + None => {} + Some(Json::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} + Some(Json::String(currency)) => { + return AuctionResponse::no_bid(provider_id, response_time_ms) + .with_metadata("unsupported_currency", serde_json::json!(currency)); + } + Some(_) => { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); } } -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct InjectedPrebidClientConfig<'a> { - account_id: &'a str, - #[serde(skip_serializing_if = "Vec::is_empty")] - managed_user_ids: Vec>, - timeout: u32, - debug: bool, - #[serde(skip_serializing_if = "Option::is_none")] - bidders: Option<&'a [String]>, - #[serde(skip_serializing_if = "Option::is_none")] - server_side_bidders: Option>, - #[serde(skip_serializing_if = "<[String]>::is_empty")] - client_side_bidders: &'a [String], - #[serde(skip_serializing_if = "<[String]>::is_empty")] - excluded_gam_ad_unit_path_suffixes: &'a [String], -} -impl<'a> From<&'a PrebidIntegrationConfig> for InjectedPrebidClientConfig<'a> { - fn from(config: &'a PrebidIntegrationConfig) -> Self { - Self { - account_id: config.account_id.as_deref().unwrap_or_default(), - managed_user_ids: config - .managed_user_ids - .iter() - .map(InjectedManagedUserId::from) - .collect(), - timeout: config.timeout_ms, - debug: config.debug, - bidders: None, - server_side_bidders: None, - client_side_bidders: &config.client_side_bidders, - excluded_gam_ad_unit_path_suffixes: &config.excluded_gam_ad_unit_path_suffixes, + let dimensions_by_slot = build_bid_dimension_index(input); + let mut diagnostics = ResponseAdmissionDiagnostics::default(); + let mut bids = Vec::new(); + if let Some(seatbids) = response_json.get("seatbid").and_then(Json::as_array) { + for seatbid in seatbids { + let returned_seat = seatbid + .get("seat") + .and_then(Json::as_str) + .filter(|seat| !seat.is_empty()); + let delivery_bidder = returned_seat.unwrap_or("unknown"); + if let Some(entries) = seatbid.get("bid").and_then(Json::as_array) { + for entry in entries { + match parse_planned_prebid_bid( + entry, + delivery_bidder, + returned_seat, + &dimensions_by_slot, + ) { + Ok(bid) => bids.push(bid), + Err(reason) => { + diagnostics.record(reason); + if reason == BidRejectionReason::InvalidBid { + let impression = entry + .get("impid") + .and_then(Json::as_str) + .unwrap_or(""); + log::warn!( + "Prebid: failed to parse bid from seat '{delivery_bidder}' for imp '{impression}'" + ); + } + } + } + } + } } } + let mut parsed = if bids.is_empty() { + AuctionResponse::no_bid(provider_id, response_time_ms) + } else { + AuctionResponse::success(provider_id, bids, response_time_ms) + }; + diagnostics.attach_to(&mut parsed); + parsed } -/// Returns `true` when `params` is not a usable PBS bidder-params object — an -/// empty object `{}` or any non-object value such as `null`. -/// -/// PBS rejects an imp whose `bidder.` carries such a value, and it cannot -/// tell a fabricated empty from an explicitly supplied one — they are identical -/// bytes on the wire. The merge uses this to stop an unusable value from -/// clobbering real params, and the final pass uses it to drop whatever remains. -#[cfg(test)] -fn is_unusable_bidder_params(params: &Json) -> bool { - // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. - params.as_object().is_none_or(serde_json::Map::is_empty) +fn enrich_planned_prebid_metadata( + demand: &PrebidServerDemand, + response_json: &Json, + parsed: &mut AuctionResponse, +) { + let ext = response_json.get("ext"); + for key in ["responsetimemillis", "errors", "warnings"] { + if let Some(value) = ext.and_then(|ext| ext.get(key)) { + parsed.metadata.insert(key.to_string(), value.clone()); + } + } + if demand.debug { + if let Some(value) = ext.and_then(|ext| ext.get("debug")) { + parsed.metadata.insert("debug".to_string(), value.clone()); + } + if let Some(value) = ext + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("bidstatus")) + { + parsed + .metadata + .insert("bidstatus".to_string(), value.clone()); + } + } } -#[cfg(test)] -fn expand_trusted_server_bidders( - configured_bidders: &[String], - params: &Json, -) -> HashMap { - let per_bidder = params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object); - - if configured_bidders.is_empty() { - return per_bidder - .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) - .unwrap_or_default(); - } - - configured_bidders - .iter() - .map(|bidder| { - let value = per_bidder - .and_then(|m| m.get(bidder).cloned()) - .unwrap_or_else(|| { - // No per-bidder map → use entire params as shared config - if per_bidder.is_some() { - Json::Object(Default::default()) - } else { - params.clone() - } - }); - (bidder.clone(), value) +fn parse_planned_prebid_bid( + bid: &Json, + delivery_bidder: &str, + returned_seat: Option<&str>, + dimensions_by_slot: &BidDimensionIndex, +) -> Result { + let slot_id = bid + .get("impid") + .and_then(Json::as_str) + .filter(|slot_id| !slot_id.is_empty()) + .ok_or(BidRejectionReason::InvalidBid)? + .to_string(); + let width = parse_optional_bid_dimension(bid, "w")?; + let height = parse_optional_bid_dimension(bid, "h")?; + let (width, height) = resolve_bid_dimensions(dimensions_by_slot, &slot_id, width, height)?; + let price = bid + .get("price") + .and_then(Json::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or(BidRejectionReason::InvalidBid)?; + let creative = bid.get("adm").and_then(Json::as_str).map(String::from); + let cache_entry = bid + .get("ext") + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("cache")) + .and_then(|cache| cache.get("bids")); + let cache_id = cache_entry + .and_then(|cache| cache.get("cacheId")) + .and_then(Json::as_str) + .map(String::from); + let (cache_host, cache_path) = cache_entry + .and_then(|cache| cache.get("url")) + .and_then(Json::as_str) + .and_then(|value| { + ParsedUrl::parse(value) + .map_err(|error| log::debug!("PBS cache URL parse failed: {error}")) + .ok() }) - .collect() -} - -/// Shallow-merges `override_obj` into `params`. -/// -/// When `params` is a JSON object, each top-level key in `override_obj` is -/// inserted or replaced in `params`. Nested objects are replaced wholesale — -/// they are not recursed into. When `params` is not an object, it is left -/// untouched to preserve pre-existing behavior. -/// -/// Returns whether `params` was an object and the merge path ran. -fn merge_bidder_param_object( - params: &mut Json, - override_obj: &serde_json::Map, -) -> bool { - if let Json::Object(base) = params { - for (k, v) in override_obj { - base.insert(k.clone(), v.clone()); - } - - true - } else { - false + .map(|url| { + let host = url.host_str().map(String::from); + let path = url.path().to_string(); + let path = (!path.is_empty() && path != "/").then_some(path); + (host, path) + }) + .unwrap_or((None, None)); + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — creative will fail to render for slot '{slot_id}'" + ); } + + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain: bid.get("adomain").and_then(Json::as_array).map(|domains| { + domains + .iter() + .filter_map(Json::as_str) + .map(String::from) + .collect() + }), + bidder: delivery_bidder.to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: bid.get("nurl").and_then(Json::as_str).map(String::from), + burl: bid.get("burl").and_then(Json::as_str).map(String::from), + bid_id: bid + .get("id") + .and_then(Json::as_str) + .filter(|value| !value.is_empty()) + .map(String::from), + ad_id: bid.get("adid").and_then(Json::as_str).map(String::from), + creative_id: bid.get("crid").and_then(Json::as_str).map(String::from), + renderer: None, + cache_id, + cache_host, + cache_path, + metadata: HashMap::new(), + }) } // ============================================================================ -// Generic bid-parameter override engine +// Prebid Auction Provider // ============================================================================ +/// The request extension [`PrebidAuctionProvider`] writes. #[cfg(test)] -fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { - if !config.bidders.iter().any(|b| b == bidder) { - if config.client_side_bidders.iter().any(|b| b == bidder) { - log::warn!( - "prebid: {field} entry targets client-side-only bidder \ - '{bidder}' — server-side override will never apply" - ); - } else { - log::warn!( - "prebid: {field} entry references unconfigured bidder \ - '{bidder}' — rule will never fire" - ); - } - } +#[derive(Debug, Serialize)] +struct RequestExt { + #[serde(skip_serializing_if = "Option::is_none")] + prebid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + trusted_server: Option, } -#[derive(Debug, Default, Clone)] -pub(crate) struct BidParamOverrideEngine { - rules: Vec, - // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. - // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. - // Both slices are in declaration order; `apply` merges them without allocation. - bidder_index: HashMap>, - wildcard_indices: Vec, -} +#[cfg(test)] +impl ToExt for RequestExt {} -#[derive(Debug, Clone, PartialEq, Eq)] -struct CompiledBidParamOverrideRule { - bidder: Option, - zone: Option, - set: serde_json::Map, +/// The `ext.prebid` object of a request. +#[cfg(test)] +#[derive(Debug, Serialize, Deserialize)] +struct PrebidExt { + #[serde(skip_serializing_if = "Option::is_none")] + debug: Option, + #[serde(skip_serializing_if = "Option::is_none")] + returnallbidstatus: Option, } -#[derive(Debug, Copy, Clone)] -struct BidParamOverrideFacts<'a> { - bidder: &'a str, - zone: Option<&'a str>, +/// The impression extension [`PrebidAuctionProvider`] writes. +#[cfg(test)] +#[derive(Debug, Serialize)] +struct ImpExt { + prebid: PrebidImpExt, } -impl BidParamOverrideEngine { - #[cfg(test)] - fn try_from_config( - config: &LegacyPrebidServerConfig, - ) -> Result> { - let mut rules = Vec::new(); - - let mut bidder_overrides = config.bid_param_overrides.iter().collect::>(); - bidder_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); +#[cfg(test)] +impl ToExt for ImpExt {} - for (bidder, set) in bidder_overrides { - warn_unconfigured_bidder(config, bidder, "bid_param_overrides"); - rules.push(CompiledBidParamOverrideRule::from_bidder_override( - bidder.as_str(), - set, - )?); - } +/// The `imp.ext.prebid` object: inline bidder parameters, or a stored +/// request reference. +#[cfg(test)] +#[derive(Debug, Default, Serialize)] +struct PrebidImpExt { + #[serde(skip_serializing_if = "HashMap::is_empty")] + bidder: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + storedrequest: Option, +} - let mut zone_overrides = config.bid_param_zone_overrides.iter().collect::>(); - zone_overrides.sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); +/// PBS imp-level stored request reference. +/// +/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, +/// populating bidder params that are not sent inline. +#[cfg(test)] +#[derive(Debug, Serialize)] +struct ImpStoredRequest { + id: String, +} - for (bidder, zone_override_sets) in zone_overrides { - warn_unconfigured_bidder(config, bidder, "bid_param_zone_overrides"); - let mut sorted_zone_overrides = zone_override_sets.iter().collect::>(); - sorted_zone_overrides - .sort_by(|(left, _), (right, _)| left.as_str().cmp(right.as_str())); +/// Legacy Prebid Server auction provider retained only for parity tests. +#[cfg(test)] +pub struct PrebidAuctionProvider { + config: LegacyPrebidServerConfig, + bid_param_override_engine: Arc, +} - for (zone, set) in sorted_zone_overrides { - rules.push(CompiledBidParamOverrideRule::from_zone_override( - bidder.as_str(), - zone.as_str(), - set, - )?); - } - } +#[cfg(test)] +#[derive(Default)] +struct PrebidImpressionDisposition { + aps_only: usize, + invalid: usize, +} - for rule in &config.bid_param_override_rules { - if let Some(bidder) = &rule.when.bidder { - warn_unconfigured_bidder(config, bidder, "bid_param_override_rules"); - } - rules.push(CompiledBidParamOverrideRule::try_from(rule)?); - } +#[cfg(test)] +struct PrebidRequestBuild { + request: OpenRtbRequest, + disposition: PrebidImpressionDisposition, +} - let mut bidder_index: HashMap> = HashMap::new(); - let mut wildcard_indices: Vec = Vec::new(); - for (idx, rule) in rules.iter().enumerate() { - match &rule.bidder { - Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(idx), - None => wildcard_indices.push(idx), - } - } +#[cfg(test)] +impl PrebidAuctionProvider { + #[cfg(test)] + fn new(config: LegacyPrebidServerConfig) -> Self { + Self::try_new(config).expect("should compile prebid bid param overrides") + } + /// Create a new Prebid auction provider with validated override rules. + /// + /// # Errors + /// + /// Returns an error when the configured bidder-param override rules are invalid. + pub fn try_new(config: LegacyPrebidServerConfig) -> Result> { Ok(Self { - rules, - bidder_index, - wildcard_indices, + bid_param_override_engine: Arc::new(BidParamOverrideEngine::try_from_config(&config)?), + config, }) } - fn try_from_profile_config( - bid_param_zone_overrides: &std::collections::BTreeMap< - String, - std::collections::BTreeMap>, - >, - bid_param_overrides: &std::collections::BTreeMap>, - bid_param_override_rules: &[BidParamOverrideRule], - ) -> Result> { - let mut rules = Vec::new(); - for (bidder, set) in bid_param_overrides { - rules.push(CompiledBidParamOverrideRule::from_bidder_override( - bidder, set, - )?); - } - for (bidder, zone_override_sets) in bid_param_zone_overrides { - for (zone, set) in zone_override_sets { - rules.push(CompiledBidParamOverrideRule::from_zone_override( - bidder, zone, set, - )?); - } - } - for rule in bid_param_override_rules { - rules.push(CompiledBidParamOverrideRule::try_from(rule)?); - } - let mut bidder_index: HashMap> = HashMap::new(); - let mut wildcard_indices = Vec::new(); - for (index, rule) in rules.iter().enumerate() { - match &rule.bidder { - Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(index), - None => wildcard_indices.push(index), - } - } - Ok(Self { - rules, - bidder_index, - wildcard_indices, - }) - } - - fn apply(&self, facts: BidParamOverrideFacts<'_>, params: &mut Json) { - let bidder_indices = self.bidder_index.get(facts.bidder).map(Vec::as_slice); - for idx in merged_rule_indices(&self.wildcard_indices, bidder_indices) { - let rule = &self.rules[idx]; - if rule.matches(facts) { - if merge_bidder_param_object(params, &rule.set) { - log::debug!( - "prebid: applying bidder param override for bidder '{}' zone {:?}: keys {:?}", - facts.bidder, - facts.zone, - rule.set.keys().collect::>() - ); - } else { - log::debug!( - "prebid: skipping bidder param override for bidder '{}' zone {:?}: params is not a JSON object", - facts.bidder, - facts.zone - ); - } - } - } - } - - /// Apply compiled profile rules to already centrally routed bidder params. - pub(crate) fn apply_routed(&self, bidder: &str, zone: Option<&str>, params: &mut Json) { - self.apply(BidParamOverrideFacts { bidder, zone }, params); - } -} - -/// Validate and compile server-side Prebid profile override fields. -/// -/// This narrow hook shares the existing override compiler without coupling -/// auction-profile availability to the browser integration's enablement. -pub(crate) fn compile_profile_override_rules( - bid_param_zone_overrides: &std::collections::BTreeMap< - String, - std::collections::BTreeMap>, - >, - bid_param_overrides: &std::collections::BTreeMap>, - bid_param_override_rules: &[BidParamOverrideRule], -) -> Result> { - BidParamOverrideEngine::try_from_profile_config( - bid_param_zone_overrides, - bid_param_overrides, - bid_param_override_rules, - ) -} - -fn merged_rule_indices<'a>( - wildcard_indices: &'a [usize], - bidder_indices: Option<&'a [usize]>, -) -> impl Iterator + 'a { - let mut wildcard = wildcard_indices.iter().copied().peekable(); - let mut bidder = bidder_indices.unwrap_or(&[]).iter().copied().peekable(); - - std::iter::from_fn(move || match (wildcard.peek(), bidder.peek()) { - (Some(wildcard_idx), Some(bidder_idx)) if wildcard_idx <= bidder_idx => wildcard.next(), - (Some(_), Some(_)) => bidder.next(), - (Some(_), None) => wildcard.next(), - (None, Some(_)) => bidder.next(), - (None, None) => None, - }) -} - -impl CompiledBidParamOverrideRule { - fn from_bidder_override( - bidder: &str, - set: &serde_json::Map, - ) -> Result> { - Self::new( - Some(bidder), - None, - set, - &format!("integrations.prebid.bid_param_overrides.{bidder}"), - false, - ) - } - - fn from_zone_override( - bidder: &str, - zone: &str, - set: &serde_json::Map, - ) -> Result> { - Self::new( - Some(bidder), - Some(zone), - set, - &format!("integrations.prebid.bid_param_zone_overrides.{bidder}.{zone}"), - false, - ) - } - - fn new( - bidder: Option<&str>, - zone: Option<&str>, - set: &serde_json::Map, - source: &str, - is_canonical: bool, - ) -> Result> { - let bidder = bidder - .map(|value| validate_override_matcher_string(value, "when.bidder", source)) - .transpose()?; - let zone = zone - .map(|value| validate_override_matcher_string(value, "when.zone", source)) - .transpose()?; - - if bidder.is_none() && zone.is_none() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{source} must include at least one matcher"), - })); - } - - Ok(Self { - bidder, - zone, - set: non_empty_override_object(set, source, is_canonical)?, - }) - } - - fn matches(&self, facts: BidParamOverrideFacts<'_>) -> bool { - if let Some(expected_bidder) = self.bidder.as_deref() - && expected_bidder != facts.bidder - { - return false; - } - - if let Some(expected_zone) = self.zone.as_deref() - && facts.zone != Some(expected_zone) - { - return false; - } - - true - } -} - -impl TryFrom for CompiledBidParamOverrideRule { - type Error = Report; - - fn try_from(rule: BidParamOverrideRule) -> Result { - Self::try_from(&rule) - } -} - -impl TryFrom<&BidParamOverrideRule> for CompiledBidParamOverrideRule { - type Error = Report; - - fn try_from(rule: &BidParamOverrideRule) -> Result { - Self::new( - rule.when.bidder.as_deref(), - rule.when.zone.as_deref(), - &rule.set, - "integrations.prebid.bid_param_override_rules[*]", - true, - ) - } -} - -// Trims leading/trailing whitespace from config-side matcher strings so that -// accidental padding in TOML/JSON does not produce a silently non-matching rule. -// Runtime facts (bidder name, zone) are not trimmed — they come from controlled -// internal sources (bidder-map keys, trustedServer.zone) where whitespace is -// never expected. Matching is exact and case-sensitive after this normalisation. -fn validate_override_matcher_string( - value: &str, - field: &str, - source: &str, -) -> Result> { - let trimmed = value.trim(); - - if trimmed.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("{source}.{field} must not be empty"), - })); - } - - Ok(trimmed.to_string()) -} - -fn non_empty_override_object( - value: &serde_json::Map, - source: &str, - is_canonical: bool, -) -> Result, Report> { - if value.is_empty() { - let message = if is_canonical { - format!("{source}.set must not be empty") - } else { - format!("{source} entry must not be empty") - }; - return Err(Report::new(TrustedServerError::Configuration { message })); - } - - Ok(value.clone()) -} - -/// Copies browser headers to the outgoing Prebid Server request. -/// -/// The inbound `X-Forwarded-For` header is never copied: on the server-side -/// publisher auction path `from` is the browser navigation request, so its -/// XFF value is client-supplied and spoofable. Instead the header is -/// synthesized from `client_ip` — the platform-attested client address — -/// matching the trusted IP already sent in `OpenRTB` `device.ip`. -/// -/// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are -/// stripped from the `Cookie` header since consent travels exclusively -/// through the `OpenRTB` body. -#[cfg(test)] -fn copy_request_headers( - from: &http::Request, - to: &mut http::Request, - consent_forwarding: ConsentForwardingMode, - client_ip: Option, -) { - apply_prebid_header_values( - from.headers().get(header::COOKIE), - from.headers().get(header::USER_AGENT), - from.headers().get(header::REFERER), - from.headers().get(header::ACCEPT_LANGUAGE), - to, - consent_forwarding, - client_ip, - ); -} - -/// Apply the common raw-header transport policy for a planned PBS request. -pub(crate) fn apply_prebid_transport_headers( - from: &PrebidTransportHeaders, - to: &mut http::Request, - consent_forwarding: ConsentForwardingMode, - client_ip: Option, -) { - apply_prebid_header_values( - from.cookie(), - from.user_agent(), - from.referer(), - from.accept_language(), - to, - consent_forwarding, - client_ip, - ); -} - -#[allow( - clippy::too_many_arguments, - reason = "the helper preserves four independently optional raw headers plus transport policy" -)] -fn apply_prebid_header_values( - cookie: Option<&HeaderValue>, - user_agent: Option<&HeaderValue>, - referer: Option<&HeaderValue>, - accept_language: Option<&HeaderValue>, - to: &mut http::Request, - consent_forwarding: ConsentForwardingMode, - client_ip: Option, -) { - for (name, value) in [ - (header::USER_AGENT, user_agent), - (header::REFERER, referer), - (header::ACCEPT_LANGUAGE, accept_language), - ] { - if let Some(value) = value { - to.headers_mut().insert(name, value.clone()); - } - } - - if let Some(ip) = client_ip - && let Ok(value) = HeaderValue::from_str(&ip.to_string()) - { - to.headers_mut() - .insert(header::HeaderName::from_static("x-forwarded-for"), value); - } - - let Some(cookie_value) = cookie else { - return; - }; - if !consent_forwarding.strips_consent_cookies() { - to.headers_mut() - .insert(header::COOKIE, cookie_value.clone()); - return; - } - match cookie_value.to_str() { - Ok(value) => { - let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); - if !stripped.is_empty() - && let Ok(cookie_header) = HeaderValue::from_str(&stripped) - { - to.headers_mut().insert(header::COOKIE, cookie_header); - } - } - Err(_) => { - to.headers_mut() - .insert(header::COOKIE, cookie_value.clone()); - } - } -} - -/// Appends query parameters to a URL, handling both URLs with and without existing query strings. -/// Returns the original URL unchanged if params are empty or already present. -#[cfg(test)] -fn append_query_params(url: &str, params: &str) -> String { - if params.is_empty() || url.contains(params) { - return url.to_string(); - } - if url.contains('?') { - format!("{}&{}", url, params) - } else { - format!("{}?{}", url, params) - } -} - -/// Parse a planned PBS response with the configured profile behavior. -/// -/// This preserves the legacy PBS status, body, bid, cache, and metadata -/// semantics while allowing each planned provider to retain its own identity. -pub(crate) async fn parse_planned_prebid_response( - provider_id: &str, - profile: &PrebidProfilePlan, - input: &ProviderAuctionInput, - response: PlatformResponse, - response_time_ms: u64, - auction_id: &str, -) -> Result> { - let response = response.response; - let status = response.status(); - let content_type = response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "prebid", - ) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to read Prebid response body".to_string(), - }); - - if !status.is_success() { - log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); - let body_bytes = match body_bytes { - Ok(body_bytes) => Some(body_bytes), - Err(error) => { - log::warn!( - "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" - ); - None - } - }; - if profile.debug - && let Some(body_bytes) = body_bytes.as_deref() - { - match prebid_body_preview(body_bytes) { - Some(preview) => { - let truncation = if preview.truncated { - " (truncated)" - } else { - "" - }; - log::warn!( - "Prebid auction {auction_id:?} error response body preview{truncation}: {}", - preview.text - ); - } - None => log::warn!( - "Prebid auction {auction_id:?} returned an empty error response body" - ), - } - } - - let status_code = status.as_u16(); - let mut parsed = AuctionResponse::error(provider_id, response_time_ms) - .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) - .with_metadata("http_status", serde_json::json!(status_code)) - .with_metadata( - "message", - serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), - ); - if profile.debug - && let Some(message) = body_bytes - .as_deref() - .and_then(|body| extract_prebid_error_message(body, content_type.as_deref())) - { - parsed.metadata.insert( - "upstream_message".to_string(), - serde_json::json!(message.text), - ); - parsed.metadata.insert( - "upstream_message_truncated".to_string(), - serde_json::json!(message.truncated), - ); - } - return Ok(parsed); - } - - let body_bytes = body_bytes?; - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { - message: "Failed to parse Prebid response".to_string(), - })?; - if profile.debug && log::log_enabled!(log::Level::Trace) { - match serde_json::to_string_pretty(&response_json) { - Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), - Err(error) => log::warn!("Prebid: failed to serialize response for logging: {error}"), - } - } - - let mut parsed = - parse_planned_prebid_openrtb(provider_id, input, &response_json, response_time_ms); - enrich_planned_prebid_metadata(profile, &response_json, &mut parsed); - log::info!( - "Prebid provider {provider_id} returned {} bids in {}ms", - parsed.bids.len(), - response_time_ms - ); - Ok(parsed) -} - -fn parse_planned_prebid_openrtb( - provider_id: &str, - input: &ProviderAuctionInput, - response_json: &Json, - response_time_ms: u64, -) -> AuctionResponse { - let Some(response) = response_json.as_object() else { - return AuctionResponse::error(provider_id, response_time_ms) - .with_metadata("error_type", serde_json::json!("parse_response")); - }; - match response.get("cur") { - None => {} - Some(Json::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} - Some(Json::String(currency)) => { - return AuctionResponse::no_bid(provider_id, response_time_ms) - .with_metadata("unsupported_currency", serde_json::json!(currency)); - } - Some(_) => { - return AuctionResponse::error(provider_id, response_time_ms) - .with_metadata("error_type", serde_json::json!("parse_response")); - } - } - - let dimensions_by_slot = build_bid_dimension_index(input); - let mut diagnostics = ResponseAdmissionDiagnostics::default(); - let mut bids = Vec::new(); - if let Some(seatbids) = response_json.get("seatbid").and_then(Json::as_array) { - for seatbid in seatbids { - let returned_seat = seatbid - .get("seat") - .and_then(Json::as_str) - .filter(|seat| !seat.is_empty()); - let delivery_bidder = returned_seat.unwrap_or("unknown"); - if let Some(entries) = seatbid.get("bid").and_then(Json::as_array) { - for entry in entries { - match parse_planned_prebid_bid( - entry, - delivery_bidder, - returned_seat, - &dimensions_by_slot, - ) { - Ok(bid) => bids.push(bid), - Err(reason) => { - diagnostics.record(reason); - if reason == BidRejectionReason::InvalidBid { - let impression = entry - .get("impid") - .and_then(Json::as_str) - .unwrap_or(""); - log::warn!( - "Prebid: failed to parse bid from seat '{delivery_bidder}' for imp '{impression}'" - ); - } - } - } - } - } - } - } - let mut parsed = if bids.is_empty() { - AuctionResponse::no_bid(provider_id, response_time_ms) - } else { - AuctionResponse::success(provider_id, bids, response_time_ms) - }; - diagnostics.attach_to(&mut parsed); - parsed -} - -fn enrich_planned_prebid_metadata( - profile: &PrebidProfilePlan, - response_json: &Json, - parsed: &mut AuctionResponse, -) { - let ext = response_json.get("ext"); - for key in ["responsetimemillis", "errors", "warnings"] { - if let Some(value) = ext.and_then(|ext| ext.get(key)) { - parsed.metadata.insert(key.to_string(), value.clone()); - } - } - if profile.debug { - if let Some(value) = ext.and_then(|ext| ext.get("debug")) { - parsed.metadata.insert("debug".to_string(), value.clone()); - } - if let Some(value) = ext - .and_then(|ext| ext.get("prebid")) - .and_then(|prebid| prebid.get("bidstatus")) - { - parsed - .metadata - .insert("bidstatus".to_string(), value.clone()); - } - } -} - -fn parse_planned_prebid_bid( - bid: &Json, - delivery_bidder: &str, - returned_seat: Option<&str>, - dimensions_by_slot: &BidDimensionIndex, -) -> Result { - let slot_id = bid - .get("impid") - .and_then(Json::as_str) - .filter(|slot_id| !slot_id.is_empty()) - .ok_or(BidRejectionReason::InvalidBid)? - .to_string(); - let width = parse_optional_bid_dimension(bid, "w")?; - let height = parse_optional_bid_dimension(bid, "h")?; - let (width, height) = resolve_bid_dimensions(dimensions_by_slot, &slot_id, width, height)?; - let price = bid - .get("price") - .and_then(Json::as_f64) - .filter(|price| price.is_finite() && *price >= 0.0) - .ok_or(BidRejectionReason::InvalidBid)?; - let creative = bid.get("adm").and_then(Json::as_str).map(String::from); - let cache_entry = bid - .get("ext") - .and_then(|ext| ext.get("prebid")) - .and_then(|prebid| prebid.get("cache")) - .and_then(|cache| cache.get("bids")); - let cache_id = cache_entry - .and_then(|cache| cache.get("cacheId")) - .and_then(Json::as_str) - .map(String::from); - let (cache_host, cache_path) = cache_entry - .and_then(|cache| cache.get("url")) - .and_then(Json::as_str) - .and_then(|value| { - ParsedUrl::parse(value) - .map_err(|error| log::debug!("PBS cache URL parse failed: {error}")) - .ok() - }) - .map(|url| { - let host = url.host_str().map(String::from); - let path = url.path().to_string(); - let path = (!path.is_empty() && path != "/").then_some(path); - (host, path) - }) - .unwrap_or((None, None)); - if cache_id.is_some() && cache_host.is_none() { - log::warn!( - "PBS bid has cache UUID but cache URL could not be parsed — creative will fail to render for slot '{slot_id}'" - ); - } - - Ok(AuctionBid { - slot_id, - price: Some(price), - currency: DEFAULT_CURRENCY.to_string(), - creative, - adomain: bid.get("adomain").and_then(Json::as_array).map(|domains| { - domains - .iter() - .filter_map(Json::as_str) - .map(String::from) - .collect() - }), - bidder: delivery_bidder.to_string(), - returned_seat: returned_seat.map(str::to_string), - width, - height, - nurl: bid.get("nurl").and_then(Json::as_str).map(String::from), - burl: bid.get("burl").and_then(Json::as_str).map(String::from), - bid_id: bid - .get("id") - .and_then(Json::as_str) - .filter(|value| !value.is_empty()) - .map(String::from), - ad_id: bid.get("adid").and_then(Json::as_str).map(String::from), - creative_id: bid.get("crid").and_then(Json::as_str).map(String::from), - renderer: None, - cache_id, - cache_host, - cache_path, - metadata: HashMap::new(), - }) -} - -// ============================================================================ -// Prebid Auction Provider -// ============================================================================ - -/// Legacy Prebid Server auction provider retained only for parity tests. -#[cfg(test)] -pub struct PrebidAuctionProvider { - config: LegacyPrebidServerConfig, - bid_param_override_engine: Arc, -} - -#[cfg(test)] -#[derive(Default)] -struct PrebidImpressionDisposition { - aps_only: usize, - invalid: usize, -} - -#[cfg(test)] -struct PrebidRequestBuild { - request: OpenRtbRequest, - disposition: PrebidImpressionDisposition, -} - -#[cfg(test)] -impl PrebidAuctionProvider { - #[cfg(test)] - fn new(config: LegacyPrebidServerConfig) -> Self { - Self::try_new(config).expect("should compile prebid bid param overrides") - } - - /// Create a new Prebid auction provider with validated override rules. - /// - /// # Errors - /// - /// Returns an error when the configured bidder-param override rules are invalid. - pub fn try_new(config: LegacyPrebidServerConfig) -> Result> { - Ok(Self { - bid_param_override_engine: Arc::new(BidParamOverrideEngine::try_from_config(&config)?), - config, - }) - } - - /// Returns the full Prebid Server `OpenRTB2` auction endpoint URL. - /// - /// Backward-compatible normalization: `server_url` may be configured as - /// either the PBS origin (path is appended here) or the full endpoint - /// already ending in `/openrtb2/auction` (used as-is, ignoring a trailing - /// slash) — both shapes produce the same request URL. - fn auction_endpoint_url(&self) -> String { - let base = self.config.server_url.trim_end_matches('/'); - if base.ends_with("/openrtb2/auction") { - base.to_string() - } else { - format!("{base}/openrtb2/auction") - } - } - - /// Build an enriched `OpenRTB` request and retain impression drop provenance. - fn build_openrtb( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - signer: Option<(&RequestSigner, String, &SigningParams)>, - request_info: RequestInfo, - ) -> PrebidRequestBuild { - let mut disposition = PrebidImpressionDisposition::default(); - let imps = request - .slots - .iter() - .filter_map(|slot| { - let slot_context = format!("slot '{}'", slot.id); - let formats: Vec<_> = slot - .formats - .iter() - .filter(|f| f.media_type == MediaType::Banner) - .filter_map(|f| { - let width = to_openrtb_i32(f.width, "format.w", &slot_context); - let height = to_openrtb_i32(f.height, "format.h", &slot_context); - - match (width, height) { - (Some(width), Some(height)) => Some(Format { - w: Some(width), - h: Some(height), - ..Default::default() - }), - _ => None, - } - }) - .collect(); - - if formats.is_empty() { - disposition.invalid += 1; - log::warn!( - "prebid: dropping imp '{}' — no valid banner formats after filtering", - slot.id - ); - return None; - } - - // Extract zone from trustedServer params (sent by the JS - // adapter from `mediaTypes.banner.name`, e.g. "header", "fixed_bottom"). - let zone: Option<&str> = slot - .bidders - .get(TRUSTED_SERVER_BIDDER) - .and_then(|p| p.get(ZONE_KEY)) - .and_then(Json::as_str); - - // Build the bidder map for PBS from two sources: - // 1. the `trustedServer` orchestrator entry, expanded into the - // configured PBS bidders (`expand_trusted_server_bidders`); - // 2. direct entries naming a configured PBS bidder. - // The JS adapter sends "trustedServer" as the bidder (our - // orchestrator adapter name); provider-specific keys like "aps" - // run their own auction and are skipped here. - // - // The two sources are collected separately so the merge below is - // deterministic regardless of `slot.bidders` HashMap iteration - // order: direct entries win, but an unusable direct value (`{}` or - // a non-object) must not clobber real params from the expansion. - let mut expanded: HashMap = HashMap::new(); - let mut direct: Vec<(String, Json)> = Vec::new(); - let mut excluded_aps = false; - for (name, params) in &slot.bidders { - if name.eq_ignore_ascii_case("aps") { - // APS is a separate OpenRTB provider. Never send native - // APS demand through PBS for the same cohort. - excluded_aps = true; - } else if name == TRUSTED_SERVER_BIDDER { - for (bidder, params) in - expand_trusted_server_bidders(&self.config.bidders, params) - { - if bidder.eq_ignore_ascii_case("aps") { - excluded_aps = true; - } else { - expanded.insert(bidder, params); - } - } - } else if self.config.bidders.iter().any(|bidder| bidder == name) { - direct.push((name.clone(), params.clone())); - } else { - // Any unrecognized key is likely a misconfiguration (a - // slot bidder absent from `config.bidders`) that silently - // yields an empty bidder map and a stored-request no-bid — - // log it so the drop is diagnosable. - log::debug!( - "prebid: dropping slot '{}' bidder '{}' — not in config.bidders and not a known provider key", - slot.id, - name - ); - } - } - - let mut bidder = expanded; - // Bidders whose params came from the slot itself rather than from - // `trustedServer` expansion. Only these can represent a publisher - // misconfiguration if they are dropped below; a dropped fabricated - // entry is the designed stored-request path. - let mut slot_supplied: HashSet = HashSet::new(); - for (name, params) in direct { - // Direct entries win over the expansion — except when an - // unusable direct value (`{}` or a non-object) would overwrite - // real params the expansion already supplied. PBS cannot tell a - // fabricated empty from an explicit one, so real params must not - // be discarded in favour of either. - let clobbers_real = is_unusable_bidder_params(¶ms) - && bidder - .get(&name) - .is_some_and(|existing| !is_unusable_bidder_params(existing)); - if !clobbers_real { - slot_supplied.insert(name.clone()); - bidder.insert(name, params); - } - } - - // Apply canonical and compatibility-derived rules in normalized - // order. An override rule can populate a bidder that arrived with - // empty params, promoting a fabricated empty into a valid bidder. - for (name, params) in &mut bidder { - self.bid_param_override_engine - .apply(BidParamOverrideFacts { bidder: name, zone }, params); - } - - // Drop any bidder whose params are still unusable after overrides — - // an empty `{}` or a non-object like `null`. Both are byte-identical - // to PBS regardless of whether they were fabricated or supplied - // explicitly, and shipping either gets the imp rejected; a single - // non-2xx then collapses into an auction-wide bid wipeout (see - // `parse_response`), so one misconfigured slot must never reach the - // wire. - // - // The two drop sources get different levels, one line per slot. A - // slot that supplied unusable params inline is a publisher mistake - // an operator can act on, so it warns — the PBS 400 body is - // otherwise discarded unless trace logging is enabled. Fabricated - // empties that no override rule filled are the designed - // creative-opportunity path (see `CreativeOpportunitySlot::to_ad_slot`), - // so they log at debug and do not bury the warn. - let mut dropped_slot_supplied: Vec = Vec::new(); - let mut dropped_fabricated: Vec = Vec::new(); - bidder.retain(|name, params| { - if is_unusable_bidder_params(params) { - if slot_supplied.contains(name) { - dropped_slot_supplied.push(name.clone()); - } else { - dropped_fabricated.push(name.clone()); - } - return false; - } - true - }); - if !dropped_slot_supplied.is_empty() { - // Sorted so the line is stable across `HashMap` iteration order. - dropped_slot_supplied.sort(); - log::warn!( - "prebid: dropping slot '{}' bidders [{}] — slot supplied empty or non-object params; slot falls back to its stored request", - slot.id, - dropped_slot_supplied.join(", ") - ); - } - if !dropped_fabricated.is_empty() { - dropped_fabricated.sort(); - log::debug!( - "prebid: dropping slot '{}' bidders [{}] — expansion fabricated empty params and no override rule filled them; slot falls back to its stored request", - slot.id, - dropped_fabricated.join(", ") - ); - } - - if excluded_aps && bidder.is_empty() { - disposition.aps_only += 1; - log::warn!( - "prebid: dropping imp '{}' because it contains only APS demand; refusing PBS stored-request fallback", - slot.id - ); - return None; - } - - // When no eligible PBS bidder params remain, tell PBS to resolve - // bidder config from the stored request keyed by this slot ID. This - // covers creative-opportunity slots whose PBS params live in stored - // requests, and slots whose configured bidders all resolved to - // unusable params and were dropped above. - // - // Note the fallback set is wider than "slots with no inline params": - // a slot carrying real inline params for a bidder absent from - // `config.bidders` also lands here — that bidder is never expanded, - // the configured bidders fabricate empties, the drop clears them, and - // the real params are lost. That param loss is pre-existing (see the - // debug log in the build loop); it is called out here because it - // means an operator with `config.bidders` set but no stored request - // provisioned for such a slot gets a no-bid. - let storedrequest = if bidder.is_empty() { - Some(ImpStoredRequest { - id: slot.id.clone(), - }) - } else { - None - }; - - Some(Imp { - id: Some(slot.id.clone()), - banner: Some(Banner { - format: formats, - ..Default::default() - }), - bidfloor: slot.floor_price, - // NOTE: Currency defaults to DEFAULT_CURRENCY. If - // multi-currency support is needed, this should come from - // config or the AdSlot itself. - bidfloorcur: slot.floor_price.map(|_| DEFAULT_CURRENCY.to_string()), - secure: Some(true), // require HTTPS creatives - tagid: Some(slot.id.clone()), - ext: ImpExt { - prebid: PrebidImpExt { - bidder, - storedrequest, - }, - } - .to_ext(), - ..Default::default() - }) - }) - .collect(); - - // Build page URL with debug query params if configured - let page_url = request.publisher.page_url.as_ref().map(|url| { - if let Some(ref params) = self.config.debug_query_params { - append_query_params(url, params) - } else { - url.clone() - } - }); - - // Build user object — populate consent at both OpenRTB 2.6 top-level - // and Prebid ext-based locations (dual placement). - // In cookies_only mode, cookie-sourced consent travels through the - // forwarded Cookie header. Policy-sourced consent has no inbound cookie - // to forward, so carry it in the OpenRTB body instead. - let consent_ctx = request.user.consent.as_ref().filter(|ctx| { - self.config.consent_forwarding.includes_body_consent() - || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) - }); - let raw_tc = consent_ctx.and_then(|c| c.raw_tc_string.clone()); - let user = Some(User { - id: request.user.id.clone(), - // OpenRTB 2.6 top-level consent field - consent: raw_tc.clone(), - ext: UserExt { - // Prebid ext-based consent field - consent: raw_tc, - consented_providers_settings: consent_ctx - .and_then(|c| c.raw_ac_string.as_ref()) - .map(|ac| ConsentedProvidersSettings { - consented_providers: Some(ac.clone()), - }), - // EIDs resolved from the KV identity graph and consent-gated - // in `handle_auction` via `gate_eids_by_consent`. - eids: request.user.eids.clone(), - } - .to_ext(), - ..Default::default() - }); - - // Extract DNT header and Accept-Language from the original request - let dnt = context - .request - .headers() - .get("DNT") - .and_then(|value| value.to_str().ok()) - .and_then(|value| { - if value.trim() == "1" { - Some(true) - } else { - None - } - }); - - let language = context - .request - .headers() - .get(header::ACCEPT_LANGUAGE) - .and_then(|value| value.to_str().ok()) - .and_then(|v| { - // Extract the primary ISO-639 language tag (e.g., "en" from - // "en-US,en;q=0.9"). Strip the region subtag so bidders get a - // normalised two-letter code that maximises match quality. - v.split(',') - .next() - .and_then(|tag| tag.split(';').next()) - .map(|tag| { - tag.split('-') - .next() - .expect("should have at least one split segment") - .trim() - .to_string() - }) - .filter(|s| !s.is_empty()) - }); - - // Build device object with user-agent, client IP, geo, DNT, and language. - // Forwarding the real client IP is critical: without it PBS infers the - // IP from the incoming connection (a data-center / edge IP), causing - // bidders like PubMatic to filter the traffic as non-human. - // - // When `request.device` is `None` we still construct a minimal `Device` - // if DNT or language were extracted from HTTP headers, so those signals - // are never silently discarded. - let device = request - .device - .as_ref() - .map(|d| Device { - ua: d.user_agent.clone(), - ip: d.ip.clone(), - geo: d.geo.as_ref().map(|geo| Geo { - country: Some(geo.country.clone()), - city: Some(geo.city.clone()), - region: geo.region.clone(), - lat: Some(geo.latitude), - lon: Some(geo.longitude), - // DMA/metro code: convert i64 to string for OpenRTB. - // Fastly returns 0 for "no metro code"; negative values - // are not realistic for DMA codes so the > 0 guard is - // sufficient. - metro: if geo.metro_code > 0 { - Some(geo.metro_code.to_string()) - } else { - None - }, - r#type: Some(2), - ..Default::default() - }), - dnt, - // Clone needed: `language` is also used in the `or_else` - // fallback below when `request.device` is `None`. - language: language.clone(), - ..Default::default() - }) - .or_else(|| { - if dnt.is_some() || language.is_some() { - Some(Device { - dnt, - language, - ..Default::default() - }) - } else { - None - } - }); - - // Build regs object from ConsentContext, populating both OpenRTB 2.6 - // top-level fields and `regs.ext` for Prebid Server compatibility. - let regs = Self::build_regs(consent_ctx); - - // Build ext object - let (version, signature, kid, ts) = signer - .map(|(s, sig, params)| { - ( - Some(SIGNING_VERSION.to_string()), - Some(sig), - Some(s.kid.clone()), - Some(params.timestamp), - ) - }) - .unwrap_or((None, None, None, None)); - - let debug_enabled = self.config.debug; - - let ext = RequestExt { - prebid: Some(PrebidExt { - debug: debug_enabled.then_some(true), - returnallbidstatus: debug_enabled.then_some(true), - }), - trusted_server: Some(TrustedServerExt { - version, - signature, - kid, - request_host: Some(request_info.host), - request_scheme: Some(request_info.scheme), - ts, - }), - } - .to_ext(); - - // Extract Referer header for site.ref - let referer = context - .request - .headers() - .get(header::REFERER) - .and_then(|value| value.to_str().ok()) - .map(std::string::ToString::to_string); - - // Advertise the effective auction budget, not the raw provider config: - // the orchestrator caps `context.timeout_ms` to the remaining auction - // budget, and the edge backend stops waiting after that long. Telling - // PBS it has more time than the edge will wait turns partial bids into - // edge timeouts. - let tmax = to_openrtb_i32(context.timeout_ms, "tmax", "request"); - - let request = OpenRtbRequest { - id: Some(request.id.clone()), - imp: imps, - site: Some(Site { - domain: Some(request.publisher.domain.clone()), - page: page_url, - r#ref: referer, - publisher: Some(Publisher { - domain: Some(request.publisher.domain.clone()), - ..Default::default() - }), - ..Default::default() - }), - user, - device, - regs, - test: self.config.test_mode.then_some(true), - tmax, - cur: vec![DEFAULT_CURRENCY.to_string()], - ext, - ..Default::default() - }; - - PrebidRequestBuild { - request, - disposition, - } - } - - /// Convert an auction request to `OpenRTB` format with all enrichments. - #[cfg(test)] - fn to_openrtb( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - signer: Option<(&RequestSigner, String, &SigningParams)>, - request_info: RequestInfo, - ) -> OpenRtbRequest { - self.build_openrtb(request, context, signer, request_info) - .request - } - - /// Builds the `regs` object from a [`ConsentContext`]. - /// - /// Populates consent fields at **both** `OpenRTB` 2.6 top-level locations - /// and the `regs.ext.*` locations that Prebid Server reads today. - /// - /// Returns [`None`] if no consent-relevant data is present (avoids sending - /// an empty `regs` object to Prebid Server). - fn build_regs(consent_ctx: Option<&crate::consent::ConsentContext>) -> Option { - let ctx = consent_ctx?; - - let has_data = ctx.gdpr_applies - || ctx.raw_us_privacy.is_some() - || ctx.raw_gpp_string.is_some() - || ctx.gpp_section_ids.is_some() - || ctx.gpc; - - if !has_data { - return None; - } - - // Use jurisdiction to inform the GDPR flag: if we know the user is in - // a GDPR jurisdiction, signal that even without a TCF string (e.g., - // GPC-only requests from EU users). When jurisdiction is unknown and - // no TCF signal exists, omit the field rather than falsely signaling - // "GDPR does not apply." - let in_gdpr_jurisdiction = matches!( - ctx.jurisdiction, - crate::consent::jurisdiction::Jurisdiction::Gdpr - ); - let gdpr = if ctx.gdpr_applies || in_gdpr_jurisdiction { - Some(true) - } else if matches!( - ctx.jurisdiction, - crate::consent::jurisdiction::Jurisdiction::Unknown - ) { - None - } else { - Some(false) - }; - - // Dual-placement: OpenRTB 2.6 top-level fields AND `regs.ext` for - // backward compatibility with older exchanges. Extract top-level - // values first, then clone once into the ext to avoid extra copies. - let us_privacy = ctx.raw_us_privacy.clone(); - let gpp = ctx.raw_gpp_string.clone(); - let gpp_sid = ctx.gpp_section_ids.clone(); - - // RegsExt uses u8 for GDPR (Prebid convention) while the top-level - // Regs uses bool (OpenRTB proto). Map accordingly. - let gdpr_u8 = gdpr.map(u8::from); - let gpp_sid_u16 = gpp_sid.clone(); - let ext = RegsExt { - gdpr: gdpr_u8, - us_privacy: us_privacy.clone(), - gpp: gpp.clone(), - gpp_sid: gpp_sid_u16, - }; - - Some(Regs { - coppa: None, - gdpr, - us_privacy, - gpp, - gpp_sid: gpp_sid - .map(|ids| ids.into_iter().map(i32::from).collect()) - .unwrap_or_default(), - ext: ext.to_ext(), - }) - } - - /// Parse `OpenRTB` response into auction response. - fn parse_openrtb_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { - let mut bids = Vec::new(); - - if let Some(seatbids) = json.get("seatbid").and_then(|v| v.as_array()) { - for seatbid in seatbids { - let seat = seatbid - .get("seat") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - - if let Some(bid_array) = seatbid.get("bid").and_then(|v| v.as_array()) { - for bid_obj in bid_array { - match self.parse_bid(bid_obj, seat) { - Ok(bid) => bids.push(bid), - Err(()) => { - let impid = bid_obj - .get("impid") - .and_then(|v| v.as_str()) - .unwrap_or(""); - log::warn!( - "Prebid: failed to parse bid from seat '{seat}' for imp '{impid}'" - ); - } - } - } - } - } - } - - if bids.is_empty() { - AuctionResponse::no_bid(PREBID_INTEGRATION_ID, response_time_ms) - } else { - AuctionResponse::success(PREBID_INTEGRATION_ID, bids, response_time_ms) - } - } - - /// Enrich an [`AuctionResponse`] with metadata extracted from the Prebid - /// Server response `ext`. - /// - /// Always-on fields: `responsetimemillis`, `errors`, `warnings`. - /// Debug-only fields (gated on `config.debug`): `debug`, `bidstatus`. - fn enrich_response_metadata( - &self, - response_json: &Json, - auction_response: &mut AuctionResponse, - ) { - let ext = response_json.get("ext"); - - // Always attach per-bidder timing and diagnostics. - if let Some(rtm) = ext.and_then(|e| e.get("responsetimemillis")) { - auction_response - .metadata - .insert("responsetimemillis".to_string(), rtm.clone()); - } - if let Some(errors) = ext.and_then(|e| e.get("errors")) { - auction_response - .metadata - .insert("errors".to_string(), errors.clone()); - } - if let Some(warnings) = ext.and_then(|e| e.get("warnings")) { - auction_response - .metadata - .insert("warnings".to_string(), warnings.clone()); - } - - // When debug is enabled, surface httpcalls, resolvedrequest, and - // per-bid status from the Prebid Server response. - if self.config.debug { - if let Some(debug) = ext.and_then(|e| e.get("debug")) { - auction_response - .metadata - .insert("debug".to_string(), debug.clone()); - } - if let Some(bidstatus) = ext - .and_then(|e| e.get("prebid")) - .and_then(|p| p.get("bidstatus")) - { - auction_response - .metadata - .insert("bidstatus".to_string(), bidstatus.clone()); - } - } - } - - async fn parse_response_inner( - &self, - response: PlatformResponse, - response_time_ms: u64, - auction_id: Option<&str>, - ) -> Result> { - let response = response.response; - let status = response.status(); - let content_type = response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - - // Parse response — collect_response_bounded caps memory from misbehaving providers. - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "prebid", - ) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to read Prebid response body".to_string(), - }); - - if !status.is_success() { - let auction_id = auction_id.unwrap_or(""); - log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); - - let body_bytes = match body_bytes { - Ok(body_bytes) => Some(body_bytes), - Err(error) => { - log::warn!( - "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" - ); - None - } - }; - - if self.config.debug - && let Some(body_bytes) = body_bytes.as_deref() - { - match prebid_body_preview(body_bytes) { - Some(preview) => { - let truncation = if preview.truncated { - " (truncated)" - } else { - "" - }; - log::warn!( - "Prebid auction {auction_id:?} error response body preview{truncation}: {}", - preview.text - ); - } - None => log::warn!( - "Prebid auction {auction_id:?} returned an empty error response body" - ), - } - } - - let status_code = status.as_u16(); - let mut auction_response = - AuctionResponse::error(PREBID_INTEGRATION_ID, response_time_ms) - .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) - .with_metadata("http_status", serde_json::json!(status_code)) - .with_metadata( - "message", - serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), - ); - - let upstream_message = if self.config.debug { - body_bytes.as_deref().and_then(|body_bytes| { - extract_prebid_error_message(body_bytes, content_type.as_deref()) - }) - } else { - None - }; - if let Some(message) = upstream_message { - auction_response.metadata.insert( - "upstream_message".to_string(), - serde_json::json!(message.text), - ); - auction_response.metadata.insert( - "upstream_message_truncated".to_string(), - serde_json::json!(message.truncated), - ); - } - - return Ok(auction_response); - } - - let body_bytes = body_bytes?; - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { - message: "Failed to parse Prebid response".to_string(), - })?; - - // Log the full response body when debug is enabled to surface - // ext.debug.httpcalls, resolvedrequest, bidstatus, errors, etc. - if self.config.debug && log::log_enabled!(log::Level::Trace) { - match serde_json::to_string_pretty(&response_json) { - Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), - Err(e) => { - log::warn!("Prebid: failed to serialize response for logging: {e}"); - } - } - } - - let mut auction_response = self.parse_openrtb_response(&response_json, response_time_ms); - self.enrich_response_metadata(&response_json, &mut auction_response); - - log::info!( - "Prebid returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); - - Ok(auction_response) - } - - fn should_suppress_bid_notifications(&self, bidder: &str) -> bool { - self.config.suppress_nurl - || self - .config - .suppress_nurl_bidders - .iter() - .any(|suppressed_bidder| suppressed_bidder == bidder) - } - - /// Parse a single bid from `OpenRTB` response. - fn parse_bid(&self, bid_obj: &Json, seat: &str) -> Result { - let slot_id = bid_obj - .get("impid") - .and_then(|v| v.as_str()) - .ok_or(())? - .to_string(); - - let price = bid_obj - .get("price") - .and_then(serde_json::Value::as_f64) - .ok_or(())?; - - let creative = bid_obj - .get("adm") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string); - - let width = bid_obj - .get("w") - .and_then(serde_json::Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(0); - let height = bid_obj - .get("h") - .and_then(serde_json::Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(0); - - let suppress_bid_notifications = self.should_suppress_bid_notifications(seat); - let nurl = if suppress_bid_notifications { - None - } else { - bid_obj - .get("nurl") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string) - }; - - let burl = if suppress_bid_notifications { - None - } else { - bid_obj - .get("burl") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string) - }; - - // `adid` is the creative/ad identifier. The OpenRTB `id` is the bid ID, - // not an ad ID, so it is not used as a fallback: surfacing it as `ad_id` - // (which is exposed raw in the debug bid) would mislead any consumer that - // treats `ad_id` as a creative identifier. Absent `adid`, `ad_id` is None. - // The bid ID is carried separately in `bid_id` instead. An empty `id` is - // treated as absent — a blank hb_adid is falsey on the page, so it would - // be no better than omitting the key. - let bid_id = bid_obj - .get("id") - .and_then(|v| v.as_str()) - .filter(|id| !id.is_empty()) - .map(String::from); - let ad_id = bid_obj - .get("adid") - .and_then(|v| v.as_str()) - .map(String::from); - let creative_id = bid_obj - .get("crid") - .and_then(|v| v.as_str()) - .map(String::from); - - let adomain = bid_obj - .get("adomain") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(std::string::ToString::to_string)) - .collect() - }); - - // Extract PBS Cache coordinates from ext.prebid.cache.bids - let cache_entry = bid_obj - .get("ext") - .and_then(|e| e.get("prebid")) - .and_then(|p| p.get("cache")) - .and_then(|c| c.get("bids")); - - let cache_id = cache_entry - .and_then(|c| c.get("cacheId")) - .and_then(|v| v.as_str()) - .map(String::from); - - let (cache_host, cache_path) = cache_entry - .and_then(|c| c.get("url")) - .and_then(|v| v.as_str()) - .and_then(|url_str| { - ParsedUrl::parse(url_str) - .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) - .ok() - }) - .map(|u| { - let host = u.host_str().map(String::from); - // path() returns "/" for root — only use if non-trivial - let path = u.path().to_string(); - let path = if path.is_empty() || path == "/" { - None - } else { - Some(path) - }; - (host, path) - }) - .unwrap_or((None, None)); - - // Guard: if we extracted a cache UUID but couldn't extract the host, - // the bid will have hb_adid set but no endpoint to fetch from — creative will fail. - if cache_id.is_some() && cache_host.is_none() { - log::warn!( - "PBS bid has cache UUID but cache URL could not be parsed — \ - creative will fail to render for slot '{}'", - slot_id - ); - } - - Ok(AuctionBid { - slot_id, - price: Some(price), // Prebid provides decoded prices - currency: DEFAULT_CURRENCY.to_string(), - creative, - adomain, - bidder: seat.to_string(), - returned_seat: None, - width, - height, - nurl, - burl, - bid_id, - ad_id, - creative_id, - renderer: None, - cache_id, - cache_host, - cache_path, - metadata: std::collections::HashMap::new(), - }) - } -} - -#[cfg(test)] -#[async_trait(?Send)] -impl AuctionProvider for PrebidAuctionProvider { - fn provider_name(&self) -> &str { - PREBID_INTEGRATION_ID - } - - async fn request_bids( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request_start = web_time::Instant::now(); - log::info!("Prebid: requesting bids for {} slots", request.slots.len()); - - // `ext.trusted_server.request_host` must be the publisher's own domain - // (matching `site.domain`/`publisher.domain` below), not the raw edge - // `Host` header of the incoming `/auction` request. On the SSAT proxy - // path the browser calls `/auction` against the trusted-server edge - // domain (e.g. `ts.example.com`), which is not what PBS's - // `trusted_server` verification module expects — see the "Host header - // value expected by the receiving service" doc on `SigningParams`. - // `scheme` is canonical `"https"` rather than detected from the - // incoming request: the publisher origin is always https in - // production, and PBS verification is expected to match on that, - // not on the edge's local scheme (e.g. http under `fastly compute - // serve`). - let request_info = RequestInfo { - host: request.publisher.domain.clone(), - scheme: "https".to_owned(), - }; - - // Create signer and compute signature if request signing is enabled - let signer_with_signature = - if let Some(request_signing_config) = &context.settings.request_signing { - if request_signing_config.enabled { - let signer = RequestSigner::from_services(context.services)?; - let params = SigningParams::new( - request.id.clone(), - request_info.host.clone(), - request_info.scheme.clone(), - ); - let signature = signer.sign_request(¶ms)?; - Some((signer, signature, params)) - } else { - None - } - } else { - None - }; - - let PrebidRequestBuild { - request: openrtb, - disposition, - } = self.build_openrtb( - request, - context, - signer_with_signature - .as_ref() - .map(|(signer, signature, params)| (signer, signature.clone(), params)), - request_info, - ); - - if openrtb.imp.is_empty() { - let all_slots_are_aps_only = !request.slots.is_empty() - && disposition.invalid == 0 - && disposition.aps_only == request.slots.len(); - if all_slots_are_aps_only { - log::info!( - "Prebid: returning no-bid because all {} valid impressions are APS-only", - disposition.aps_only - ); - return Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( - PREBID_INTEGRATION_ID, - request_start.elapsed().as_millis() as u64, - ))); - } - - log::info!("Prebid: skipping request — no valid impressions after filtering"); - return Err(Report::new(TrustedServerError::Prebid { - message: "No valid impressions after filtering".to_string(), - })); - } - - // Log the outgoing OpenRTB request for debugging. - if log::log_enabled!(log::Level::Debug) { - match serde_json::to_string_pretty(&openrtb) { - Ok(json) => log::debug!( - "Prebid OpenRTB request to {}:\n{}", - self.auction_endpoint_url(), - json - ), - Err(e) => { - log::warn!("Prebid: failed to serialize OpenRTB request for logging: {e}") - } - } - } - - // Create HTTP request - let mut pbs_req = http::Request::builder() - .method(http::Method::POST) - .uri(self.auction_endpoint_url()) - .body(EdgeBody::empty()) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid request".to_string(), - })?; - copy_request_headers( - context.request, - &mut pbs_req, - self.config.consent_forwarding, - context.services.client_info().client_ip, - ); - - let pbs_body = serde_json::to_vec(&openrtb).change_context(TrustedServerError::Prebid { - message: "Failed to serialize Prebid request body".to_string(), - })?; - pbs_req.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - ); - *pbs_req.body_mut() = EdgeBody::from(pbs_body); - - // Uses context.timeout_ms (auction-scoped) rather than the 15 s fixed - // timeout in ensure_integration_backend, which is for proxy endpoints. - // Send request asynchronously with auction-scoped timeout - let backend_name = ensure_integration_backend_with_timeout( - context.services, - &self.config.server_url, - "prebid", - Duration::from_millis(u64::from(context.timeout_ms)), - ) - .change_context(TrustedServerError::Auction { - message: format!( - "Failed to resolve backend for Prebid Server endpoint: {}", - self.config.server_url - ), - })?; - let pending = context - .services - .http_client() - .send_async(PlatformHttpRequest::new(pbs_req, backend_name)) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to send async request to Prebid Server".to_string(), - })?; - - Ok(ProviderRequestOutcome::pending(pending)) - } - - async fn parse_response( - &self, - response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - self.parse_response_inner(response, response_time_ms, None) - .await - } - - async fn parse_response_with_context( - &self, - response: PlatformResponse, - response_time_ms: u64, - request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - self.parse_response_inner(response, response_time_ms, Some(request.id.as_str())) - .await - } - - fn supports_media_type(&self, media_type: &MediaType) -> bool { - matches!(media_type, MediaType::Banner) - } - - fn timeout_ms(&self) -> u32 { - self.config.timeout_ms - } - - fn is_enabled(&self) -> bool { - self.config.enabled - } - - fn backend_name(&self, services: &RuntimeServices, timeout_ms: u32) -> Option { - predict_integration_backend_name( - services, - &self.config.server_url, - PREBID_INTEGRATION_ID, - Duration::from_millis(u64::from(timeout_ms)), - ) - .inspect_err(|e| { - log::error!( - "Failed to predict backend name for Prebid server URL '{}': {e:?}", - self.config.server_url - ); - }) - .ok() - } -} - -// ============================================================================ -// Provider Auto-Registration -// ============================================================================ - -/// Auto-register Prebid provider based on settings configuration. -/// -/// This function checks the settings for Prebid configuration and returns -/// the provider if enabled. -/// -/// # Errors -/// -/// Returns an error when the Prebid provider is enabled with invalid -/// configuration. -#[cfg(test)] -pub fn register_auction_provider( - settings: &Settings, -) -> Result>, Report> { - let Some(integration) = build(settings)? else { - log::info!("Prebid auction provider not registered: integration not found or disabled"); - return Ok(Vec::new()); - }; - - if let Some(config) = integration.legacy_config.as_ref() { - log::info!( - "Registering Prebid auction provider (server_url={})", - config.server_url - ); - } - if integration.config.debug { - log::warn!( - "Prebid debug mode is ON — debug data (httpcalls, resolvedrequest, \ - bidstatus) will be included in /auction responses" - ); - } - - Ok(vec![Arc::new(integration.auction_provider())]) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::*; - use crate::auction::formats::convert_to_openrtb_response; - use crate::auction::orchestrator::OrchestrationResult; - use crate::auction::plan::{ - AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, - ProviderId, RoutingMode, - }; - use crate::auction::test_support::{ - canonical_parity_auction_request, - create_test_auction_context as shared_test_auction_context, - }; - use crate::auction::types::{ - AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, - }; - - use crate::consent::{ConsentContext, ConsentSource}; - use crate::geo::GeoInfo; - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; - use crate::integrations::{ - AttributeRewriteAction, IntegrationDocumentState, IntegrationRegistry, - }; - use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopGeo, NoopHttpClient, - NoopSecretStore, StubHttpClient, build_services_with_config_secret_and_http_client, - build_services_with_http_client, build_services_with_http_client_and_client_ip, - }; - use crate::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, RuntimeServices, - }; - use crate::settings::Settings; - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; - use crate::test_support::tests::create_test_settings; - use base64::engine::general_purpose::STANDARD as TEST_BASE64_STANDARD; - use bytes::Bytes; - use http::Method; - use serde_json::json; - use std::collections::{BTreeMap, HashMap}; - use std::io::Cursor; - use std::str::FromStr as _; - - #[test] - fn external_bundle_sha256_validation_matches_hex_pattern() { - use validator::Validate as _; - - let config = |sha: &str| -> PrebidIntegrationConfig { - serde_json::from_value(serde_json::json!({ - "external_bundle_sha256": sha, - })) - .expect("should deserialize prebid config") - }; - - // Exactly 64 hex digits (either case) passes. - config(&"a".repeat(64)) - .validate() - .expect("64-char lowercase hex sha256 should pass"); - config("ABCDEF0123456789abcdef0123456789ABCDEF0123456789abcdef0123456789") - .validate() - .expect("mixed-case 64-char hex sha256 should pass"); - - // Wrong length or non-hex characters are rejected. - for bad in [ - "a".repeat(63), - "a".repeat(65), - "g".repeat(64), - String::new(), - ] { - config(&bad) - .validate() - .expect_err(&format!("invalid sha256 {bad:?} should be rejected")); - } - } - - fn make_settings() -> Settings { - create_test_settings() - } - - #[test] - fn injected_prebid_config_escapes_every_less_than_sign() { - let config_json = serialize_injected_prebid_config(&json!({ - "accountId": "x + + + + + + Example Publisher + + + + + + + diff --git a/crates/tag/google-tag/src/fixtures/head-with-tag-manager.html b/crates/tag/google-tag/src/fixtures/head-with-tag-manager.html new file mode 100644 index 0000000000..d15e7406ce --- /dev/null +++ b/crates/tag/google-tag/src/fixtures/head-with-tag-manager.html @@ -0,0 +1,40 @@ + + + + + + + + Example Publisher + + + + + + + + diff --git a/crates/tag/google-tag/src/fixtures/page-change.settings.toml b/crates/tag/google-tag/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..9dab3a1154 --- /dev/null +++ b/crates/tag/google-tag/src/fixtures/page-change.settings.toml @@ -0,0 +1,9 @@ +[tag] +modules = ["google-tag"] + +[tag.google-tag] +analytics_tag_id = "G-XXXXXXXXXX" + +[[serve]] +media_type = "text/html" +middleware = ["tag.google-tag"] diff --git a/crates/tag/google-tag/src/lib.rs b/crates/tag/google-tag/src/lib.rs new file mode 100644 index 0000000000..da59372b2d --- /dev/null +++ b/crates/tag/google-tag/src/lib.rs @@ -0,0 +1,1479 @@ +//! Google's tag, Tag Manager container and AdSense loader, put on the page +//! from the publisher's own configuration. +//! +//! This is not the Google Tag Manager integration next door. That one takes a +//! tag the page already carries and re-serves it from the publisher's own +//! origin. This one puts the tag on a page that carries none, which is what a +//! publisher needs when their templates forbid a third-party script and every +//! tag has to arrive from the proxy instead. +//! +//! The two work together. When `google-tag-manager` is also selected and it +//! serves the configured tag or container, the loader written here names the +//! first-party path rather than Google's host, so the script the browser +//! fetches is the one the publisher's own origin answers for. +//! +//! Everything emitted here is copied from Google's own documentation rather +//! than paraphrased, so a reader can hold the two side by side: +//! +//! - The tag snippet and where it goes: +//! +//! - Consent mode defaults and why they come first: +//! +//! - Installing a Tag Manager web container, which is the two-part snippet: +//! +//! - The AdSense code on every page: +//! +//! - The AdSense loader tag: +//! + +use std::rc::Rc; +use std::sync::{Arc, LazyLock}; + +use error_stack::Report; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + AttributeRewriteAction, IntegrationBuilder, IntegrationRegistration, +}; +use trusted_server_core::middleware::{ + AttributeRewrite, AttributeRewriteFn, ElementHandler, HTML_MEDIA_TYPE, Middleware, + MiddlewareAction, MiddlewareContext, MiddlewarePhase, +}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; +use trusted_server_tag_google_tag_manager as google_tag_manager; +use validator::{Validate, ValidationError}; + +/// The id this integration's registration carries. +const GOOGLE_TAG_INTEGRATION_ID: &str = "google_tag"; + +/// The name this module is selected by, in `[tag]`. +pub const MODULE: &str = "tag.google-tag"; + +/// The name a module's builder gives as its source, this crate's package. +const SOURCE: &str = "trusted-server-tag-google-tag"; + +/// The builder the registry runs when a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(GOOGLE_TAG_INTEGRATION_ID, SOURCE, register, validate) + .with_module_name(MODULE) +} + +/// Google's own address for the tag loader. +/// +/// From the install page, which gives the loader as +/// `https://www.googletagmanager.com/gtag/js?id=TAG_ID`. +const GTAG_JS_URL: &str = "https://www.googletagmanager.com/gtag/js"; + +/// Google's own address for the AdSense loader. +/// +/// From the ad unit code page, which gives the loader as +/// `https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=…`. +const ADSENSE_JS_URL: &str = "https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"; + +/// Google's own address for the Tag Manager container script. +/// +/// From the container install page, which builds it in the snippet as +/// `'https://www.googletagmanager.com/gtm.js?id='+i+dl`. +const GTM_JS_URL: &str = "https://www.googletagmanager.com/gtm.js"; + +/// Google's own address for the Tag Manager `noscript` iframe. +/// +/// This one has no first-party route behind it, so it is never substituted. +const GTM_NS_URL: &str = "https://www.googletagmanager.com/ns.html"; + +/// The path Google serves the tag loader from, and the one a first-party +/// route ends with. +const GTAG_JS_PATH_SUFFIX: &str = "/gtag/js"; + +/// An AdSense publisher id, `ca-pub-` and sixteen digits. +static ADSENSE_CLIENT_ID_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"^ca-pub-[0-9]{16}$").expect("the AdSense client id regex should compile") +}); + +/// Whether a consent type is allowed before the visitor has said anything. +/// +/// The names are Google's, from the consent mode guide, and they reach the +/// page as the words `granted` and `denied`. +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ConsentState { + /// The default, and the only one allowed before a permission signal has + /// been read. + #[default] + Denied, + /// Allowed before the visitor has said anything, which only a publisher + /// who knows their own legal basis may configure. + Granted, +} + +impl ConsentState { + /// The word Google's consent command expects. + const fn as_command_value(self) -> &'static str { + match self { + Self::Denied => "denied", + Self::Granted => "granted", + } + } +} + +/// How long the tag waits for a consent management platform's update before +/// it fires with the defaults, when no setting says otherwise. +const DEFAULT_WAIT_FOR_UPDATE_MS: u32 = 500; + +const fn default_wait_for_update() -> u32 { + DEFAULT_WAIT_FOR_UPDATE_MS +} + +/// The four consent types Google requires for its own services, how long the +/// tag waits for a consent management platform before it fires with them, +/// and the regions they apply to. +/// +/// All four default to denied. That is Google's own recommended starting +/// point, and no Google identifier may be stored before a permission signal +/// has been read. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = validate_region_codes))] +pub struct ConsentDefaults { + /// Storage for advertising, such as cookies. + #[serde(default)] + pub ad_storage: ConsentState, + /// Sending user data to Google for advertising. + #[serde(default)] + pub ad_user_data: ConsentState, + /// Personalised advertising. + #[serde(default)] + pub ad_personalization: ConsentState, + /// Storage for analytics, such as cookies. + #[serde(default)] + pub analytics_storage: ConsentState, + /// How long, in milliseconds, the tag waits for a consent management + /// platform's update before it fires with these defaults, which Google's + /// consent guide asks for where a platform sets consent on the page. Zero + /// fires at once. + #[serde(default = "default_wait_for_update")] + #[validate(range(max = 60000, message = "must be at most 60000 milliseconds"))] + pub wait_for_update: u32, + /// The regions the defaults apply to, as ISO 3166-1 country codes or + /// ISO 3166-2 region codes such as `GB` or `US-CA`. Empty applies them + /// everywhere. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub region: Vec, +} + +impl Default for ConsentDefaults { + fn default() -> Self { + Self { + ad_storage: ConsentState::default(), + ad_user_data: ConsentState::default(), + ad_personalization: ConsentState::default(), + analytics_storage: ConsentState::default(), + wait_for_update: DEFAULT_WAIT_FOR_UPDATE_MS, + region: Vec::new(), + } + } +} + +/// Refuses a region that is not a country or region code, two capitals with +/// an optional dash and one to three capitals or digits, so the snippet only +/// ever carries codes. +/// +/// # Errors +/// +/// Returns a [`ValidationError`] naming the first region that is not a code. +fn validate_region_codes(defaults: &ConsentDefaults) -> Result<(), ValidationError> { + static CODE: LazyLock = LazyLock::new(|| { + Regex::new(r"^[A-Z]{2}(-[A-Z0-9]{1,3})?$").expect("the region code pattern compiles") + }); + if let Some(region) = defaults.region.iter().find(|code| !CODE.is_match(code)) { + let mut error = ValidationError::new("google_region_code"); + error.message = Some( + format!("region `{region}` must be a country or region code such as GB or US-CA") + .into(), + ); + return Err(error); + } + Ok(()) +} + +/// Configuration for the Google tag, the Tag Manager container and the +/// AdSense loader. +#[derive(Debug, Clone, Default, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +#[validate(schema(function = validate_at_least_one_id))] +pub struct GoogleConfig { + /// A Google tag id, such as a GA4 measurement id `G-XXXXXXXXXX`. + /// + /// Leaving this out means no tag snippet is written, which is what a + /// publisher running AdSense on its own wants. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(custom(function = validate_analytics_tag_id))] + pub analytics_tag_id: Option, + /// A Tag Manager container id, `GTM-` and four to twenty characters. + /// + /// A container is a different thing from a tag. A tag id names one + /// property to measure and is placed with the `gtag.js` snippet, whereas a + /// container holds whatever tags its owner has configured inside it and is + /// placed with Tag Manager's own two-part installation code. A publisher + /// may have both, and setting both writes both. + /// + /// Leaving this out means no container is written. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(custom(function = validate_container_id))] + pub container_id: Option, + /// An AdSense publisher id, `ca-pub-` followed by sixteen digits. + /// + /// Leaving this out means no AdSense loader is written. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(custom(function = validate_adsense_client_id))] + pub adsense_client_id: Option, + /// What each consent type is before the visitor has said anything. + #[serde(default)] + #[validate(nested)] + pub consent_defaults: ConsentDefaults, +} + +impl IntegrationConfig for GoogleConfig {} + +/// Refuses a configuration naming no tag, no container and no AdSense account. +/// +/// Such a configuration selects the integration and then asks it to write +/// nothing, which reads on the page as the integration being broken rather +/// than as the configuration being empty. +/// +/// # Errors +/// +/// Returns a [`ValidationError`] when all three ids are absent. +fn validate_at_least_one_id(config: &GoogleConfig) -> Result<(), ValidationError> { + if config.analytics_tag_id.is_none() + && config.container_id.is_none() + && config.adsense_client_id.is_none() + { + let mut error = ValidationError::new("google_no_ids"); + error.message = Some( + "[tag.google-tag] needs at least one of analytics_tag_id, container_id and \ + adsense_client_id" + .into(), + ); + return Err(error); + } + Ok(()) +} + +/// Refuses a tag id Google does not serve from `gtag/js`. +/// +/// The shape is the one the Google Tag Manager integration already accepts. +/// +/// # Errors +/// +/// Returns a [`ValidationError`] when the id is not a Google tag id. +fn validate_analytics_tag_id(value: &str) -> Result<(), ValidationError> { + if google_tag_manager::tag_id_is_well_formed(value) { + return Ok(()); + } + let mut error = ValidationError::new("google_tag_id"); + error.message = + Some("analytics_tag_id must be a G-, GT-, GTM-, AW-, DC-, MC- or UA- tag id".into()); + Err(error) +} + +/// Refuses a container id Google Tag Manager would not recognise. +/// +/// The shape is the one the Google Tag Manager integration already accepts. +/// +/// # Errors +/// +/// Returns a [`ValidationError`] when the id is not a container id. +fn validate_container_id(value: &str) -> Result<(), ValidationError> { + if google_tag_manager::container_id_is_well_formed(value) { + return Ok(()); + } + let mut error = ValidationError::new("google_container_id"); + error.message = Some("container_id must be GTM- followed by 4 to 20 capitals or digits".into()); + Err(error) +} + +/// Refuses an AdSense publisher id that is not `ca-pub-` and sixteen digits. +/// +/// # Errors +/// +/// Returns a [`ValidationError`] when the id does not match. +fn validate_adsense_client_id(value: &str) -> Result<(), ValidationError> { + if ADSENSE_CLIENT_ID_PATTERN.is_match(value) { + return Ok(()); + } + let mut error = ValidationError::new("google_adsense_client_id"); + error.message = Some("adsense_client_id must be ca-pub- followed by sixteen digits".into()); + Err(error) +} + +/// The consent defaults command. +/// +/// Copied from . +/// It carries its own `dataLayer` and `gtag` lines because it has to run +/// before anything else, including before the tag snippet that would +/// otherwise define them, and because it is written even when no tag is +/// configured and AdSense alone reads it. +fn consent_defaults_snippet(defaults: &ConsentDefaults) -> String { + // The regions are validated as codes, so the quotes cannot be escaped. + let region = if defaults.region.is_empty() { + String::new() + } else { + let codes: Vec = defaults + .region + .iter() + .map(|code| format!("'{code}'")) + .collect(); + format!(",\n 'region': [{}]", codes.join(", ")) + }; + format!( + "", + ad_storage = defaults.ad_storage.as_command_value(), + ad_user_data = defaults.ad_user_data.as_command_value(), + ad_personalization = defaults.ad_personalization.as_command_value(), + analytics_storage = defaults.analytics_storage.as_command_value(), + wait_for_update = defaults.wait_for_update, + ) +} + +/// Google's tag snippet. +/// +/// Copied from , +/// which says to place it "immediately after the opening `` HTML tag on +/// every page you want to measure". The middleware writes it in the head +/// immediately before the script bundle's tag, so after whatever the fetch +/// phase wrote there. The only substitutions are the tag id and, where the +/// Google Tag Manager integration serves this tag, the loader address. +fn analytics_snippet(tag_id: &str, src: &str) -> String { + format!( + "\n\ + \n\ + " + ) +} + +/// Tag Manager's container script, the first of its two parts. +/// +/// Copied from , which +/// says to paste it "as high in the `` tag as possible". The only +/// substitutions are the container id and, where this origin serves the same +/// container, the address the snippet builds. +fn container_script_snippet(container_id: &str, gtm_js_src: &str) -> String { + format!( + "\n\ + \n\ + " + ) +} + +/// Tag Manager's `noscript` iframe, the second of its two parts. +/// +/// Copied from the same page, which says to paste it "immediately after the +/// opening `` tag". That is why the middleware writes it at the start of +/// ``, because Google places this markup there and nowhere else. +/// +/// The address is never substituted, because `ns.html` has no first-party +/// route behind it. +fn container_noscript_snippet(container_id: &str) -> String { + format!( + "\n\ + \n\ + " + ) +} + +/// The AdSense page-level loader. +/// +/// Copied from . The +/// per-unit `` markup is not written here, because where a unit goes is +/// a property of the publisher's own page rather than of this configuration. +fn adsense_snippet(client_id: &str) -> String { + format!( + "" + ) +} + +/// Whether `value` is a `gtag/js` loader naming `tag_id`. +/// +/// Matching is on the parsed path and the `id` parameter rather than on the +/// whole string, so a loader carrying other parameters, a protocol-relative +/// address or a first-party path is recognised as the same thing. +fn url_is_gtag_js_for(value: &str, tag_id: &str) -> bool { + let trimmed = value.trim(); + // A protocol-relative or rooted address carries no scheme or host of its + // own, so it is given one purely to be parsed. Nothing reads it afterwards. + let absolute = if trimmed.starts_with("//") { + format!("https:{trimmed}") + } else if trimmed.starts_with('/') { + format!("https://placeholder.invalid{trimmed}") + } else { + trimmed.to_owned() + }; + let Ok(parsed) = url::Url::parse(&absolute) else { + return false; + }; + + if !parsed.path().ends_with(GTAG_JS_PATH_SUFFIX) { + return false; + } + + parsed + .query_pairs() + .any(|(name, value)| name == "id" && value == tag_id) +} + +/// The integration, holding what it will write. +struct GoogleIntegration { + config: GoogleConfig, + /// Where the tag loader points. + /// + /// Google's own address, or the first-party path when the Google Tag + /// Manager module is selected and serves this tag. Settled once at + /// startup because it depends on configuration rather than on the + /// request, and because the middleware runs on every document. + gtag_js_src: Option, + /// Where the container script's own loader points, decided the same way. + gtm_js_src: Option, +} + +impl GoogleIntegration { + /// The integration for `config`, with both loader addresses settled. + const fn new( + config: GoogleConfig, + gtag_js_src: Option, + gtm_js_src: Option, + ) -> Self { + Self { + config, + gtag_js_src, + gtm_js_src, + } + } +} + +impl GoogleIntegration { + /// The snippets, in the order Google's own documentation requires. + /// + /// Consent defaults first, because the consent guide says the default + /// command must come "before any commands that send measurement data + /// (such as `config` or `event`)". The container next, because Tag Manager + /// asks to be as high in the head as possible and it may carry tags of its + /// own that send data. Then the tag snippet, then the AdSense loader. + fn page_inserts(&self) -> Vec { + let mut inserts = vec![consent_defaults_snippet(&self.config.consent_defaults)]; + + if let (Some(container_id), Some(src)) = ( + self.config.container_id.as_deref(), + self.gtm_js_src.as_deref(), + ) { + inserts.push(container_script_snippet(container_id, src)); + } + + if let (Some(tag_id), Some(src)) = ( + self.config.analytics_tag_id.as_deref(), + self.gtag_js_src.as_deref(), + ) { + inserts.push(analytics_snippet(tag_id, src)); + } + + if let Some(client_id) = self.config.adsense_client_id.as_deref() { + inserts.push(adsense_snippet(client_id)); + } + + inserts + } +} + +impl GoogleIntegration { + /// Tag Manager's `noscript` iframe, and nothing else. + /// + /// Google's container install page puts this part immediately after the + /// opening `` tag, so it is the one piece of this integration's + /// markup that does not belong in the head. + fn body_inserts(&self) -> Vec { + self.config + .container_id + .as_deref() + .map(container_noscript_snippet) + .into_iter() + .collect() + } +} + +impl GoogleIntegration { + /// Drops a second loader for the tag this module already wrote. + /// + /// A page also carrying the loader for the same id would fetch the same + /// library twice, so the element is removed rather than rewritten. The + /// loader is recognised on its own host and on a first-party path, which is + /// how it arrives here once the fetch phase's rewrite has made it first + /// party. + /// + /// **An inline `gtag('config', …)` in the publisher's own page is left + /// alone and will still send a second page view.** The module's guide + /// says the same. + fn change_for_src(&self, value: &str) -> AttributeRewriteAction { + let Some(tag_id) = self.config.analytics_tag_id.as_deref() else { + return AttributeRewriteAction::Keep; + }; + + if url_is_gtag_js_for(value, tag_id) { + // The id is the publisher's own configured value, so it is not + // logged. The message says what happened and not which tag. + log::debug!( + "Google: the page carries its own loader for the configured tag, \ + so the duplicate is dropped" + ); + return AttributeRewriteAction::RemoveElement; + } + + AttributeRewriteAction::Keep + } +} + +/// Writes the module's markup onto the page for each reader, and removes the +/// page's own loader for the configured tag so the library is fetched once. +/// +/// Serve only, because it adds to the page, and the removal belongs with the +/// addition it makes room for. +struct PageTag(Arc); + +impl Middleware for PageTag { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == HTML_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let mut element_handlers: Vec> = Vec::new(); + if self.0.config.analytics_tag_id.is_some() { + let integration = Arc::clone(&self.0); + let change: Rc = + Rc::new(move |matched| integration.change_for_src(matched.value)); + element_handlers.push(Box::new(AttributeRewrite::new("src", change))); + } + MiddlewareAction { + head_inserts: self.0.page_inserts(), + body_open_inserts: self.0.body_inserts(), + element_handlers, + ..MiddlewareAction::pass() + } + } +} + +/// Validates the Google configuration and reports whether a section selects +/// the module. +/// +/// # Errors +/// +/// Returns an error when the configuration cannot be parsed or fails +/// validation. +pub fn validate(settings: &Settings) -> Result> { + Ok(settings.module_config::(MODULE)?.is_some()) +} + +/// Register the Google tag integration when a section selects it. +/// +/// # Errors +/// +/// Returns an error when the Google configuration, or the Google Tag Manager +/// configuration consulted for first-party delivery, cannot be parsed or +/// fails validation. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + + // Where each loader points is decided once, here, because it depends on + // another module's configuration rather than on the request. + let gtag_js_src = match config.analytics_tag_id.as_deref() { + Some(tag_id) => Some( + google_tag_manager::first_party_gtag_js_path(settings, tag_id)? + .unwrap_or_else(|| GTAG_JS_URL.to_owned()), + ), + None => None, + }; + + let gtm_js_src = match config.container_id.as_deref() { + Some(container_id) => Some( + google_tag_manager::first_party_gtm_js_path(settings, container_id)? + .unwrap_or_else(|| GTM_JS_URL.to_owned()), + ), + None => None, + }; + + let integration = Arc::new(GoogleIntegration::new(config, gtag_js_src, gtm_js_src)); + + Ok(Some( + IntegrationRegistration::builder(GOOGLE_TAG_INTEGRATION_ID) + .with_middleware(Arc::new(PageTag(integration))) + .build(), + )) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use trusted_server_core::test_support::tests::create_test_settings; + + /// An invented tag id of the shape Google's own pages use. + const TEST_TAG_ID: &str = "G-XXXXXXXXXX"; + /// An invented AdSense account of the shape Google's own pages use. + const TEST_ADSENSE_CLIENT_ID: &str = "ca-pub-0000000000000000"; + /// An invented container id. + const TEST_CONTAINER_ID: &str = "GTM-EXAMPLE1"; + + /// Google's snippet as the install page gives it. + const GOOGLE_PUBLISHED_TAG_SNIPPET: &str = "\n\ + \n\ + "; + + /// Google's AdSense loader exactly as + /// gives it. + const GOOGLE_PUBLISHED_ADSENSE_LOADER: &str = concat!( + "" + ); + /// Tag Manager's container script exactly as + /// gives it. + const GOOGLE_PUBLISHED_CONTAINER_SCRIPT: &str = "\n\ + \n\ + "; + + /// Tag Manager's `noscript` part, from the same page. + const GOOGLE_PUBLISHED_CONTAINER_NOSCRIPT: &str = "\n\ + \n\ + "; + + /// An invented publisher's head carrying its own tag manager, so the + /// insertion is checked against a head shaped the way a content + /// management system writes one. + const HEAD_WITH_TAG_MANAGER: &str = include_str!("fixtures/head-with-tag-manager.html"); + /// An invented publisher's head that already loads the configured tag. + const HEAD_WITH_ITS_OWN_TAG: &str = include_str!("fixtures/head-with-its-own-tag.html"); + + /// Settings selecting the `google-tag` module with `config`. + fn settings_with_google(config: &serde_json::Value) -> Settings { + let mut settings = create_test_settings(); + settings + .insert_module_config("tag", MODULE, config) + .expect("should write the google integration config"); + settings + } + + /// Settings selecting both `google-tag` and `google-tag-manager`. + fn settings_with_google_and_tag_manager( + google: &serde_json::Value, + tag_manager: &serde_json::Value, + ) -> Settings { + let mut settings = settings_with_google(google); + settings + .insert_module_config("tag", google_tag_manager::MODULE, tag_manager) + .expect("should write the google_tag_manager config"); + settings + } + + /// The registration a named google integration produces for `settings`. + fn registration_for(settings: &Settings) -> IntegrationRegistration { + register(settings) + .expect("should read the google config") + .expect("a named google integration should register") + } + + /// The refusal `register` answers `settings` with. + /// + /// `IntegrationRegistration` holds trait objects and has no `Debug`, so + /// the error is taken with a pattern rather than with `expect_err`. + fn refusal_for(settings: &Settings) -> String { + let Err(error) = register(settings) else { + panic!("should refuse the configuration"); + }; + format!("{error:?}") + } + + /// The middleware a named module registers for `settings`. + fn page_tag_for(settings: &Settings) -> Arc { + Arc::clone( + registration_for(settings) + .middleware + .first() + .expect("the module should register its middleware"), + ) + } + + /// The head snippets the middleware writes for `settings`. + fn head_inserts_for(settings: &Settings) -> Vec { + let document_state = trusted_server_core::integrations::IntegrationDocumentState::default(); + let context = trusted_server_core::middleware::test_support::context( + MiddlewarePhase::Serve, + &document_state, + ); + page_tag_for(settings).create(&context).head_inserts + } + + /// `html` after the serve phase has run the middleware for `settings`, with + /// the Tag Manager module beside it where the settings select it. + fn process(settings: &Settings, html: &str) -> String { + use trusted_server_core::html_processor::test_support::{ + place_on_every_page, processed_page, + }; + let mut placed = settings.clone(); + place_on_every_page(&mut placed, MiddlewarePhase::Serve, &[MODULE]); + let registry = trusted_server_core::integrations::IntegrationRegistry::with_registrations( + &placed, + &[builder(), google_tag_manager::builder()], + ) + .expect("should build a registry with the google modules"); + processed_page(&placed, ®istry, html, 8192) + } + + #[test] + fn an_unnamed_integration_does_not_register() { + // Arrange + let settings = create_test_settings(); + + // Act + let registration = register(&settings).expect("should read an unnamed integration"); + + // Assert + assert!( + registration.is_none(), + "a google integration nothing names should not register" + ); + } + + #[test] + fn a_config_naming_no_id_is_refused() { + // Arrange + let settings = settings_with_google(&json!({})); + + // Act + let refusal = refusal_for(&settings); + + // Assert + assert!( + refusal.contains("analytics_tag_id") + && refusal.contains("container_id") + && refusal.contains("adsense_client_id"), + "the refusal should name all three settings, one of which is needed: {refusal}" + ); + } + + #[test] + fn an_id_of_the_wrong_shape_is_refused_by_name() { + // Arrange + // A tag id with a prefix Google does not serve, a container id in the + // wrong case, and an AdSense id missing its `ca-` prefix. + let cases = [ + ("analytics_tag_id", "GA-XXXXXXXXXX"), + ("container_id", "GTM-lowercase"), + ("adsense_client_id", "pub-0000000000000000"), + ]; + + for (setting, value) in cases { + let settings = settings_with_google(&json!({ setting: value })); + + // Act + let refusal = refusal_for(&settings); + + // Assert + assert!( + refusal.contains(setting), + "the refusal should name `{setting}` as the setting at fault: {refusal}" + ); + } + } + + #[test] + fn an_unknown_setting_is_refused() { + // Arrange + let settings = settings_with_google(&json!({ + "analytics_tag_id": TEST_TAG_ID, + "analytics_tag": "G-OTHER", + })); + + // Act + let refusal = refusal_for(&settings); + + // Assert + assert!( + refusal.contains("analytics_tag") || refusal.contains("unknown field"), + "the refusal should name the unknown setting: {refusal}" + ); + } + + #[test] + fn the_tag_snippet_is_googles_own_with_the_id_substituted() { + // Arrange + let settings = settings_with_google(&json!({ "analytics_tag_id": TEST_TAG_ID })); + let expected = GOOGLE_PUBLISHED_TAG_SNIPPET.replace("TAG_ID", TEST_TAG_ID); + + // Act + let inserts = head_inserts_for(&settings); + + // Assert + let written = inserts + .iter() + .find(|insert| insert.starts_with("") + .expect("should keep the publisher's own tag manager"); + assert!( + ours < theirs, + "the snippet should come before the publisher's own scripts" + ); + assert_eq!( + processed.matches("gtag('config'").count(), + 1, + "the tag should be configured once" + ); + } + + #[test] + fn a_page_already_loading_the_same_tag_ends_with_one_loader() { + // Arrange + // A page carrying its own loader for the tag this integration writes + // would otherwise fetch the same library twice. + let settings = settings_with_google(&json!({ "analytics_tag_id": TEST_TAG_ID })); + + // Act + let processed = process(&settings, HEAD_WITH_ITS_OWN_TAG); + + // Assert + assert_eq!( + processed.matches("gtag/js?id=").count(), + 1, + "the page should end with one loader for the tag, not two: {processed}" + ); + } + + #[test] + fn a_loader_the_fetch_rewrite_made_first_party_is_removed_as_well() { + // Arrange + // By the time the middleware runs, a fetch phase rewrite may already have + // pointed the page's own loader at the first-party path. + let settings = settings_with_google(&json!({ "analytics_tag_id": TEST_TAG_ID })); + let html = format!( + "" + ); + + // Act + let processed = process(&settings, &html); + + // Assert + assert_eq!( + processed.matches("gtag/js?id=").count(), + 1, + "the page should end with one loader for the tag, not two: {processed}" + ); + } + + #[test] + fn a_page_loading_another_tag_keeps_it() { + // Arrange + let settings = settings_with_google(&json!({ "analytics_tag_id": TEST_TAG_ID })); + let html = "Example Publisher\ + "; + + // Act + let processed = process(&settings, html); + + // Assert + assert!( + processed.contains("id=G-OTHERTAG"), + "a second property the publisher measures is theirs to keep" + ); + assert!( + processed.contains(&format!("id={TEST_TAG_ID}")), + "the configured tag should still be written" + ); + } + + #[test] + fn the_container_script_is_googles_own_with_the_id_substituted() { + // Arrange + let settings = settings_with_google(&json!({ "container_id": TEST_CONTAINER_ID })); + let expected = GOOGLE_PUBLISHED_CONTAINER_SCRIPT.replace("CONTAINER_ID", TEST_CONTAINER_ID); + + // Act + let inserts = head_inserts_for(&settings); + + // Assert + let written = inserts + .iter() + .find(|insert| insert.starts_with("")) + .expect("should write the container script"); + assert_eq!( + written, &expected, + "the container script should be Google's own, changed only where the id goes" + ); + } + + #[test] + fn the_container_noscript_is_googles_own_with_the_id_substituted() { + // Arrange + let settings = settings_with_google(&json!({ "container_id": TEST_CONTAINER_ID })); + let expected = + GOOGLE_PUBLISHED_CONTAINER_NOSCRIPT.replace("CONTAINER_ID", TEST_CONTAINER_ID); + + // Act + let processed = process(&settings, ""); + + // Assert + assert!( + processed.contains(&format!("{expected}")), + "the noscript part should be Google's own, changed only where the id goes, and \ + written immediately after the opening body tag: {processed}" + ); + } + + #[test] + fn consent_defaults_come_before_the_container() { + // Arrange + let settings = settings_with_google(&json!({ "container_id": TEST_CONTAINER_ID })); + + // Act + let written = head_inserts_for(&settings).join(""); + + // Assert + let consent = written + .find("gtag('consent', 'default'") + .expect("should write the consent defaults"); + let container = written + .find("") + .expect("should write the container script"); + assert!( + consent < container, + "a container may carry tags that send measurement data, so the defaults come first" + ); + } + + #[test] + fn a_container_and_a_tag_are_both_written_container_first() { + // Arrange + let settings = settings_with_google(&json!({ + "container_id": TEST_CONTAINER_ID, + "analytics_tag_id": TEST_TAG_ID, + })); + + // Act + let written = head_inserts_for(&settings).join(""); + + // Assert + let container = written + .find("") + .expect("should write the container script"); + let tag = written + .find("") + .expect("should write the tag snippet"); + assert!( + container < tag, + "Tag Manager asks to be as high in the head as possible" + ); + } + + #[test] + fn the_container_loader_names_googles_host_when_tag_manager_is_not_enabled() { + // Arrange + let settings = settings_with_google(&json!({ "container_id": TEST_CONTAINER_ID })); + + // Act + let written = head_inserts_for(&settings).join(""); + + // Assert + assert!( + written.contains("'https://www.googletagmanager.com/gtm.js?id='+i+dl"), + "with nothing serving the container first party, the snippet is Google's own" + ); + } + + #[test] + fn the_container_loader_names_the_first_party_path_for_the_same_container() { + // Arrange + let settings = settings_with_google_and_tag_manager( + &json!({ "container_id": TEST_CONTAINER_ID }), + &json!({ "container_id": TEST_CONTAINER_ID }), + ); + + // Act + let written = head_inserts_for(&settings).join(""); + + // Assert + assert!( + written.contains("'/integrations/google_tag_manager/gtm.js?id='+i+dl"), + "the container this origin serves should be fetched from this origin: {written}" + ); + } + + #[test] + fn the_container_loader_names_googles_host_for_a_different_container() { + // Arrange + // That endpoint always answers with the container it is configured + // for, so a first-party path would serve the wrong one. + let settings = settings_with_google_and_tag_manager( + &json!({ "container_id": TEST_CONTAINER_ID }), + &json!({ "container_id": "GTM-OTHER123" }), + ); + + // Act + let written = head_inserts_for(&settings).join(""); + + // Assert + assert!( + written.contains("'https://www.googletagmanager.com/gtm.js?id='+i+dl"), + "a container this origin does not serve stays with Google" + ); + } + + #[test] + fn the_noscript_lands_immediately_after_the_opening_body_tag_once() { + // Arrange + let settings = settings_with_google(&json!({ "container_id": TEST_CONTAINER_ID })); + let html = "Example Publisher\ +

Headline

"; + + // Act + let processed = process(&settings, html); + + // Assert + assert_eq!( + processed.matches("ns.html?id=").count(), + 1, + "the noscript iframe should be written once" + ); + let body_open = processed.find("").expect("should keep the body tag"); + let noscript = processed + .find("") + .expect("should write the noscript part"); + let headline = processed + .find("

") + .expect("should keep the page content"); + assert!( + body_open < noscript && noscript < headline, + "Google's install page puts this part immediately after the opening body tag: \ + {processed}" + ); + } + + #[test] + fn a_second_body_does_not_get_a_second_noscript() { + // Arrange + // Template fragments sometimes carry more than one body element. + let settings = settings_with_google(&json!({ "container_id": TEST_CONTAINER_ID })); + let html = "Example Publisher\ +

Headline

Second"; + + // Act + let processed = process(&settings, html); + + // Assert + assert_eq!( + processed.matches("ns.html?id=").count(), + 1, + "the noscript iframe should be written once per document" + ); + } + + #[test] + fn a_page_with_no_container_configured_is_left_alone_at_the_body() { + // Arrange + let settings = settings_with_google(&json!({ "analytics_tag_id": TEST_TAG_ID })); + let html = "Example Publisher\ +

Headline

"; + + // Act + let processed = process(&settings, html); + + // Assert + assert!( + !processed.contains("noscript"), + "nothing should be written at the body when no container is configured" + ); + } +} diff --git a/crates/testing/seam-probe/.gitattributes b/crates/testing/seam-probe/.gitattributes new file mode 100644 index 0000000000..02c57b3117 --- /dev/null +++ b/crates/testing/seam-probe/.gitattributes @@ -0,0 +1,5 @@ +# The crate embeds js/probe.js with `include_str!` and states its SHA-256 as a +# literal, so the file must arrive byte identical on every platform. Without +# this, a Windows checkout under `core.autocrlf = true` would rewrite the line +# endings and the declared hash would no longer match its source. +js/probe.js text eol=lf diff --git a/crates/testing/seam-probe/Cargo.toml b/crates/testing/seam-probe/Cargo.toml new file mode 100644 index 0000000000..a3221b34d4 --- /dev/null +++ b/crates/testing/seam-probe/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "trusted-server-integration-seam-probe" +description = "Test fixture integration that exercises the Trusted Server integration seam from outside the core crate. Never ship in a deployment." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +hex = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } +sha2 = { workspace = true } +tokio = { workspace = true } + +# The visible owner of this crate, the way Prebid.js requires a named +# maintainer of every adapter. The seam probe is a test fixture the project +# keeps, so the Trusted Server maintainers own it. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "project owned" diff --git a/crates/testing/seam-probe/js/probe.js b/crates/testing/seam-probe/js/probe.js new file mode 100644 index 0000000000..8f11c90c79 --- /dev/null +++ b/crates/testing/seam-probe/js/probe.js @@ -0,0 +1,3 @@ +(function () { + window.__tsSeamProbe = { module: "seam_probe", carried: true }; +})(); diff --git a/crates/testing/seam-probe/src/lib.rs b/crates/testing/seam-probe/src/lib.rs new file mode 100644 index 0000000000..ae85c75cbb --- /dev/null +++ b/crates/testing/seam-probe/src/lib.rs @@ -0,0 +1,1351 @@ +//! An integration that lives outside `trusted-server-core` and exercises every +//! part of the integration seam from a vendor crate's position. +//! +//! One registration carries a browser module, a proxy route, a geo module, an +//! Edge Cookie identity module, a device module, a prefix module, a robots.txt +//! contributor and a middleware for each phase of a page, alongside its own +//! configuration block, and the builder adds a request preparer, a demand +//! implementation `[demand]` can name and an ad server implementation +//! `[ad-server]` can name. The round-trip tests in +//! `crates/trusted-server-adapter-axum/tests/seam_probe.rs` and +//! `crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs` drive +//! each of those through a real adapter, so the seam is proven by a caller that +//! core does not know about. +//! +//! This crate is a test fixture and must never ship in a deployment. It +//! reports internal request state over an unauthenticated route, serves a +//! browser module that does nothing useful, and resolves location from static +//! configuration rather than from the request. It is a development dependency +//! of the Axum and Fastly adapters only, so no adapter's production build +//! reaches it. + +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::rc::Rc; +use std::sync::{Arc, LazyLock, Mutex}; + +use async_trait::async_trait; +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::header::{self, HeaderValue}; +use http::{Request, Response}; +use serde::Deserialize; +use serde_json::json; +use trusted_server_core::auction::demand::{ + AdServerImplementation, CompiledDemand, DemandFieldPolicy, DemandImplementation, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RequestExtensions, accept_endpoint, +}; +use trusted_server_core::auction::provider::{AuctionProvider, ProviderRequestOutcome}; +use trusted_server_core::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; +use trusted_server_core::ec::device::{ + DeviceAttributes, DeviceModule, DeviceSignals, HumanConfidence, +}; +use trusted_server_core::ec::module::{EdgeCookieModule, GeneratedEdgeCookie, ModuleCode}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::RequestInfo; +use trusted_server_core::integrations::{ + AttributeRewriteAction, CarriedJsModule, IntegrationBuilder, IntegrationEndpoint, + IntegrationProxy, IntegrationRegistration, ScriptRewriteAction, +}; +use trusted_server_core::middleware::{ + AttributeRewrite, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, TextHandler, +}; +use trusted_server_core::module_context::{ModuleCall, ModuleRequest, Withheld}; +use trusted_server_core::platform::{ + GeoInfo, PlatformError, PlatformGeo, PlatformResponse, RuntimeServices, ServerKeyPrefix, +}; +use trusted_server_core::prefix::{PrefixModule, PublicPrefix}; +use trusted_server_core::robots_txt::{Contribution, RobotsTxtContributor}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; +use validator::Validate; + +/// Integration id, which names the probe's routes and its browser module. +pub const SEAM_PROBE_ID: &str = "seam_probe"; + +/// The name this module is selected by, its crate folder below `crates/`, so +/// `[testing] modules = ["seam-probe"]` selects it. Its geo, Edge Cookie and +/// device modules carry the same name, so `[geo] module`, `[ec] module` and +/// `[device] module` select each of them as `testing.seam-probe`, and so does +/// `[robots-txt] modules` its robots.txt contributor. +#[must_use] +pub fn module_name() -> &'static str { + trusted_server_core::module_name!() +} + +/// The crawler the probe's robots.txt rules are addressed to. +pub const SEAM_PROBE_ROBOTS_AGENT: &str = "SeamProbeBot"; + +/// Source label the registry uses in duplicate-id errors. +pub const SEAM_PROBE_SOURCE: &str = "trusted-server-integration-seam-probe"; + +/// Path of the proxy route that reports what the seam delivered. +/// +/// The route itself is built through [`IntegrationProxy::get`], so +/// `report_route_is_the_documented_path` pins this constant to the path the +/// registry actually routes. +pub const SEAM_PROBE_REPORT_PATH: &str = "/integrations/seam_probe/report"; + +/// Path suffix of the report route, relative to the proxy prefix. +const REPORT_ROUTE_SUFFIX: &str = "/report"; + +/// The browser module this registration carries, built outside +/// `trusted-server-js`. +pub const PROBE_JS: &str = include_str!("../js/probe.js"); + +/// SHA-256 of [`PROBE_JS`], hex encoded, lower case. +/// +/// The registry rejects a registration whose declared hash does not match its +/// source, so a stale literal is a startup error rather than a stale script. +/// `carried_module_hash_literal_matches_its_source` keeps this honest. +pub const PROBE_JS_SHA256: &str = + "4711826cac0ce3df585c4a2b30f6ad5d565900a81e89f0ee00148efff2342079"; + +/// Message the probe's own deploy rule rejects a bad country with, so a test +/// can prove deploy validation reached a vendor's rules rather than stopping +/// at the core ones. +pub const SEAM_PROBE_COUNTRY_MESSAGE: &str = + "`[testing.seam-probe] country` must be exactly two letters"; + +/// Request header a caller sets to have the preparer count that request. +/// +/// The header's value is a token the caller chooses, so tests running in +/// parallel count into separate entries. +pub const SEAM_PROBE_COUNT_HEADER: &str = "x-seam-probe-count"; + +/// How many times the preparer ran for each counting token. +/// +/// Request extensions die with the request, so a route that reports nothing +/// about them cannot say how many times its request was prepared. This +/// counter lets a caller pin the once-per-request invariant on any route, +/// including the named routes that return a fixed response. +static PREPARE_COUNTS: LazyLock>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); + +/// How many times the preparer ran for requests carrying `token` in +/// [`SEAM_PROBE_COUNT_HEADER`]. +/// +/// # Panics +/// +/// Panics when the counter's lock is poisoned. +#[must_use] +pub fn prepare_runs_for(token: &str) -> usize { + PREPARE_COUNTS + .lock() + .expect("should lock the seam probe prepare counts") + .get(token) + .copied() + .unwrap_or(0) +} + +/// Records one preparer run against the token the request carries, if any. +fn count_prepare_run(request: &Request) { + let Some(token) = request + .headers() + .get(SEAM_PROBE_COUNT_HEADER) + .and_then(|value| value.to_str().ok()) + else { + return; + }; + let mut counts = PREPARE_COUNTS + .lock() + .expect("should lock the seam probe prepare counts"); + *counts.entry(token.to_string()).or_insert(0) += 1; +} + +/// What the probe's middleware writes at the start of `` when the probe +/// is set to `country`. +#[must_use] +pub fn seam_probe_head_marker(country: &str) -> String { + format!("") +} + +/// The markup the probe's middleware writes immediately after the opening +/// body tag, so a test can see that place is reached. +#[must_use] +pub fn seam_probe_body_marker() -> String { + "".to_owned() +} + +/// The word the probe's middleware swaps for the country, in the text of a +/// script that carries `data-seam-probe`. +pub const SEAM_PROBE_SCRIPT_WORD: &str = "SEAM_PROBE_COUNTRY"; + +/// The probe's page change, which an entry names as the module is named. +/// +/// It marks the head with the country the probe is set to, points every link +/// that carries `data-seam-probe` at the probe's report route, and writes the +/// country in place of [`SEAM_PROBE_SCRIPT_WORD`] in the text of a script +/// that carries `data-seam-probe`. +pub struct SeamProbeMiddleware { + country: String, +} + +impl Middleware for SeamProbeMiddleware { + fn middleware_id(&self) -> &'static str { + module_name() + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + head_inserts: vec![seam_probe_head_marker(&self.country)], + body_open_inserts: vec![seam_probe_body_marker()], + element_handlers: vec![Box::new(AttributeRewrite::matching( + "a[data-seam-probe]", + "href", + Rc::new(|_matched| AttributeRewriteAction::replace(SEAM_PROBE_REPORT_PATH)), + ))], + text_handlers: vec![Box::new(CountryInScript { + country: self.country.clone(), + held: String::new(), + })], + ..MiddlewareAction::pass() + } + } +} + +/// The name an entry gives the probe's serve middleware, which is the +/// module's name with a part of its own after it. +#[must_use] +pub fn reader_middleware_name() -> &'static str { + trusted_server_core::module_name!("reader") +} + +/// What the probe's serve middleware writes ahead of the script bundle for a +/// reader who asked for `request_host`. +#[must_use] +pub fn seam_probe_reader_marker(request_host: &str) -> String { + format!("") +} + +/// The probe's change to one reader's copy of a page, which marks the head +/// with the host that reader asked for. +pub struct SeamProbeReaderMiddleware; + +impl Middleware for SeamProbeReaderMiddleware { + fn middleware_id(&self) -> &'static str { + reader_middleware_name() + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + head_inserts: vec![seam_probe_reader_marker(context.request_host)], + ..MiddlewareAction::pass() + } + } +} + +/// Holds a script's text until its last chunk arrives, then writes the whole +/// of it back with the word swapped, because the word may lie across chunks. +struct CountryInScript { + country: String, + held: String, +} + +impl TextHandler for CountryInScript { + fn selector(&self) -> &str { + "script[data-seam-probe]" + } + + fn decide(&mut self, text: &str, is_last: bool) -> ScriptRewriteAction { + self.held.push_str(text); + if !is_last { + return ScriptRewriteAction::remove_node(); + } + ScriptRewriteAction::replace( + std::mem::take(&mut self.held).replace(SEAM_PROBE_SCRIPT_WORD, &self.country), + ) + } +} + +/// Marker the request preparer inserts into the request extensions. +/// +/// The proxy route reports [`Self::runs`], so a test observes both that the +/// preparer ran and how many times the request path ran it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SeamProbePrepared { + /// How many times the preparer ran for this request. + pub runs: usize, +} + +/// The probe's own configuration block, `[testing.seam-probe]`. +#[derive(Debug, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct SeamProbeConfig { + /// Country the probe's geo module resolves for every request. + /// + /// Rejected by [`validate`] when it is not two letters, which is the + /// vendor-owned rule the deploy validation round trip exercises. + #[serde(default = "default_country")] + pub country: String, + /// Whether the registration declares the probe's geo module. + /// + /// Set this to `false` to build a selected module that supplies no geo + /// module, which is what `[geo] module = "testing.seam-probe"` must + /// reject at startup. + #[serde(default = "default_declares_geo")] + pub declares_geo: bool, +} + +impl IntegrationConfig for SeamProbeConfig {} + +/// `ZZ` is reserved in ISO 3166-1 for private use, so it names no real place. +fn default_country() -> String { + "ZZ".to_string() +} + +const fn default_declares_geo() -> bool { + true +} + +/// Geo module that resolves the country the probe's configuration names, +/// whatever the client address is. +pub struct SeamProbeGeo { + country: String, +} + +impl SeamProbeGeo { + /// Creates a module that resolves `country` for every request. + #[must_use] + pub const fn new(country: String) -> Self { + Self { country } + } +} + +/// The probe's Edge Cookie module, declared on its registration so identity +/// reaches core the same way location does. +/// +/// It derives a value from the request evidence rather than a constant, so a +/// test can tell the difference between the module's module running and +/// core's built-in one running. +#[derive(Debug)] +pub struct SeamProbeEc; + +/// The four-character code core stamps on identifiers this module owns. +const SEAM_PROBE_EC_CODE: ModuleCode = trusted_server_core::module_code!("sprb"); + +impl SeamProbeEc { + /// An identifier derived from the client IP in the request's evidence. + fn derive(&self, request_info: &dyn RequestInfo) -> GeneratedEdgeCookie { + // Derived from evidence so the test can prove this ran rather than the + // built-in module, and prove the evidence actually arrived. + let ip = request_info.client_ip(); + let id = format!("seam-probe-{}", if ip.is_empty() { "no-ip" } else { ip }); + GeneratedEdgeCookie { + id: Some(id), + response_headers: Vec::new(), + } + } +} + +#[async_trait::async_trait(?Send)] +impl EdgeCookieModule for SeamProbeEc { + fn id(&self) -> &'static str { + module_name() + } + + fn code(&self) -> ModuleCode { + SEAM_PROBE_EC_CODE + } + + async fn generate( + &self, + call: ModuleCall<'_>, + ) -> Result> { + Ok(call.inject(self, Self::derive).unwrap_or_default()) + } + + fn accepts_id(&self, value: &str) -> bool { + value.starts_with("seam-probe-") + } +} + +/// The probe's device module, declared on the same registration. +#[derive(Debug)] +pub struct SeamProbeDevice; + +#[async_trait::async_trait(?Send)] +impl DeviceModule for SeamProbeDevice { + fn id(&self) -> &'static str { + module_name() + } + + async fn detect(&self, _call: ModuleCall<'_>) -> DeviceSignals { + DeviceSignals { + is_mobile: 1, + platform_class: Some("seam-probe".to_owned()), + known_browser: Some(true), + looks_like_browser: true, + ja4_class: None, + h2_fp_hash: None, + // A scoring module's answer, which core's own modules never give. + human_confidence: HumanConfidence::Medium, + } + } + + async fn advertising_attributes(&self, _call: ModuleCall<'_>) -> Option { + Some(SeamProbeDevice::attributes()) + } +} + +impl SeamProbeDevice { + /// What the probe tells a bidder about the device, fixed so a test can + /// read it back. + #[must_use] + pub fn attributes() -> DeviceAttributes { + DeviceAttributes { + // A tablet, as the module gives `OpenRTB`'s own code. + device_type: Some(5), + make: Some("Probe".to_owned()), + model: Some("P-1".to_owned()), + os: Some("ProbeOS".to_owned()), + os_version: Some("1.0".to_owned()), + screen_width: Some(800), + screen_height: Some(1280), + } + } +} + +/// The head the probe's prefix module gives, so a test can tell its names +/// from core's own `ts`. +pub const SEAM_PROBE_PREFIX_HEAD: &str = "_probe"; + +/// The probe's prefix module, declared on the same registration, which gives +/// `_probe` for the names the browser holds and `probe.` +/// for the keys in a shared store. +#[derive(Debug)] +pub struct SeamProbePrefix; + +impl SeamProbePrefix { + fn server_key(&self, settings: &Settings) -> ServerKeyPrefix { + ServerKeyPrefix::new("probe", &settings.publisher.domain) + } +} + +impl PrefixModule for SeamProbePrefix { + fn public_prefix(&self, _call: ModuleCall<'_>) -> Result { + Ok(PublicPrefix::new([SEAM_PROBE_PREFIX_HEAD]).expect("the head is a valid prefix")) + } + + fn server_key_prefix(&self, call: ModuleCall<'_>) -> Result { + call.inject(self, Self::server_key) + } +} + +/// The probe's robots.txt contributor, declared on the same registration. +/// +/// Its one rule names the request it was asked for, so a test can read back +/// that the contributor was handed the host and the path and not the query. +#[derive(Debug)] +pub struct SeamProbeRobotsTxt; + +impl SeamProbeRobotsTxt { + fn rules(&self, request: ModuleRequest<'_>) -> Contribution { + Contribution::parse(&format!( + "User-Agent: {SEAM_PROBE_ROBOTS_AGENT}\nDisallow: /{}{}?{}", + request.host(), + request.path(), + request.query() + )) + } +} + +#[async_trait::async_trait(?Send)] +impl RobotsTxtContributor for SeamProbeRobotsTxt { + fn refresh(&self) -> core::time::Duration { + core::time::Duration::from_secs(60) + } + + fn fingerprint(&self) -> String { + "seam-probe".to_owned() + } + + async fn contribute( + &self, + call: ModuleCall<'_>, + ) -> Result> { + Ok(call.inject(self, Self::rules)?) + } +} + +#[async_trait::async_trait(?Send)] +impl PlatformGeo for SeamProbeGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &trusted_server_core::platform::RuntimeServices, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: "Example City".to_string(), + country: self.country.clone(), + continent: "Example Continent".to_string(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + })) + } +} + +/// The name an `[ad-server.]` or `[demand.]` `implementation` +/// line selects the probe's ad server or demand source by, which is the +/// probe's module name. +/// +/// A static cannot call [`module_name`], so the literal is written out and +/// `auction_implementations_are_named_as_the_module` keeps it honest. +pub const SEAM_PROBE_ADSERVER_NAME: &str = "testing.seam-probe"; + +/// The probe's demand implementation, registered on its builder. +pub static SEAM_PROBE_DEMAND: DemandImplementation = DemandImplementation { + id: SEAM_PROBE_ADSERVER_NAME, + default_timeout: DemandTimeoutDefault::Auction, + allows_all_eligible: false, + serves_stored_requests: false, + canonicalize_endpoint: accept_endpoint, + compile: compile_demand, +}; + +/// A demand source that sends the standard request with nothing of its own +/// and reads every answer as no bid, so the auction's demand seam is reached +/// from a crate core does not know. +struct SeamProbeDemand; + +/// Compiles the probe's demand source, which takes no settings of its own. +/// +/// # Errors +/// +/// Returns an error naming the first setting the table carries, because the +/// source reads none. +fn compile_demand( + settings: &serde_json::Map, +) -> Result, Report> { + if let Some(key) = settings.keys().next() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("the seam probe's demand source takes no setting, and `{key}` is set"), + })); + } + Ok(Arc::new(SeamProbeDemand)) +} + +#[async_trait(?Send)] +impl CompiledDemand for SeamProbeDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy::default() + } + + fn augment_request( + &self, + _extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + _response: PlatformResponse, + ) -> Result> { + Ok(AuctionResponse::no_bid( + context.provider_id, + context.response_time_ms, + )) + } + + fn as_any(&self) -> &dyn core::any::Any { + self + } +} + +/// The probe's ad server implementation, registered on its builder. +pub static SEAM_PROBE_ADSERVER: AdServerImplementation = AdServerImplementation { + id: SEAM_PROBE_ADSERVER_NAME, + build: build_adserver, +}; + +/// How long the orchestrator waits for the probe's ad server, in milliseconds. +const SEAM_PROBE_ADSERVER_TIMEOUT_MS: u32 = 100; + +/// An ad server that answers at once and picks nothing, so the auction's ad +/// server seam is reached from a crate core does not know. +struct SeamProbeAdServer { + name: String, +} + +/// Builds the probe's ad server under the name its table gave it. +/// +/// # Errors +/// +/// Never returns an error; the result type matches `BuildAdServerFn`. +fn build_adserver( + name: &str, + _settings: &serde_json::Map, +) -> Result, Report> { + Ok(Arc::new(SeamProbeAdServer { + name: name.to_owned(), + })) +} + +#[async_trait(?Send)] +impl AuctionProvider for SeamProbeAdServer { + fn provider_name(&self) -> &str { + &self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.name.clone(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::no_bid(self.name.clone(), response_time_ms)) + } + + fn timeout_ms(&self) -> u32 { + SEAM_PROBE_ADSERVER_TIMEOUT_MS + } +} + +/// Proxy that reports what the seam delivered to this request. +pub struct SeamProbeProxy; + +impl SeamProbeProxy { + async fn route( + &self, + req: Request, + services: &RuntimeServices, + ) -> Result, Report> { + let country = match services + .geo() + .lookup(services.client_info().client_ip, services) + .await + { + Ok(geo) => geo.map(|info| info.country), + Err(error) => { + log::warn!("seam probe geo lookup failed: {error}"); + None + } + }; + let prepared = req.extensions().get::().copied(); + + let body = json!({ + "module": module_name(), + "geo_country": country, + "request_preparer_runs": prepared.map_or(0, |marker| marker.runs), + "public_prefix": services.public_prefix().as_str(), + "server_key_prefix": services.server_key_prefix().as_str(), + }) + .to_string(); + + let mut response = Response::new(EdgeBody::from(body)); + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + Ok(response) + } +} + +#[async_trait(?Send)] +impl IntegrationProxy for SeamProbeProxy { + fn integration_name(&self) -> &'static str { + SEAM_PROBE_ID + } + + fn routes(&self) -> Vec { + vec![self.get(REPORT_ROUTE_SUFFIX)] + } + + async fn handle( + &self, + call: trusted_server_core::module_context::ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::route)?.await + } +} + +/// Rejects a country that is not exactly two ASCII letters. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Integration`] carrying +/// [`SEAM_PROBE_COUNTRY_MESSAGE`] and the offending value. +fn check_country(country: &str) -> Result<(), Report> { + if country.len() == 2 && country.bytes().all(|byte| byte.is_ascii_alphabetic()) { + return Ok(()); + } + + Err(Report::new(TrustedServerError::Integration { + integration: SEAM_PROBE_ID.to_string(), + message: format!("{SEAM_PROBE_COUNTRY_MESSAGE}, not `{country}`"), + })) +} + +/// Reads the probe's settings table, or `None` when no section selects the +/// probe. +/// +/// # Errors +/// +/// Returns an error when the block is present but cannot be parsed or fails +/// validation. +fn read_config(settings: &Settings) -> Result, Report> { + settings.module_config::(module_name()) +} + +/// Builds the probe's registration, or `None` when no section selects the +/// probe. +/// +/// # Errors +/// +/// Returns an error when the configuration block cannot be parsed or fails +/// validation, or when the probe is selected with a country that is not two +/// letters. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = read_config(settings)? else { + return Ok(None); + }; + check_country(&config.country)?; + + let mut registration = IntegrationRegistration::builder(SEAM_PROBE_ID) + .with_proxy(Arc::new(SeamProbeProxy)) + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_middleware(Arc::new(SeamProbeMiddleware { + country: config.country.clone(), + })) + .with_middleware(Arc::new(SeamProbeReaderMiddleware)); + if config.declares_geo { + registration = registration + .with_geo_module(module_name(), Arc::new(SeamProbeGeo::new(config.country))); + } + { + // Identity, device, the prefixes and robots.txt rules are declared + // the same way location is, so one module supplies all five and there + // is no second mechanism. + registration = registration + .with_ec_module(module_name(), Arc::new(SeamProbeEc)) + .with_device_module(module_name(), Arc::new(SeamProbeDevice)) + .with_prefix_module(module_name(), Arc::new(SeamProbePrefix)) + .with_robots_txt_contributor(module_name(), Arc::new(SeamProbeRobotsTxt)); + } + + Ok(Some(registration.build())) +} + +/// Validates the probe's configuration for deployment and reports whether the +/// probe is selected. +/// +/// # Errors +/// +/// Returns an error when the configuration block cannot be parsed or fails +/// validation, or when the probe is selected with a country that is not two +/// letters. +pub fn validate(settings: &Settings) -> Result> { + let Some(config) = read_config(settings)? else { + return Ok(false); + }; + check_country(&config.country)?; + Ok(true) +} + +/// Inserts [`SeamProbePrepared`] into the request extensions, counting how +/// many times the request path ran it. +/// +/// Runs whether or not the probe is selected, which is the contract the +/// registry gives preparers. +/// +/// # Errors +/// +/// Never returns an error; the result type matches +/// `IntegrationPrepareRequestFn`. +pub fn prepare_request( + _settings: &Settings, + request: &mut Request, +) -> Result<(), Report> { + count_prepare_run(request); + let runs = request + .extensions() + .get::() + .map_or(0, |marker| marker.runs); + request + .extensions_mut() + .insert(SeamProbePrepared { runs: runs + 1 }); + Ok(()) +} + +/// The probe's integration builder, for an adapter's +/// `routes_with_registrations` or `build_state_with_registrations`. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_integration_seam_probe::{builder, SEAM_PROBE_ID}; +/// +/// assert_eq!(builder().id(), SEAM_PROBE_ID); +/// ``` +#[must_use] +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(SEAM_PROBE_ID, SEAM_PROBE_SOURCE, register, validate) + .with_module_name(module_name()) + .with_request_preparer(prepare_request) + .with_demand(&SEAM_PROBE_DEMAND) + .with_adserver(&SEAM_PROBE_ADSERVER) +} + +/// The vendor's name, which names the table `[shared.seam-probe]` that runs +/// [`shared_builder`]. +pub const SEAM_PROBE_SHARED_VENDOR: &str = "seam-probe"; + +/// The id of the builder the shared table runs. +pub const SEAM_PROBE_SHARED_ID: &str = "seam_probe_shared"; + +/// The route the shared builder answers with the table's marker. +pub const SEAM_PROBE_SHARED_PATH: &str = "/integrations/seam_probe_shared/marker"; + +/// The shared route's suffix under the builder's proxy prefix. +const SHARED_ROUTE_SUFFIX: &str = "/marker"; + +/// What `[shared.seam-probe]` carries. +#[derive(Debug, Clone, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct SeamProbeShared { + /// The text the shared route answers with. + #[validate(length(min = 1))] + pub marker: String, +} + +/// A route answering the shared table's marker, which proves the table ran +/// the builder and reached what it registered. +#[derive(Debug)] +struct SharedMarkerRoute { + marker: String, +} + +impl SharedMarkerRoute { + async fn route( + &self, + _req: Request, + ) -> Result, Report> { + Response::builder() + .status(200) + .header( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ) + .body(EdgeBody::from(self.marker.clone().into_bytes())) + .map_err(|error| { + Report::new(TrustedServerError::Integration { + integration: SEAM_PROBE_SHARED_ID.to_string(), + message: error.to_string(), + }) + }) + } +} + +#[async_trait(?Send)] +impl IntegrationProxy for SharedMarkerRoute { + fn integration_name(&self) -> &'static str { + SEAM_PROBE_SHARED_ID + } + + fn routes(&self) -> Vec { + vec![self.get(SHARED_ROUTE_SUFFIX)] + } + + async fn handle( + &self, + call: ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::route)?.await + } +} + +/// Registers the shared route when `[shared.seam-probe]` is present. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read or fails its validation. +pub fn register_shared( + settings: &Settings, +) -> Result, Report> { + let Some(shared) = settings.shared_config::(SEAM_PROBE_SHARED_VENDOR)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(SEAM_PROBE_SHARED_ID) + .without_js() + .with_proxy(Arc::new(SharedMarkerRoute { + marker: shared.marker, + })) + .build(), + )) +} + +/// Reports whether the shared table is present, validating it. +/// +/// # Errors +/// +/// Returns an error when the table cannot be read or fails its validation. +pub fn validate_shared(settings: &Settings) -> Result> { + Ok(settings + .shared_config::(SEAM_PROBE_SHARED_VENDOR)? + .is_some()) +} + +/// The builder `[shared.seam-probe]` runs, which proves a vendor's shared +/// table from outside core. +#[must_use] +pub fn shared_builder() -> IntegrationBuilder { + IntegrationBuilder::new( + SEAM_PROBE_SHARED_ID, + SEAM_PROBE_SOURCE, + register_shared, + validate_shared, + ) + .with_shared_settings(SEAM_PROBE_SHARED_VENDOR) +} + +#[cfg(test)] +mod tests { + use super::*; + + use sha2::{Digest as _, Sha256}; + use trusted_server_core::platform::{ + BackendNamingPolicy, ClientInfo, DisabledGeo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformSecretStore, StoreId, StoreName, UnavailableHttpClient, + UnavailableKvStore, + }; + + /// Config store that answers nothing, so a test can build + /// [`RuntimeServices`] without a real platform. + /// + /// [`SeamProbeGeo::lookup`] resolves from the country it holds and reads + /// none of the services, so the store is never queried. + struct StubConfigStore; + + impl PlatformConfigStore for StubConfigStore { + fn get( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + /// Secret store that answers nothing, paired with [`StubConfigStore`]. + struct StubSecretStore; + + impl PlatformSecretStore for StubSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + /// Backend manager that registers nothing, paired with the stub stores. + struct StubBackend; + + impl PlatformBackend for StubBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + /// Builds a minimal [`RuntimeServices`] the geo lookup can be handed. Every + /// service is a stub or a core-provided unavailable implementation because + /// [`SeamProbeGeo::lookup`] reads none of them. + fn build_probe_services() -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(StubConfigStore)) + .secret_store(Arc::new(StubSecretStore)) + .kv_store(Arc::new(UnavailableKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(Arc::new(UnavailableHttpClient)) + .geo(Arc::new(DisabledGeo)) + .client_info(ClientInfo::default()) + .build() + } + + /// Settings carrying a `[testing.seam-probe]` block with `body` + /// appended to it. + fn settings_with_probe(body: &str) -> Settings { + Settings::from_toml(&format!( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "seam-probe-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + [testing] + modules = ["seam-probe"] + + [testing.seam-probe] + {body} + "# + )) + .expect("should parse seam probe test settings") + } + + /// A page with one link and one script the probe's middleware changes, + /// and one of each that it leaves alone. + const PROBE_PAGE: &str = concat!( + "Page", + "probe", + "other", + "", + "", + "", + ); + + /// [`PROBE_PAGE`] as a reader receives it from a deployment running + /// `settings` with the probe registered, read `chunk_size` bytes at a + /// time. + fn probe_page(settings: &Settings, chunk_size: usize) -> String { + use trusted_server_core::html_processor::test_support::processed_page; + use trusted_server_core::integrations::IntegrationRegistry; + + let registry = IntegrationRegistry::with_registrations(settings, &[builder()]) + .expect("should build a registry with the probe registered"); + processed_page(settings, ®istry, PROBE_PAGE, chunk_size) + } + + #[test] + fn the_middleware_changes_a_page_where_an_entry_names_it() { + use trusted_server_core::html_processor::test_support::REQUEST_HOST; + + let settings = settings_with_probe( + r#"country = "NZ" + + [[fetch]] + media_type = "text/html" + middleware = ["testing.seam-probe"] + "#, + ); + + // Read whole, and then seven bytes at a time so the script's text + // reaches the handler in pieces. + for chunk_size in [8192, 7] { + let page = probe_page(&settings, chunk_size); + + assert_eq!( + page.matches(&seam_probe_head_marker("NZ")).count(), + 1, + "should mark the head once, with the country the probe is set to: {page}" + ); + assert_eq!( + page.matches(&seam_probe_body_marker()).count(), + 1, + "should mark the body once: {page}" + ); + assert!( + page.contains(&format!("{}", seam_probe_body_marker())), + "should mark immediately after the opening body tag: {page}" + ); + assert!( + page.contains(&format!( + "probe" + )), + "should point the probe's link at its report route: {page}" + ); + assert!( + page.contains(&format!( + "other" + )), + "should leave another link to core, which moves it to the reader's host: {page}" + ); + assert!( + page.contains(""), + "should write the country into the probe's script: {page}" + ); + assert!( + page.contains(""), + "should leave another script as it was: {page}" + ); + } + } + + #[test] + fn the_serve_middleware_marks_a_reader_s_copy_where_an_entry_names_it() { + use trusted_server_core::html_processor::test_support::REQUEST_HOST; + + let settings = settings_with_probe( + r#"country = "NZ" + + [[fetch]] + media_type = "text/html" + middleware = ["testing.seam-probe"] + + [[serve]] + media_type = "text/html" + middleware = ["testing.seam-probe.reader"] + "#, + ); + + let page = probe_page(&settings, 8192); + + let country = page + .find(&seam_probe_head_marker("NZ")) + .expect("should mark the head with the country"); + let reader = page + .find(&seam_probe_reader_marker(REQUEST_HOST)) + .expect("should mark the head with the host the reader asked for"); + let bundle = page + .find("id=\"trustedserver-js\"") + .expect("should carry the script bundle"); + assert!( + country < reader && reader < bundle, + "should write the serve middleware's mark after the fetch one's and ahead of the \ + bundle: {page}" + ); + assert_eq!( + reader_middleware_name(), + "testing.seam-probe.reader", + "should name the serve middleware after the module" + ); + } + + #[test] + fn the_middleware_changes_nothing_where_no_entry_names_it() { + use trusted_server_core::html_processor::test_support::REQUEST_HOST; + + let page = probe_page(&settings_with_probe(r#"country = "NZ""#), 8192); + + assert!( + !page.contains("seam-probe-country") && !page.contains("seam-probe-reader"), + "should not mark the head of a page no entry places it on: {page}" + ); + assert!( + page.contains(&format!( + "probe" + )), + "should leave the probe's link to core: {page}" + ); + assert!( + page.contains(""), + "should leave the probe's script as it was: {page}" + ); + } + + /// The registry rejects a carried module whose declared hash is wrong, so + /// this keeps the fixture's literal honest rather than letting every other + /// test fail with a startup error. + #[test] + fn carried_module_hash_literal_matches_its_source() { + assert_eq!( + hex::encode(Sha256::digest(PROBE_JS.as_bytes())), + PROBE_JS_SHA256, + "PROBE_JS_SHA256 should be the SHA-256 of js/probe.js; if the source is unchanged, check that the file was checked out with LF line endings as .gitattributes requires" + ); + } + + #[test] + fn auction_implementations_are_named_as_the_module() { + assert_eq!( + SEAM_PROBE_ADSERVER.id, + module_name(), + "the ad server is named as the crate's folder below `crates/` names it" + ); + assert_eq!( + SEAM_PROBE_DEMAND.id, + module_name(), + "the demand source is named as the crate's folder below `crates/` names it" + ); + } + + #[test] + fn the_demand_source_refuses_a_setting_it_does_not_read() { + let mut settings = serde_json::Map::new(); + settings.insert("unexpected".to_owned(), serde_json::Value::Bool(true)); + + assert!( + compile_demand(&serde_json::Map::new()).is_ok(), + "an empty table should compile" + ); + let error = compile_demand(&settings) + .err() + .expect("should refuse a setting the source does not read"); + assert!( + error.to_string().contains("`unexpected`"), + "should name the setting: {error}" + ); + } + + #[test] + fn report_route_is_the_documented_path() { + let routes = SeamProbeProxy.routes(); + + assert_eq!(routes.len(), 1, "should register exactly one route"); + assert_eq!( + routes[0].path, SEAM_PROBE_REPORT_PATH, + "should route the documented report path" + ); + } + + #[test] + fn validate_accepts_a_two_letter_country() { + let settings = settings_with_probe("country = \"ZZ\""); + + assert!( + validate(&settings).expect("should accept a two letter country"), + "should report the probe as selected" + ); + } + + #[test] + fn validate_rejects_a_country_that_is_not_two_letters() { + let settings = settings_with_probe("country = \"ZZZ\""); + + let error = validate(&settings).expect_err("should reject a three letter country"); + + assert!( + error.to_string().contains(SEAM_PROBE_COUNTRY_MESSAGE), + "should reject with the probe's own message: {error}" + ); + } + + #[test] + fn validate_reports_not_selected_without_a_module_entry() { + let settings = Settings::from_toml( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "seam-probe-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse settings that do not name the probe"); + + assert!( + !validate(&settings).expect("should validate settings that do not name the probe"), + "should report the probe as not selected" + ); + assert!( + register(&settings) + .expect("should build no registration") + .is_none(), + "should build no registration when the probe is not selected" + ); + } + + #[test] + fn registration_declares_no_geo_module_when_the_block_turns_it_off() { + let settings = settings_with_probe("country = \"ZZ\"\ndeclares_geo = false"); + + let registration = register(&settings) + .expect("should build a registration") + .expect("should be selected"); + + assert!( + registration.geo_module.is_none(), + "should declare no geo module when declares_geo is false" + ); + } + + #[tokio::test] + async fn geo_module_resolves_the_configured_country() { + let module = SeamProbeGeo::new("ZZ".to_string()); + let services = build_probe_services(); + + let geo = module + .lookup(None, &services) + .await + .expect("should resolve location") + .expect("should return geo information"); + + assert_eq!(geo.country, "ZZ", "should resolve the configured country"); + } + + #[test] + fn prepare_request_counts_each_run() { + let settings = settings_with_probe("country = \"ZZ\""); + let mut request = Request::new(EdgeBody::from(Vec::new())); + + prepare_request(&settings, &mut request).expect("should prepare the request"); + prepare_request(&settings, &mut request).expect("should prepare the request again"); + + assert_eq!( + request.extensions().get::(), + Some(&SeamProbePrepared { runs: 2 }), + "should count both runs" + ); + } +} diff --git a/crates/testing/testlight/.gitattributes b/crates/testing/testlight/.gitattributes new file mode 100644 index 0000000000..fad78ffdb5 --- /dev/null +++ b/crates/testing/testlight/.gitattributes @@ -0,0 +1,4 @@ +# The recording is compared byte for byte with the page made from the input +# beside it, so both must arrive with the line endings they were committed +# with on every platform. +src/fixtures/page-change.*.html text eol=lf diff --git a/crates/testing/testlight/Cargo.toml b/crates/testing/testlight/Cargo.toml new file mode 100644 index 0000000000..1bbfe8086a --- /dev/null +++ b/crates/testing/testlight/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "trusted-server-testing-testlight" +description = "The Testlight example integration, which rewrites an auction request body and proxies it to a test endpoint." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +edgezero-core = { workspace = true } +error-stack = { workspace = true } +http = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +trusted-server-core = { workspace = true } +validator = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +trusted-server-core = { workspace = true, features = ["test-utils"] } + +# The visible owner of this crate, the way Prebid.js requires a named +# maintainer of every adapter. Testlight is an example the project keeps for +# testing an auction integration end to end, so the Trusted Server maintainers +# own it. +[package.metadata.maintainers] +owner = "Trusted Server maintainers" +status = "project owned" diff --git a/crates/testing/testlight/src/fixtures/page-change.input.html b/crates/testing/testlight/src/fixtures/page-change.input.html new file mode 100644 index 0000000000..74055212dc --- /dev/null +++ b/crates/testing/testlight/src/fixtures/page-change.input.html @@ -0,0 +1,16 @@ + + + + +Testlight + + + + + + +An article on the origin + +

testlight.js is named here in text, which is left as it is.

+ + diff --git a/crates/testing/testlight/src/fixtures/page-change.recorded.html b/crates/testing/testlight/src/fixtures/page-change.recorded.html new file mode 100644 index 0000000000..386f2effe4 --- /dev/null +++ b/crates/testing/testlight/src/fixtures/page-change.recorded.html @@ -0,0 +1,16 @@ + + + + +Testlight + + + + + + +An article on the origin + +

testlight.js is named here in text, which is left as it is.

+ + diff --git a/crates/testing/testlight/src/fixtures/page-change.settings.toml b/crates/testing/testlight/src/fixtures/page-change.settings.toml new file mode 100644 index 0000000000..aa1a4e0c53 --- /dev/null +++ b/crates/testing/testlight/src/fixtures/page-change.settings.toml @@ -0,0 +1,11 @@ +[auction] +modules = ["testing.testlight"] + +[auction.testing.testlight] +endpoint = "https://example.com/openrtb2/auction" +rewrite_scripts = true +shim_src = "https://edge.example.com/static/testlight.js" + +[[fetch]] +media_type = "text/html" +middleware = ["testing.testlight"] diff --git a/crates/trusted-server-core/src/integrations/testlight.rs b/crates/testing/testlight/src/lib.rs similarity index 50% rename from crates/trusted-server-core/src/integrations/testlight.rs rename to crates/testing/testlight/src/lib.rs index 9e68492bd3..f819feafe6 100644 --- a/crates/trusted-server-core/src/integrations/testlight.rs +++ b/crates/testing/testlight/src/lib.rs @@ -1,3 +1,15 @@ +#![cfg_attr( + test, + allow( + clippy::print_stdout, + clippy::print_stderr, + clippy::panic, + clippy::dbg_macro, + clippy::unwrap_used, + reason = "tests use direct diagnostics and panic-on-failure helpers" + ) +)] +use std::rc::Rc; use std::sync::Arc; use async_trait::async_trait; @@ -9,24 +21,44 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; -use crate::edge_cookie::get_ec_id; -use crate::error::TrustedServerError; -use crate::integrations::{ - AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, - IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, - UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_body_bounded, collect_response_bounded, +use trusted_server_core::edge_cookie::recognized_ec_id; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::{ + AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationEndpoint, IntegrationProxy, + IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_body_bounded, + collect_response_bounded, }; -use crate::platform::RuntimeServices; -use crate::proxy::{ProxyRequestConfig, proxy_request}; -use crate::settings::{IntegrationConfig, Settings}; -use crate::tsjs; +use trusted_server_core::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, MiddlewareContext, + MiddlewarePhase, +}; +use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::proxy::{ProxyRequestConfig, proxy_request}; +use trusted_server_core::settings::{IntegrationConfig, Settings}; +use trusted_server_core::tsjs; const TESTLIGHT_INTEGRATION_ID: &str = "testlight"; +/// The name this module is selected by, in `[auction]`. +pub const MODULE: &str = "testing.testlight"; + +/// The builder a deployment hands to an adapter, which the registry runs when +/// a section selects [`MODULE`]. +#[must_use] +pub fn builder() -> trusted_server_core::integrations::IntegrationBuilder { + trusted_server_core::integrations::IntegrationBuilder::new( + TESTLIGHT_INTEGRATION_ID, + env!("CARGO_PKG_NAME"), + register, + validate, + ) + .with_module_name(MODULE) + .selected_in("auction") +} + #[derive(Debug, Deserialize, Validate)] +#[serde(deny_unknown_fields)] pub struct TestlightConfig { - #[serde(default = "default_enabled")] - pub enabled: bool, #[validate(url)] pub endpoint: String, #[serde(default = "default_timeout_ms")] @@ -39,11 +71,7 @@ pub struct TestlightConfig { pub rewrite_scripts: bool, } -impl IntegrationConfig for TestlightConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} +impl IntegrationConfig for TestlightConfig {} #[derive(Debug, Default, Deserialize, Serialize, Validate)] struct TestlightRequestBody { @@ -133,19 +161,31 @@ impl TestlightIntegration { fn build( settings: &Settings, ) -> Result>, Report> { - let Some(config) = settings.integration_config::(TESTLIGHT_INTEGRATION_ID)? - else { + let Some(config) = settings.module_config::(MODULE)? else { return Ok(None); }; Ok(Some(TestlightIntegration::new(config))) } -/// Register the Testlight integration when enabled. +/// Validates the Testlight configuration for deployment and reports whether +/// a section selects the integration's module. +/// +/// # Errors +/// +/// Returns an error when the Testlight configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) +} + +/// Register the Testlight integration when a section selects it. /// /// # Errors /// -/// Returns an error when the Testlight integration is enabled with invalid +/// Returns an error when the Testlight integration runs with invalid /// configuration. pub fn register( settings: &Settings, @@ -157,26 +197,17 @@ pub fn register( Ok(Some( IntegrationRegistration::builder(TESTLIGHT_INTEGRATION_ID) .with_proxy(integration.clone()) - .with_attribute_rewriter(integration) + .with_middleware(Arc::new(ShimScript(integration))) .build(), )) } -#[async_trait(?Send)] -impl IntegrationProxy for TestlightIntegration { - fn integration_name(&self) -> &'static str { - TESTLIGHT_INTEGRATION_ID - } - - fn routes(&self) -> Vec { - vec![self.post("/auction")] - } - - async fn handle( +impl TestlightIntegration { + async fn route( &self, + req: http::Request, settings: &Settings, services: &RuntimeServices, - req: http::Request, ) -> Result, Report> { let (parts, body) = req.into_parts(); let payload_bytes = @@ -184,13 +215,17 @@ impl IntegrationProxy for TestlightIntegration { .await?; let req = http::Request::from_parts(parts, EdgeBody::empty()); - // Read EC ID from the ts-ec cookie forwarded by the client. - // The registry strips x-ts-ec before dispatching, so only the cookie is available here. - let ec_id = get_ec_id(&req) + // Read the EC ID from the ts-ec cookie forwarded by the client. The + // registry strips x-ts-ec before dispatching, so only the cookie is + // available here. The value goes into the proxied body as `user.id` and + // leaves the edge, so only one the selected module recognizes is + // accepted, and a stateless deployment supplies none. + let ec_id = recognized_ec_id(settings, services, &req) .change_context(Self::error("Failed to read EC ID"))? .ok_or_else(|| { Report::new(Self::error( - "EC ID not found in ts-ec cookie — the client must carry a valid EC cookie", + "No EC ID this deployment's Edge Cookie module recognizes was found \ + in the ts-ec cookie", )) })?; @@ -231,26 +266,30 @@ impl IntegrationProxy for TestlightIntegration { } } -impl IntegrationAttributeRewriter for TestlightIntegration { - fn integration_id(&self) -> &'static str { +#[async_trait(?Send)] +impl IntegrationProxy for TestlightIntegration { + fn integration_name(&self) -> &'static str { TESTLIGHT_INTEGRATION_ID } - fn handles_attribute(&self, attribute: &str) -> bool { - self.config.rewrite_scripts && matches!(attribute, "src" | "href") + fn routes(&self) -> Vec { + vec![self.post("/auction")] } - fn rewrite( + async fn handle( &self, - _attr_name: &str, - attr_value: &str, - _ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if !self.config.rewrite_scripts { - return AttributeRewriteAction::keep(); - } + call: trusted_server_core::module_context::ModuleCall<'_>, + req: http::Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::route)?.await + } +} - let lowered = attr_value.to_ascii_lowercase(); +impl TestlightIntegration { + /// What becomes of a `src` or an `href`, which is pointed at the shim + /// when it is the vendor's script. + fn rewrite_script_address(&self, value: &str) -> AttributeRewriteAction { + let lowered = value.to_ascii_lowercase(); if lowered.contains("testlight.js") { AttributeRewriteAction::replace(self.config.shim_src.clone()) } else { @@ -259,6 +298,34 @@ impl IntegrationAttributeRewriter for TestlightIntegration { } } +/// Loads the shim in place of the vendor's script, on the pages a `[[fetch]]` +/// entry names [`MODULE`] for. It does nothing unless `rewrite_scripts` is +/// set. +struct ShimScript(Arc); + +impl Middleware for ShimScript { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + if !self.0.config.rewrite_scripts { + return MiddlewareAction::pass(); + } + let integration = Arc::clone(&self.0); + let decide: Rc = + Rc::new(move |matched| integration.rewrite_script_address(matched.value)); + MiddlewareAction { + element_handlers: AttributeRewrite::each(&["src", "href"], &decide), + ..MiddlewareAction::pass() + } + } +} + fn default_timeout_ms() -> u32 { 1000 } @@ -270,19 +337,17 @@ fn default_shim_src() -> String { tsjs::tsjs_unified_script_src() } -fn default_enabled() -> bool { - false -} - #[cfg(test)] mod tests { use super::*; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::test_support::tests::{VALID_SYNTHETIC_ID, create_test_settings}; - use crate::tsjs; use http::Method; use serde_json::json; use std::sync::Arc; + use trusted_server_core::platform::test_support::{ + StubHttpClient, build_services_with_http_client, + }; + use trusted_server_core::test_support::tests::{VALID_SYNTHETIC_ID, create_test_settings}; + use trusted_server_core::tsjs; #[test] fn build_requires_config() { @@ -299,7 +364,6 @@ mod tests { fn html_rewriter_replaces_integration_script() { let shim_src = tsjs::tsjs_unified_script_src(); let config = TestlightConfig { - enabled: true, endpoint: "https://example.com/openrtb".to_string(), timeout_ms: 1000, shim_src: shim_src.clone(), @@ -307,16 +371,8 @@ mod tests { }; let integration = TestlightIntegration::new(config); - let ctx = IntegrationAttributeContext { - attribute_name: "src", - element_name: "script", - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - }; - let rewritten = - integration.rewrite("src", "https://cdn.testlight.net/v1/testlight.js", &ctx); + integration.rewrite_script_address("https://cdn.testlight.net/v1/testlight.js"); assert!( matches!( rewritten, @@ -330,36 +386,32 @@ mod tests { fn html_rewriter_is_noop_when_disabled() { let shim_src = tsjs::tsjs_unified_script_src(); let config = TestlightConfig { - enabled: true, endpoint: "https://example.com/openrtb".to_string(), timeout_ms: 1000, shim_src, rewrite_scripts: false, }; let integration = TestlightIntegration::new(config); - let ctx = IntegrationAttributeContext { - attribute_name: "src", - element_name: "script", - request_host: "edge.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - }; + let document_state = trusted_server_core::integrations::IntegrationDocumentState::default(); + let context = trusted_server_core::middleware::test_support::context( + MiddlewarePhase::Fetch, + &document_state, + ); - assert!(matches!( - integration.rewrite("src", "https://cdn.testlight.net/script.js", &ctx), - AttributeRewriteAction::Keep - )); + assert!( + ShimScript(integration).create(&context).is_pass(), + "should leave every page alone when script rewriting is off" + ); } #[test] fn build_uses_settings_integration_block() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - TESTLIGHT_INTEGRATION_ID.to_string(), + .insert_module_config( + "auction", + "testing.testlight", &json!({ - "enabled": true, "endpoint": "https://example.com/bid", "rewrite_scripts": true, }), @@ -425,11 +477,10 @@ mod tests { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, br#"{"ok":true}"#.to_vec()); let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + Arc::clone(&stub) as Arc ); let settings = create_test_settings(); let integration = TestlightIntegration::new(TestlightConfig { - enabled: true, endpoint: "https://example.com/openrtb".to_string(), timeout_ms: 1000, shim_src: tsjs::tsjs_unified_script_src(), @@ -441,12 +492,12 @@ mod tests { .body(EdgeBody::from(br#"{"imp":[{"id":"slot-1"}]}"#.to_vec())) .expect("should build request"); req.headers_mut().insert( - crate::constants::HEADER_X_TS_EC.clone(), + trusted_server_core::constants::HEADER_X_TS_EC.clone(), http::HeaderValue::from_static(VALID_SYNTHETIC_ID), ); let response = integration - .handle(&settings, &services, req) + .route(req, &settings, &services) .await .expect("should proxy Testlight request"); @@ -469,4 +520,183 @@ mod tests { ); }); } + + /// A well-formed identifier carrying a module code no deployment here + /// reads, the shape a partner or another deployment would hand back. + const FOREIGN_CODED_EC_ID: &str = "zz00~someone-elses-identifier"; + + fn testlight_auction_request(ec_id: &str) -> http::Request { + let mut req = http::Request::builder() + .method(Method::POST) + .uri("https://edge.example.com/integrations/testlight/auction") + .body(EdgeBody::from(br#"{"imp":[{"id":"slot-1"}]}"#.to_vec())) + .expect("should build request"); + req.headers_mut().insert( + trusted_server_core::constants::HEADER_X_TS_EC.clone(), + http::HeaderValue::from_str(ec_id).expect("should build EC header value"), + ); + req + } + + fn testlight_integration() -> Arc { + TestlightIntegration::new(TestlightConfig { + endpoint: "https://example.com/openrtb".to_string(), + timeout_ms: 1000, + shim_src: tsjs::tsjs_unified_script_src(), + rewrite_scripts: true, + }) + } + + #[test] + fn handle_refuses_to_egress_an_ec_id_the_module_does_not_recognize() { + futures::executor::block_on(async { + // The identifier ends up in the proxied body as `user.id` and leaves + // the edge, so a value this deployment did not issue must stop here + // rather than be handed to the upstream endpoint. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, br#"{"ok":true}"#.to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let settings = create_test_settings(); + + let refused = testlight_integration() + .route( + testlight_auction_request(FOREIGN_CODED_EC_ID), + &settings, + &services, + ) + .await + .expect_err("a foreign module code should not be proxied upstream"); + drop(refused); + + assert!( + stub.recorded_backend_names().is_empty(), + "no upstream call should be made with an unrecognized identifier" + ); + }); + } + + #[test] + fn handle_refuses_to_egress_any_ec_id_in_a_stateless_deployment() { + futures::executor::block_on(async { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, br#"{"ok":true}"#.to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + // The value is one the built-in module would recognize, so only + // the absence of a selected module can withhold it. + let mut settings = create_test_settings(); + settings.ec.module = None; + settings.ec.module_blocks.clear(); + + let refused = testlight_integration() + .route( + testlight_auction_request(VALID_SYNTHETIC_ID), + &settings, + &services, + ) + .await + .expect_err("a deployment that creates no identifier should proxy none"); + drop(refused); + + assert!( + stub.recorded_backend_names().is_empty(), + "no upstream call should be made without a recognized identifier" + ); + }); + } + + #[test] + fn module_constant_is_the_crate_folder() { + assert_eq!( + super::MODULE, + trusted_server_core::module_name!(), + "should be named by the folder this crate lives in" + ); + } + + #[test] + fn a_served_page_loads_the_shim_in_place_of_the_vendor_script() { + use std::io::Cursor; + + use trusted_server_core::html_processor::HtmlProcessorConfig; + use trusted_server_core::html_processor::test_support::{ + create_page_processor, place_on_every_page, + }; + use trusted_server_core::integrations::IntegrationRegistry; + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; + + let html = r#" + + "#; + + let mut settings = Settings::default(); + let shim_src = "https://edge.example.com/static/testlight.js".to_owned(); + settings + .insert_module_config( + "auction", + MODULE, + &json!({ + "endpoint": "https://example.com/openrtb2/auction", + "rewrite_scripts": true, + "shim_src": shim_src, + }), + ) + .expect("should insert testlight config"); + place_on_every_page(&mut settings, MiddlewarePhase::Fetch, &[MODULE]); + + let registry = IntegrationRegistry::with_registrations(&settings, &[builder()]) + .expect("should create registry"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "test.example.com", + "https", + ); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size: 8192, + }, + create_page_processor(&settings, ®istry, config), + ); + + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should process the page"); + + let processed = String::from_utf8_lossy(&output); + assert!( + processed.contains(&shim_src), + "the shim should replace the vendor's script reference" + ); + assert!( + !processed.contains("cdn.testlight.com"), + "the vendor's own URL should be gone from the page" + ); + } + + /// The page a reader receives with this module running, kept as a file + /// so that changing how the page change is made can be shown to leave + /// the page as it was. + #[test] + fn the_page_a_reader_receives_is_the_recorded_one() { + trusted_server_core::html_processor::test_support::assert_page_is_recorded( + include_str!("fixtures/page-change.settings.toml"), + &[builder()], + include_str!("fixtures/page-change.input.html"), + include_str!("fixtures/page-change.recorded.html"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/fixtures/page-change.recorded.html" + ), + ); + } } diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index 94ad02170c..eacd4c24d9 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -29,11 +29,21 @@ reqwest = { workspace = true } simple_logger = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } trusted-server-core = { workspace = true } +trusted-server-modules = { workspace = true } +trusted-server-permission-signal-gpc = { workspace = true } +trusted-server-permission-signal-gpp = { workspace = true } +trusted-server-permission-signal-mtm = { workspace = true } +trusted-server-permission-signal-tcf = { workspace = true } +trusted-server-permission-signal-us-privacy = { workspace = true } [dev-dependencies] trusted-server-core = { workspace = true, features = ["test-utils"] } axum = { workspace = true } base64 = { workspace = true } +serde_json = { workspace = true } temp-env = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tower = { workspace = true, features = ["util"] } +# Test fixture only. It proves the integration seam from outside core and must +# never reach a production build, so it stays a dev-dependency. +trusted-server-integration-seam-probe = { path = "../testing/seam-probe" } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index caba714d68..26a46c7d6f 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -9,37 +9,44 @@ use edgezero_core::http::{ }; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::attestation::{PlatformIdentity, handle_prefix, prefix_routes}; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ - AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::closed_paths::closed_path_response; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{ - admin_ec_lookup_not_supported, deny_admin_diagnostic_fallback, handle_admin_eids_lookup, -}; -use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::ec::module::ensure_module_available; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; -use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; +use trusted_server_core::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH, handle_config}; +use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; +use trusted_server_core::inspect::permissions::{ + PERMISSIONS_JSON_PATH, PERMISSIONS_PAGE_PATH, handle_permissions, +}; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, +}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, buffer_publisher_response_async, - handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; use trusted_server_core::settings::Settings; use trusted_server_core::settings_data::{ - default_config_key, default_config_store_name, get_settings_from_config_store, + default_config_key, default_config_store_name, get_settings_from_config_store_with, }; use trusted_server_core::platform::RuntimeServices; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, SanitizeRequestMiddleware}; +use crate::middleware::{FinalizeResponseMiddleware, SanitizeRequestMiddleware}; use crate::platform::{AxumPlatformConfigStore, AxumPlatformSecretStore, build_runtime_services}; // --------------------------------------------------------------------------- @@ -47,11 +54,47 @@ use crate::platform::{AxumPlatformConfigStore, AxumPlatformSecretStore, build_ru // --------------------------------------------------------------------------- /// Application state built once at startup and shared across all requests. +/// +/// Built once here, unlike on the other three adapters, because `main` +/// calls [`TrustedServerApp::routes`] before serving, so the router holds +/// one state for the process's lifetime. pub struct AppState { settings: Arc, orchestrator: Arc, registry: Arc, + /// The permission signal modules `[permission-signal] modules` selects + /// from the scheme crates this adapter links, in the order they run. + /// Selected once here so a name no crate answers to fails startup rather + /// than the first request, and handed to every request's services. + permission_signal_modules: + Arc<[Arc]>, + /// Services a caller supplied for every request, rather than services built + /// from the request context. `None` in a deployment. services: Option, + /// The loading caches the `[cache.]` tables select from the linked + /// builders, taken when this state is built and handed to every request's + /// services. + loading_caches: Arc, +} + +/// The permission signal modules this adapter links, in the order they run +/// when configuration names none. Global Privacy Control is first because it +/// is a browser setting with no interface of its own, and the three that +/// carry a choice someone made through an interface follow, so an answer +/// given at a prompt amends the header the visitor arrived with. +/// +/// Core supplies no module of its own, so this is where a deployment's +/// schemes are decided. A scheme is added by linking its crate here, and a +/// scheme core has never heard of plugs in the same way. +fn shipped_signal_modules() +-> Vec> { + vec![ + Arc::new(trusted_server_permission_signal_gpc::GpcModule::new()), + Arc::new(trusted_server_permission_signal_gpp::GppSaleOptOutModule::new()), + Arc::new(trusted_server_permission_signal_us_privacy::UsPrivacyModule::new()), + Arc::new(trusted_server_permission_signal_tcf::TcfModule::new()), + Arc::new(trusted_server_permission_signal_mtm::MtmModule::new()), + ] } /// Build the application state, loading settings and constructing all per-application components. @@ -63,12 +106,15 @@ pub struct AppState { fn build_state() -> Result, Report> { let store_name = default_config_store_name(); let config_key = default_config_key(); - let settings = get_settings_from_config_store( + // The settings are validated as they load, against the same builders the + // state is built with. + let settings = get_settings_from_config_store_with( &AxumPlatformConfigStore, &AxumPlatformSecretStore, &store_name, &config_key, &trusted_server_core::settings_data::default_secret_store_name(), + &trusted_server_modules::builders(), )?; build_state_with_settings(settings) } @@ -77,36 +123,145 @@ fn build_state() -> Result, Report> { /// /// # Errors /// -/// Returns an error when the auction orchestrator or the integration -/// registry fail to initialise. +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, or when the auction orchestrator or the integration registry +/// fail to initialize. fn build_state_with_settings( settings: Settings, ) -> Result, Report> { - build_state_with_services(settings, None) + build_state_with_registrations(settings, &[]) } +/// Build the application state from explicit settings, composing the modules +/// a stock build ships with the externally supplied builders in +/// `integrations`. +/// +/// A deployment that ships a vendor crate calls this to add that crate's +/// integration builder without the adapter naming the vendor. Auction +/// providers come from the compiled auction plan, as they do without any +/// external builders. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, when the auction plan does not compile or cannot run on this +/// adapter, or when the auction orchestrator or the integration registry fail +/// to initialize, which includes two builders claiming the same integration +/// id. +pub fn build_state_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], +) -> Result, Report> { + build_state_with_registrations_and_services(settings, integrations, None) +} + +/// Build the application state with the services every request will use, +/// rather than services built per request from the request context. fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { - let plan = Arc::new(compile_auction_plan(&settings)?); + build_state_with_registrations_and_services(settings, &[], services) +} + +fn build_state_with_registrations_and_services( + settings: Settings, + integrations: &[IntegrationBuilder], + services: Option, +) -> Result, Report> { + // The modules a stock build ships come first and the deployment's own + // follow, which is the order their hooks run in. + let integrations = trusted_server_modules::builders_with(integrations); + let integrations = integrations.as_slice(); + + // The plan is compiled with the integrations this adapter was given, so an + // `[ad-server]` or `[demand]` name one of their builders supplies resolves + // here. Compiling without them would drop the implementation and report the + // name as one no builder registers. + let plan = Arc::new(compile_auction_plan_with(&settings, integrations)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; - let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; - let registry = IntegrationRegistry::with_plan(&settings, plan)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan))?; + let registry = IntegrationRegistry::with_plan_and_registrations(&settings, plan, integrations)? + .with_platform(PlatformIdentity::named("axum")); + + // Composition root: reject a module selection this adapter can never + // supply, once, before any request is served. The registry is built first + // because a module can supply the Edge Cookie module the selector names, + // and `services_for_request` hands that module to every request, so the + // check is given the module the request path sees. A caller supplying its + // own `RuntimeServices` may have resolved one already, and that one comes + // first because it is what `EcContext` will see. + // + // This adapter checks rather than keeps what the check resolved, unlike the + // Fastly, Cloudflare and Spin adapters, because it is a long-lived process + // whose application state is built once at start-up while theirs is rebuilt + // for every request, so `EcContext` resolves the selection on every + // request. It supplies no host signals, so the host-signals argument is + // `None`. + ensure_module_available( + &settings.ec, + None, + services + .as_ref() + .and_then(RuntimeServices::resolved_ec_module) + .or_else(|| registry.ec_module()), + )?; + let permission_signal_modules = + trusted_server_core::permission_signal::build_permission_signal_modules( + &settings, + &shipped_signal_modules(), + )?; + let loading_caches = + trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), + permission_signal_modules, services, + loading_caches, })) } +// --------------------------------------------------------------------------- +// Per-request RuntimeServices +// --------------------------------------------------------------------------- + impl AppState { + /// Build per-request [`RuntimeServices`], applying the module-supplied geo, + /// Edge Cookie and device modules selected by `[geo]`, `[ec]` and + /// `[device] module`. + /// + /// Unset and `"none"` both resolve nothing for geo, so no client IP reaches + /// a host geo service. `"platform"` opts in to this adapter's own lookup, + /// and any other key names an integration module that declares a geo + /// module. Identity and device are applied the same way when a module + /// supplies them. fn services_for_request(&self, ctx: &RequestContext) -> RuntimeServices { - self.services - .clone() - .unwrap_or_else(|| build_runtime_services(ctx)) + let mut services = self.services.clone().unwrap_or_else(|| { + build_runtime_services(ctx, &self.settings, &self.permission_signal_modules) + .with_loading_caches(Arc::clone(&self.loading_caches)) + }); + if let Some(module) = self.registry.geo_module() { + services = services.with_geo(module); + } + if let Some(module) = self.registry.ec_module() { + services = services.with_ec_module(module); + } + if let Some(module) = self.registry.device_module() { + services = services.with_device_module(module); + } + // The publisher this request resolved to names every key in a shared + // store and every name the browser holds. + let prefixes = self.registry.prefixes( + &self.settings, + trusted_server_core::module_context::ModuleRequest::of( + ctx.request(), + services.client_info(), + ), + ); + services.with_prefixes(prefixes) } } @@ -129,23 +284,6 @@ pub(crate) fn http_error(report: &Report) -> Response { response } -/// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` -/// aliases on the Axum dev server. -/// -/// These non-`/_ts` aliases are not matched by the `^/_ts/admin` basic-auth -/// handler, so they fail closed locally rather than fall through to the -/// publisher fallback — which would forward the caller's `Authorization` header -/// and key-management payload to the origin, leaking admin credentials. -fn legacy_admin_alias_denied() -> Response { - let mut response = Response::new(edgezero_core::body::Body::from("Not found\n")); - *response.status_mut() = StatusCode::NOT_FOUND; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - // --------------------------------------------------------------------------- // Shared handler executor // --------------------------------------------------------------------------- @@ -161,10 +299,11 @@ where { let services = state.services_for_request(&ctx); let mut req = ctx.into_request(); - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + // The single preparation point for the adapter. Every route in the table + // except `/health` is registered to `named_route_handler` or + // `fallback_handler`, and both wrap this function, so each request has its + // modules' preparers run exactly once and always before routing. + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); } Ok(handler(state, services, req) @@ -179,41 +318,54 @@ where /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, /// `/_ts/page-bids`, and the publisher fallback). /// -/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction -/// Unknown, which fails the auction consent gate closed even for consented -/// users. Geo comes from the platform (a no-op on the local Axum dev server, so -/// jurisdiction stays Unknown there unless the request carries TCF consent). A -/// malformed consent string is logged and falls back to the default -/// (fail-closed) context rather than being silently swallowed. -fn build_ec_context(state: &AppState, services: &RuntimeServices, req: &Request) -> EcContext { - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - EcContext::read_from_request_with_geo(&state.settings, req, services, geo_info.as_ref()) - .unwrap_or_else(|e| { - log::warn!("EC context read failed: {e:?}"); - EcContext::default() - }) +/// The geo lookup runs inside +/// [`EcContext::read_from_request_resolving_geo`], so every adapter reports the +/// same distinction: no location falls back to the top of the +/// `permissions.yaml` rules tree, while a failed lookup resolves every +/// permission at the requires-signal floor and is logged at error level. +/// The platform geo is a no-op on the local Axum dev server, so a request there +/// resolves at that top node unless it carries a signal. +/// +/// Mirrors the Fastly entry point, which keeps the report and answers with an +/// error response: when the Edge Cookie context cannot be read the request +/// fails rather than continuing with `EcContext::default()`, which would serve +/// every request with no identity. A malformed cookie value, a bad consent +/// string and a failed geo lookup do not reach this error path at all, so +/// failing here does not fail requests for ordinary parse problems. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this request, or when the request's `Cookie` header is not valid UTF-8. +async fn build_ec_context( + state: &AppState, + services: &RuntimeServices, + req: &Request, +) -> Result> { + EcContext::read_from_request_resolving_geo(&state.settings, req, services).await } // --------------------------------------------------------------------------- // Fallback dispatcher (tsjs / integration proxy / publisher) // --------------------------------------------------------------------------- +/// Dispatches the fallback routes: tsjs assets, integration proxy routes, and +/// the publisher origin. +/// +/// The request arrives already prepared. Every route that reaches here is +/// registered to [`fallback_handler`], which runs the request through +/// [`execute_handler`], and [`execute_handler`] calls +/// `registry.prepare_request` before it calls this function, so there is +/// nothing left to prepare here. async fn dispatch_fallback( state: &AppState, services: &RuntimeServices, - mut req: Request, + req: Request, ) -> Result> { - if let Some(response) = deny_admin_diagnostic_fallback(&req) { + if let Some(response) = closed_path_response(&req) { return Ok(response); } - trusted_server_core::integrations::gpt_diagnostics::prepare_request(&state.settings, &mut req)?; let path = req.uri().path().to_string(); let method = req.method().clone(); @@ -222,7 +374,9 @@ async fn dispatch_fallback( } if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); + // A module route is handed what was resolved for its request, the + // permissions among them, as the publisher path is. + let mut ec_context = build_ec_context(state, services, &req).await?; return state .registry .handle_proxy(ProxyDispatchInput { @@ -244,14 +398,17 @@ async fn dispatch_fallback( // Run the server-side auction with the configured creative-opportunity // slots; `handle_publisher_request` matches them against the request path. - let mut ec_context = build_ec_context(state, services, &req); + let mut ec_context = build_ec_context(state, services, &req).await?; let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, }; let publisher_response = handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, services, None, &mut ec_context, @@ -294,13 +451,9 @@ fn fallback_handler( enum NamedRouteHandler { TrustedServerDiscovery, VerifySignature, - AdminNotSupported, - CachePurgeNotSupported, - AdminEcNotSupported, - AdminEidsLookup, - /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never - /// reach the publisher fallback (which would leak admin credentials). - LegacyAdminDenied, + Permissions, + Config, + Data, Auction, PageBids, FirstPartyProxy, @@ -315,17 +468,7 @@ struct NamedRoute { handler: NamedRouteHandler, } -const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ - Method::GET, - Method::POST, - Method::HEAD, - Method::OPTIONS, - Method::PUT, - Method::PATCH, - Method::DELETE, -]; - -fn named_routes() -> [NamedRoute; 17] { +fn named_routes() -> [NamedRoute; 14] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -337,62 +480,34 @@ fn named_routes() -> [NamedRoute; 17] { primary_methods: &[Method::POST], handler: NamedRouteHandler::VerifySignature, }, - // Canonical admin key routes. These match `Settings::ADMIN_ENDPOINTS` - // and the production basic-auth handler regex (`^/_ts/admin`), so they - // are auth-gated under a production-shaped config. + // What the deployment decided for the asking request, shown to anyone. NamedRoute { - path: "/_ts/admin/keys/rotate", - primary_methods: &[Method::POST], - handler: NamedRouteHandler::AdminNotSupported, - }, - NamedRoute { - path: "/_ts/admin/keys/deactivate", - primary_methods: &[Method::POST], - handler: NamedRouteHandler::AdminNotSupported, - }, - // Every method, for the same reason as the Fastly adapter: a method this route - // does not claim falls through to the publisher with the caller's `Authorization` - // header still attached. - NamedRoute { - path: "/_ts/admin/cache/purge", - primary_methods: LEGACY_ADMIN_DENY_METHODS, - handler: NamedRouteHandler::CachePurgeNotSupported, - }, - // Admin EC lookup routes. Registered explicitly (like the key routes - // above) so they never fall through to the publisher fallback, and - // they match `Settings::ADMIN_ENDPOINTS` for auth coverage. - NamedRoute { - path: "/_ts/admin/ec", + path: PERMISSIONS_PAGE_PATH, primary_methods: &[Method::GET], - handler: NamedRouteHandler::AdminEcNotSupported, + handler: NamedRouteHandler::Permissions, }, NamedRoute { - path: "/_ts/admin/ec/{id}", + path: PERMISSIONS_JSON_PATH, primary_methods: &[Method::GET], - handler: NamedRouteHandler::AdminEcNotSupported, + handler: NamedRouteHandler::Permissions, }, - // Admin EIDs echo: pure request inspection (no KV), so the dev - // server serves the real handler. + // The settings the deployment is running, masked, shown to anyone. NamedRoute { - path: "/_ts/admin/eids", + path: CONFIG_PAGE_PATH, primary_methods: &[Method::GET], - handler: NamedRouteHandler::AdminEidsLookup, + handler: NamedRouteHandler::Config, }, - // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with - // a 404, matching the Fastly and Cloudflare adapters: the production - // basic-auth handler regex `^/_ts/admin` does not match them, and letting - // any publisher-fallback method fall through would forward the caller's - // `Authorization` header and key-management payload to the origin, - // leaking admin credentials. NamedRoute { - path: "/admin/keys/rotate", - primary_methods: LEGACY_ADMIN_DENY_METHODS, - handler: NamedRouteHandler::LegacyAdminDenied, + path: CONFIG_JSON_PATH, + primary_methods: &[Method::GET], + handler: NamedRouteHandler::Config, }, + // What is held against the request's own Edge Cookie. This adapter + // keeps no identity graph, so the page says nothing is held. NamedRoute { - path: "/admin/keys/deactivate", - primary_methods: LEGACY_ADMIN_DENY_METHODS, - handler: NamedRouteHandler::LegacyAdminDenied, + path: DATA_PAGE_PATH, + primary_methods: &[Method::GET], + handler: NamedRouteHandler::Data, }, NamedRoute { path: "/auction", @@ -458,54 +573,18 @@ fn named_route_handler( NamedRouteHandler::VerifySignature => { handle_verify_signature(&state.settings, &services, req) } - NamedRouteHandler::CachePurgeNotSupported => { - // The Axum dev server has no template cache to purge. 501 rather - // than a fallthrough 404, so a CMS webhook can tell "not supported - // here" from "endpoint does not exist". - let body = edgezero_core::body::Body::from( - "Template cache purge is not supported on the Axum dev server.\n\ - Use the Fastly adapter (via Viceroy or deployed) to purge.\n", - ); - let mut resp = Response::new(body); - *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - Ok(resp) + NamedRouteHandler::Permissions => { + handle_permissions(&state.settings, &services, &req).await } - NamedRouteHandler::AdminNotSupported => { - // Config/secret-store writes are backed by read-only env vars on the - // Axum dev server. Returning 501 is clearer than failing on the first - // store write. - let body = edgezero_core::body::Body::from( - "Admin key management is not supported on the Axum dev server.\n\ - Use the Fastly adapter (via Viceroy or deployed) to rotate or deactivate keys.\n", - ); - let mut resp = Response::new(body); - *resp.status_mut() = StatusCode::NOT_IMPLEMENTED; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - Ok(resp) + NamedRouteHandler::Config => Ok(handle_config(&state.settings, &req)), + NamedRouteHandler::Data => { + handle_data(None, None, services.public_prefix(), &req) } - NamedRouteHandler::AdminEcNotSupported => { - // The EC identity graph is Fastly KV backed; the Axum - // dev server has no store to read. - Ok(admin_ec_lookup_not_supported()) - } - NamedRouteHandler::AdminEidsLookup => { - let partner_registry = - PartnerRegistry::from_config(&state.settings.ec.partners)?; - handle_admin_eids_lookup(&partner_registry, &req) - } - NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { // Build the geo-aware EC context so the auction consent // gate sees the caller's jurisdiction — `EcContext::default()` // fails it closed for consented users. - let mut ec_context = build_ec_context(&state, &services, &req); + let mut ec_context = build_ec_context(&state, &services, &req).await?; handle_auction( &state.settings, &state.orchestrator, @@ -524,7 +603,7 @@ fn named_route_handler( if req.method() == Method::OPTIONS { Ok(page_bids_preflight_denied()) } else { - let mut ec_context = build_ec_context(&state, &services, &req); + let mut ec_context = build_ec_context(&state, &services, &req).await?; let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), @@ -646,6 +725,28 @@ impl TrustedServerApp { Ok(build_router(&state)) } + /// Build the full application router from explicit settings, composing the + /// built-in integrations with the externally supplied builders in + /// `integrations`. + /// + /// The route table is the one [`TrustedServerApp::routes_with_settings`] + /// builds, so a composed deployment routes exactly as the plain one does. + /// + /// # Errors + /// + /// Returns an error when the selected Edge Cookie module cannot be built + /// for this adapter, when the auction plan does not compile or cannot run + /// on this adapter, or when the auction orchestrator or the integration + /// registry fail to initialize, which includes two builders claiming the + /// same integration id. + pub fn routes_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], + ) -> Result> { + let state = build_state_with_registrations(settings, integrations)?; + Ok(build_router(&state)) + } + /// Build the full router with explicit settings and runtime services. /// /// Each request receives a clone of the supplied services, allowing callers @@ -671,6 +772,23 @@ impl TrustedServerApp { } } +/// Answers the attestation pages without the request preparers, because +/// the pages are read only. +fn attestation_route_handler( + state: Arc, +) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { + move |ctx: RequestContext| { + let state = Arc::clone(&state); + Box::pin(async move { + Ok(handle_prefix( + &state.settings, + state.registry.platform(), + &ctx.into_request(), + )) + }) + } +} + fn build_router(state: &Arc) -> RouterService { let fallback = fallback_handler(Arc::clone(state)); @@ -680,8 +798,7 @@ fn build_router(state: &Arc) -> RouterService { // any middleware registered ahead of it would observe the // shared-secret authentication header. .middleware(SanitizeRequestMiddleware::new(Arc::clone(&state.settings))) - .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) - .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); + .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))); router = router.route("/health", Method::GET, |_ctx: RequestContext| async { Ok::( @@ -708,6 +825,21 @@ fn build_router(state: &Arc) -> RouterService { } } + // Routes chosen by the settings, so not in the fixed table. + let routes = prefix_routes(&state.settings); + if !routes.is_empty() { + let attestation = attestation_route_handler(Arc::clone(state)); + for route in &routes { + for method in publisher_fallback_methods() { + if matches!(method, Method::GET | Method::HEAD) || route.every_method { + router = router.route(&route.path, method, attestation.clone()); + } else { + router = router.route(&route.path, method, fallback.clone()); + } + } + } + } + for method in publisher_fallback_methods() { router = router.route("/", method.clone(), fallback.clone()); router = router.route("/{*rest}", method, fallback.clone()); @@ -715,3 +847,126 @@ fn build_router(state: &Arc) -> RouterService { router.build() } + +#[cfg(test)] +mod tests { + use edgezero_core::http::request_builder; + use edgezero_core::params::PathParams; + // These fixtures supply no integration builders, so they compile the plan + // without them rather than through the module-aware compiler the adapter + // itself uses. + use trusted_server_core::auction::compile_auction_plan; + + use super::*; + + /// Settings selecting a vendor Edge Cookie module this adapter does not + /// inject, with the `[ec.acme]` block that module's settings live in. + /// `acme` is a fictional vendor key. + const UNINJECTED_MODULE_TOML: &str = r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + module = "acme" + + [ec.acme] + endpoint = "https://ec.acme.example.com" + + # An Edge Cookie module is configured, so single-jurisdiction + # operation is acknowledged because no geo module is selected. + [geo] + assume_single_jurisdiction = true + "#; + + /// Every request's services carry the prefixes of the publisher its + /// settings describe, which start every shared store key and every name + /// the browser holds. + #[test] + fn a_requests_services_carry_the_publishers_prefixes() { + let settings = Settings::from_toml( + r#" + [publisher] + domain = "www.publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret-32-bytes-ok" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse the settings"); + let state = build_state_with_settings(settings).expect("should build the state"); + let request = request_builder() + .method("GET") + .uri("https://www.publisher.example/") + .body(edgezero_core::body::Body::empty()) + .expect("should build the request"); + let ctx = RequestContext::new(request, PathParams::default()); + + let services = state.services_for_request(&ctx); + + assert_eq!( + services.server_key_prefix().as_str(), + "ts.publisher.example" + ); + assert_eq!(services.public_prefix().as_str(), "ts"); + } + + /// Builds application state directly, bypassing the composition root's + /// startup check, so the per-request behavior can be exercised with a + /// selection the adapter cannot supply. + fn state_with_uninjected_module() -> AppState { + let settings = Settings::from_toml(UNINJECTED_MODULE_TOML) + .expect("should parse settings selecting an uninjected module"); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); + let orchestrator = + build_orchestrator_with_plan(Arc::clone(&plan)).expect("should build orchestrator"); + let registry = + IntegrationRegistry::with_plan(&settings, plan).expect("should build registry"); + AppState { + settings: Arc::new(settings), + orchestrator: Arc::new(orchestrator), + registry: Arc::new(registry), + // These tests exercise the Edge Cookie module path, and a + // request with no signal module resolves at the place baseline. + permission_signal_modules: Arc::default(), + // This test drives the per-request path, which builds its services + // from the request context. + services: None, + loading_caches: Arc::default(), + } + } + + /// The per-request Edge Cookie read must return its error rather than a + /// default context. + /// + /// Continuing with `EcContext::default()` would serve every request with + /// no identity when the selected module cannot be built. The call sites + /// propagate the error to `http_error`, matching the Fastly adapter. + #[tokio::test] + async fn build_ec_context_fails_when_the_selected_module_is_unavailable() { + let state = state_with_uninjected_module(); + let req = request_builder() + .method("POST") + .uri("https://test-publisher.example.com/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build test request"); + let ctx = RequestContext::new(req, PathParams::default()); + let services = + build_runtime_services(&ctx, &state.settings, &state.permission_signal_modules); + let req = ctx.into_request(); + + let error = build_ec_context(&state, &services, &req) + .await + .expect_err("an unavailable Edge Cookie module must fail the request"); + + assert!( + error.to_string().contains("acme"), + "the error should name the selected module, got: {error}" + ); + } +} diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index f099bc15b2..afeb310356 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -5,9 +5,10 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; -use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; -use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; +use trusted_server_core::http_util::{ + ROUTE_NAMESPACE, sanitize_trusted_client_ip_headers, strip_deployment_headers, +}; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -53,13 +54,11 @@ impl Middleware for SanitizeRequestMiddleware { /// Response-finalization middleware: injects all standard TS response headers. /// /// Geo lookup is unavailable in the Axum dev server — `X-Geo-Info-Available: false` -/// is always emitted. `X-TS-Version` carries the compiled-in git version. The -/// Fastly-specific headers (`X-TS-Fastly-Version`, `X-TS-ENV`) are skipped -/// because the corresponding env vars are not set in a local dev context. +/// is always emitted. The Fastly adapter's `X-TS-ENV` is skipped because its +/// env var is not set in a local dev context. /// -/// Registered directly inside [`SanitizeRequestMiddleware`] and ahead of -/// [`AuthMiddleware`] so that every outgoing response — including auth-rejected -/// ones — carries a consistent set of headers. +/// Registered directly inside [`SanitizeRequestMiddleware`], so that every +/// outgoing response carries a consistent set of headers. pub struct FinalizeResponseMiddleware { settings: Arc, } @@ -75,50 +74,15 @@ impl FinalizeResponseMiddleware { #[async_trait(?Send)] impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { + let path = ctx.request().uri().path().to_owned(); let mut response = next.run(ctx).await?; + // Before finalizing, so a header finalizing writes is kept. + strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, &mut response); Ok(response) } } -// --------------------------------------------------------------------------- -// AuthMiddleware -// --------------------------------------------------------------------------- - -/// Inner middleware: enforces basic-auth before the handler runs. -/// -/// - `Ok(Some(response))` from [`enforce_basic_auth`] → auth failed; return the -/// challenge response (bubbles through [`FinalizeResponseMiddleware`] for header injection). -/// - `Ok(None)` → no auth required or credentials accepted; continue the chain. -/// - `Err(report)` → internal error; log and convert to a 500 HTTP response. -pub struct AuthMiddleware { - settings: Arc, -} - -impl AuthMiddleware { - /// Creates a new [`AuthMiddleware`] with the given settings. - #[must_use] - pub fn new(settings: Arc) -> Self { - Self { settings } - } -} - -#[async_trait(?Send)] -impl Middleware for AuthMiddleware { - async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { - Ok(Some(response)) => return Ok(response), - Ok(None) => {} - Err(report) => { - log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); - } - } - - next.run(ctx).await - } -} - // --------------------------------------------------------------------------- // apply_finalize_headers — extracted for unit testing // --------------------------------------------------------------------------- @@ -126,8 +90,7 @@ impl Middleware for AuthMiddleware { /// Applies standard Trusted Server response headers to the given response. /// /// Unlike the Fastly variant, geo is always unavailable so `X-Geo-Info-Available: false` -/// is unconditionally emitted, followed by the compiled-in `X-TS-Version`. -/// Fastly-specific headers are omitted. +/// is unconditionally emitted. Fastly-specific headers are omitted. /// Operator-configured `settings.response_headers` are applied last (with the /// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Response) { @@ -136,13 +99,15 @@ pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Respons HeaderValue::from_static("false"), ); - trusted_server_core::version_header::apply_git_version_header(response); - // Cookie-bearing responses stay private to shared caches and operator // headers cannot re-enable caching for uncacheable per-user payloads. trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( settings, response, ); + + // Last, so that nothing an origin or an operator's header set can loosen + // a site taken out of search. + trusted_server_core::robots_txt::apply_response_robots_tag(settings, response); } // --------------------------------------------------------------------------- @@ -185,11 +150,6 @@ mod tests { fn settings_with_response_headers(headers: Vec<(&str, &str)>) -> Settings { let mut s = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -197,7 +157,13 @@ mod tests { proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should load test settings"); @@ -292,17 +258,58 @@ mod tests { ); } + fn ctx_for(path: &str) -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri(path) + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } + + /// A response as an origin behind a cache sends it, naming the node that + /// answered and how the cache treated the request. + fn response_naming_its_node() -> Response { + response_builder() + .header("x-served-by", "cache-node-1") + .header("x-cache", "HIT") + .header("x-cache-hits", "3") + .body(Body::empty()) + .expect("should build test response") + } + + /// What the finalize middleware makes of that response for a request to + /// `path`. + fn finalized(path: &str, settings: Settings) -> Response { + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let handler = Arc::new(|_ctx: RequestContext| async move { + Ok::(response_naming_its_node()) + }); + + block_on(middleware.handle(ctx_for(path), Next::new(&[], &*handler))) + .expect("should run middleware") + } + #[test] - fn emits_git_version_header() { - let mut response = empty_response(); - apply_finalize_headers(&settings_with_response_headers(vec![]), &mut response); + fn finalize_middleware_strips_deployment_headers_before_it_writes_the_operators() { + let response = finalized( + "/articles/an-article", + settings_with_response_headers(vec![("X-Cache", "configured")]), + ); + + for name in ["x-served-by", "x-cache-hits"] { + assert!( + !response.headers().contains_key(name), + "should strip {name} from a publisher's page" + ); + } assert_eq!( response .headers() - .get("x-ts-version") + .get("x-cache") .and_then(|v| v.to_str().ok()), - trusted_server_core::constants::TS_GIT_VERSION, - "should report the compiled-in git version as x-ts-version" + Some("configured"), + "should carry the operator's header and not the origin's" ); } } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index e2713f47f2..2c1491eefb 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -180,8 +180,13 @@ impl PlatformBackend for AxumPlatformBackend { /// No-op geo implementation — geographic lookup is unavailable in local development. pub struct AxumPlatformGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for AxumPlatformGeo { - fn lookup(&self, _client_ip: Option) -> Result, Report> { + async fn lookup( + &self, + _client_ip: Option, + _services: &trusted_server_core::platform::RuntimeServices, + ) -> Result, Report> { Ok(None) } } @@ -527,7 +532,13 @@ impl PlatformHttpClient for AxumPlatformHttpClient { /// The generic runtime KV slot uses /// [`trusted_server_core::platform::UnavailableKvStore`]. A `warn` log is /// emitted once per process. -pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { +pub fn build_runtime_services( + ctx: &edgezero_core::context::RequestContext, + settings: &trusted_server_core::settings::Settings, + permission_signal_modules: &Arc< + [Arc], + >, +) -> RuntimeServices { static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); KV_WARNED.get_or_init(|| { log::warn!("Axum dev server: generic runtime KV is unavailable (UnavailableKvStore)."); @@ -569,9 +580,16 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R // API-route integration flow by reusing a poisoned connection after a // truncated POST. Revisit pooling if profiling shows allocation cost. .http_client(Arc::new(AxumPlatformHttpClient::new())) - .geo(Arc::clone(GEO.get_or_init(|| { - Arc::new(AxumPlatformGeo) as Arc - }))) + // Route through the [geo] module selector like the Fastly adapter, + // so the selector behaves the same on every adapter. + .geo(trusted_server_core::platform::build_geo_module( + settings, + Arc::clone(GEO.get_or_init(|| Arc::new(AxumPlatformGeo) as Arc)), + )) + // The signal modules were selected once at startup from the scheme + // crates this adapter links, so every request asks exactly the ones + // configuration named, in that order. + .permission_signal_modules(Arc::clone(permission_signal_modules)) .client_info(ClientInfo { client_ip, tls_protocol: None, @@ -589,6 +607,25 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R mod tests { use super::*; use edgezero_core::body::Body as EdgeBody; + + /// The services graph a module is handed, built the same way the request + /// path builds it so the tests exercise the production shape. + fn test_services() -> RuntimeServices { + let req = edgezero_core::http::request_builder() + .method("GET") + .uri("https://example.com/") + .body(EdgeBody::empty()) + .expect("should build test request"); + let ctx = edgezero_core::context::RequestContext::new( + req, + edgezero_core::params::PathParams::default(), + ); + build_runtime_services( + &ctx, + &trusted_server_core::settings::Settings::default(), + &Arc::default(), + ) + } use std::time::Duration; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -697,13 +734,20 @@ mod tests { ); } - #[test] - fn geo_always_returns_none() { + #[tokio::test] + async fn geo_always_returns_none() { let geo = AxumPlatformGeo; - let no_ip = geo.lookup(None).expect("should not error"); + let no_ip = geo + .lookup(None, &test_services()) + .await + .expect("should not error"); assert!(no_ip.is_none(), "should return None for no IP"); let with_ip = geo - .lookup(Some("127.0.0.1".parse().expect("should parse IP"))) + .lookup( + Some("127.0.0.1".parse().expect("should parse IP")), + &test_services(), + ) + .await .expect("should not error"); assert!(with_ip.is_none(), "should return None for any IP"); } diff --git a/crates/trusted-server-adapter-axum/tests/permission_signals.rs b/crates/trusted-server-adapter-axum/tests/permission_signals.rs new file mode 100644 index 0000000000..8d77cc204c --- /dev/null +++ b/crates/trusted-server-adapter-axum/tests/permission_signals.rs @@ -0,0 +1,685 @@ +//! The four shipped signal modules assembled together, as a deployment +//! runs them. +//! +//! Each module crate tests its own scheme in isolation, in its own unit +//! tests. What is tested here is what only shows when multiple modules +//! run in order through core's assembly: an answer to a prompt applying +//! over an opt-out, one opt-out standing when another is removed, a scheme +//! left off the list not running at all, and withdrawal being TCF's alone +//! and scoped to the place. This sits in the Axum adapter's tests because +//! it is the first crate that links all four, and core deliberately links +//! none. The names a deployment writes in configuration are checked here for +//! the same reason, against the identifiers the real crates answer to. +//! +//! The consent records here are built by hand, so nothing in core's consent +//! pipeline runs. In a deployment that pipeline also synthesizes a US Privacy +//! opt-out from a Global Privacy Control header in a US state when the consent +//! settings say to, and the `us-privacy` module then acts on it, which is +//! why removing `gpc` from the list alone does not make that header inert. + +use std::sync::Arc; + +use trusted_server_core::consent::types::{GppConsent, TcfConsent, UsPrivacy}; +use trusted_server_core::consent::{ConsentContext, PrivacyFlag}; +use trusted_server_core::ec::consent::{GeoStatus, assemble_permissions}; +use trusted_server_core::evidence::OwnedRequestInfo; +use trusted_server_core::module_context::{ModuleContext, test_support}; +use trusted_server_core::permission_signal::{ + PermissionSignalModule, build_permission_signal_modules, +}; +use trusted_server_core::permissions::{Permission, PermissionState}; +use trusted_server_core::platform::GeoInfo; +use trusted_server_core::settings::Settings; +use trusted_server_permission_signal_gpc::GpcModule; +use trusted_server_permission_signal_gpp::GppSaleOptOutModule; +use trusted_server_permission_signal_mtm::MtmModule; +use trusted_server_permission_signal_tcf::TcfModule; +use trusted_server_permission_signal_us_privacy::UsPrivacyModule; + +/// The four IAB modules an adapter offers, in the default order. MTM is +/// offered after them and is tested with them below. +fn all_four() -> Vec> { + vec![ + Arc::new(GpcModule::new()), + Arc::new(GppSaleOptOutModule::new()), + Arc::new(UsPrivacyModule::new()), + Arc::new(TcfModule::new()), + ] +} + +/// All five, as an adapter offers them. +fn all_five() -> Vec> { + let mut modules = all_four(); + modules.push(Arc::new(MtmModule::new())); + modules +} + +/// Settings naming these identifiers in `[permission-signal] modules`. +fn settings_naming(names: &[&str]) -> Settings { + let mut settings = Settings::default(); + settings.permission_signal.modules = + Some(names.iter().map(|name| (*name).to_owned()).collect()); + settings +} + +/// The modules a deployment gets from naming these identifiers in +/// `[permission-signal] modules`, through the same entry point an adapter's +/// composition root uses. +fn configured(names: &[&str]) -> Arc<[Arc]> { + build_permission_signal_modules(&settings_naming(names), &all_four()) + .expect("should select modules this build offers") +} + +/// Every module except the one named, in the default order, as a +/// publisher removes one from configuration. +fn all_but(excluded: &str) -> Arc<[Arc]> { + let names: Vec<&str> = all_four() + .iter() + .map(|module| trusted_server_core::permission_signal::short_name(module.as_ref())) + .filter(|name| *name != excluded) + .collect(); + configured(&names) +} + +fn no_evidence() -> OwnedRequestInfo { + OwnedRequestInfo::default() +} + +fn assembled( + consent: &ConsentContext, + geo: GeoStatus<'_>, + modules: &[Arc], +) -> PermissionState { + assembled_with(consent, &no_evidence(), geo, modules) +} + +/// The permissions assembled for a request carrying `consent` and `evidence`, +/// with the modules handed the request's module context. +fn assembled_with( + consent: &ConsentContext, + evidence: &OwnedRequestInfo, + geo: GeoStatus<'_>, + modules: &[Arc], +) -> PermissionState { + let context = ModuleContext::new(test_support::request("/")) + .with_consent(consent) + .with_evidence(evidence); + assemble_permissions(&context, geo, modules) +} + +/// Builds a minimal decoded TCF record consenting to the given 1-indexed +/// purposes, with everything else refused. +fn tcf_with_purposes(consented: &[usize]) -> TcfConsent { + let mut purpose_consents = vec![false; 24]; + for &purpose in consented { + purpose_consents[purpose - 1] = true; + } + TcfConsent { + version: 2, + cmp_id: 0, + cmp_version: 0, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 0, + tcf_policy_version: 2, + created_ds: 0, + last_updated_ds: 0, + purpose_consents, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: Vec::new(), + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + } +} + +fn us_privacy_opted_out() -> UsPrivacy { + UsPrivacy { + version: 1, + notice_given: PrivacyFlag::Yes, + opt_out_sale: PrivacyFlag::Yes, + lspa_covered: PrivacyFlag::NotApplicable, + } +} + +fn gpp_sale_opted_out() -> GppConsent { + GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(true), + } +} + +/// A US opt-out state, where the baseline grants storage without a signal, so +/// a revoke is observable as a drop and a refusal is never a withdrawal. +fn us_ca_geo() -> GeoInfo { + GeoInfo { + city: String::new(), + country: "US".to_owned(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: Some("CA".to_owned()), + asn: None, + } +} + +// ---------------------------------------------------------------------- +// Which modules run. +// ---------------------------------------------------------------------- + +#[test] +fn the_documented_names_select_every_shipped_module_in_order() { + // The names the guide and the example configuration list, which must be + // the names the shipped crates answer to within the section. + let documented = ["gpc", "gpp", "us-privacy", "tcf"]; + let selected: Vec<&str> = configured(&documented) + .iter() + .map(|module| trusted_server_core::permission_signal::short_name(module.as_ref())) + .collect(); + assert_eq!( + selected, documented, + "each documented name selects the shipped module it names, in the order written" + ); +} + +#[test] +fn an_old_name_is_refused_naming_the_names_available() { + for old in ["gpp_sale_opt_out", "us_privacy"] { + let Err(error) = build_permission_signal_modules(&settings_naming(&[old]), &all_four()) + else { + panic!("should refuse the old name `{old}`"); + }; + let message = format!("{error:?}"); + assert!( + message.contains(&format!("`{old}` is not available in this build")) + && message.contains("Available modules are gpc, gpp, us-privacy, tcf"), + "the refusal names the old name and the names to write instead: {message}" + ); + } +} + +#[test] +fn gpc_revokes_the_granted_baseline_in_a_us_opt_out_state() { + // A US-style opt-out drops a granted baseline, because the map granted + // these purposes and Global Privacy Control revokes them. + let consent = ConsentContext { + gpc: true, + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + !state.is_set(Permission::StoreOnDevice) + && !state.is_set(Permission::SelectPersonalisedAds), + "GPC should revoke the granted necessary.operations.storage and advertising_marketing.first_party.targeted baseline" + ); +} + +#[test] +fn a_module_left_off_the_list_does_not_run() { + let consent = ConsentContext { + gpc: true, + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + + let everything = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + !everything.is_set(Permission::StoreOnDevice), + "with every module running, the header takes storage away" + ); + + let pruned = assembled(&consent, GeoStatus::Located(&geo), &all_but("gpc")); + assert!( + pruned.is_set(Permission::StoreOnDevice), + "a publisher who does not want to act on Global Privacy Control removes it from \ + the list, and the module that read the header then does not run" + ); +} + +#[test] +fn removing_one_opt_out_leaves_the_others_working() { + // The reason the three opt-outs are separate modules rather than one. + let consent = ConsentContext { + us_privacy: Some(us_privacy_opted_out()), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_but("gpc")); + assert!( + !state.is_set(Permission::StoreOnDevice), + "dropping Global Privacy Control must not drop the US Privacy opt-out with it" + ); +} + +#[test] +fn gpc_suppresses_storage_even_when_us_privacy_reports_no_opt_out() { + let consent = ConsentContext { + gpc: true, + us_privacy: Some(UsPrivacy { + opt_out_sale: PrivacyFlag::No, + ..us_privacy_opted_out() + }), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + !state.is_set(Permission::StoreOnDevice), + "any one opt-out module should suppress, whatever the others say" + ); +} + +// ---------------------------------------------------------------------- +// Opt-out and prompt precedence. +// +// The modules are asked in order and each amends what the ones before it +// settled, so a later module can amend an opt-out. The default order asks +// Global Privacy Control first, being a browser setting with no interface of +// its own, and the schemes carrying a choice someone made through an +// interface after, which is why an answer given at a prompt amends the +// header the visitor arrived with. A deployment wanting the opposite puts +// the module it wants to win last. +// ---------------------------------------------------------------------- + +#[test] +fn a_prompt_answer_applies_over_a_gpc_signal() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[1, 4])), + gpc: true, + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + state.is_set(Permission::StoreOnDevice), + "the visitor answered a prompt after arriving with GPC set, and under the \ + default order the answer they gave is applied over the header they sent" + ); +} + +#[test] +fn a_prompt_answer_applies_over_a_us_privacy_opt_out_signal() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[1, 4])), + us_privacy: Some(us_privacy_opted_out()), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + state.is_set(Permission::StoreOnDevice), + "the visitor answered a prompt after arriving with a US Privacy opt-out, and under \ + the default order the answer they gave amends the signal they sent" + ); +} + +#[test] +fn a_prompt_answer_applies_over_a_gpp_sale_opt_out_signal() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[1, 4])), + gpp: Some(gpp_sale_opted_out()), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + state.is_set(Permission::StoreOnDevice), + "the visitor answered a prompt after arriving with a GPP sale opt-out, and under \ + the default order the answer they gave amends the signal they sent" + ); +} + +#[test] +fn the_opt_out_wins_when_a_deployment_puts_it_last() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[1, 4])), + gpc: true, + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let modules = configured(&["tcf", "gpc"]); + let state = assembled(&consent, GeoStatus::Located(&geo), &modules); + assert!( + !state.is_set(Permission::StoreOnDevice), + "the same request, with the order reversed in configuration, lets the header win" + ); +} + +// ---------------------------------------------------------------------- +// The TCF mapping, now the TCF crate's, still reaches every purpose. +// ---------------------------------------------------------------------- + +#[test] +fn tcf_resolves_every_mapped_purpose_not_just_storage_and_ads() { + // Consent to all purposes except Purpose 7 (measure ad performance), in a + // US opt-out state where the baseline granted them all, so a revoke is + // observable as a drop. + let consented: Vec = (1..=11).filter(|&purpose| purpose != 7).collect(); + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&consented)), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + + assert!( + state.is_set(Permission::SelectBasicAds), + "Purpose 2 consent should set advertising_marketing.first_party.contextual" + ); + assert!( + !state.is_set(Permission::MeasureAdPerformance), + "Purpose 7 refusal should revoke analytics.ad_reporting.measure_ad_performance" + ); + assert!( + state.is_set(Permission::StoreOnDevice) && state.is_set(Permission::SelectPersonalisedAds), + "Purposes 1 and 4 remain resolved from the TCF record" + ); +} + +// ---------------------------------------------------------------------- +// Withdrawal scoping: only a TCF storage refusal withdraws, and only where +// the baseline did not grant storage outright. Opt-outs suppress use but +// never destroy an already-issued identifier. +// +// No location resolves at the policy's top node, the gdpr-eu group, where +// storage requires a signal. A US opt-out state grants it outright. +// ---------------------------------------------------------------------- + +#[test] +fn tcf_storage_refusal_withdraws_under_a_requires_signal_baseline() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[4])), + ..ConsentContext::default() + }; + let state = assembled(&consent, GeoStatus::NoLocation, &all_four()); + assert!( + state.storage_withdrawn(), + "refusing the signal storage depends on should withdraw" + ); +} + +#[test] +fn tcf_storage_refusal_does_not_withdraw_under_a_granted_baseline() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[4])), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + !state.storage_withdrawn(), + "storage never depended on the record here, so refusal suppresses without destroying" + ); +} + +#[test] +fn tcf_storage_consent_is_not_a_withdrawal() { + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[1])), + ..ConsentContext::default() + }; + let state = assembled(&consent, GeoStatus::NoLocation, &all_four()); + assert!( + !state.storage_withdrawn(), + "a consenting record is not a withdrawal" + ); +} + +#[test] +fn gpc_alone_never_withdraws() { + let consent = ConsentContext { + gpc: true, + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + assert!( + !assembled(&consent, GeoStatus::Located(&geo), &all_four()).storage_withdrawn() + && !assembled(&consent, GeoStatus::NoLocation, &all_four()).storage_withdrawn(), + "GPC suppresses use for the request but never destroys the identifier" + ); +} + +#[test] +fn us_style_opt_outs_never_withdraw() { + let consent = ConsentContext { + us_privacy: Some(us_privacy_opted_out()), + gpp: Some(gpp_sale_opted_out()), + ..ConsentContext::default() + }; + let state = assembled(&consent, GeoStatus::NoLocation, &all_four()); + assert!( + !state.storage_withdrawn(), + "sale opt-outs suppress use but never destroy the identifier" + ); +} + +#[test] +fn no_signal_is_not_a_withdrawal() { + let state = assembled( + &ConsentContext::default(), + GeoStatus::NoLocation, + &all_four(), + ); + assert!( + !state.storage_withdrawn(), + "absence of a signal must never destroy an identifier" + ); +} + +#[test] +fn a_withdrawal_needs_the_tcf_module_to_be_running() { + // The withdrawal is TCF's answer, so a deployment that removed the TCF + // module from the list has no scheme left that can withdraw. + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[4])), + ..ConsentContext::default() + }; + let state = assembled(&consent, GeoStatus::NoLocation, &all_but("tcf")); + assert!( + !state.storage_withdrawn(), + "a scheme that does not run cannot withdraw, whatever the request carries" + ); +} + +// ---------------------------------------------------------------------- +// Unreadable and expired records, assembled with the real modules. +// ---------------------------------------------------------------------- + +#[test] +fn an_unreadable_tcf_record_is_the_tcf_modules_refusal() { + let consent = ConsentContext { + raw_tc_string: Some("not-a-tc-string".to_owned()), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + !state.is_set(Permission::StoreOnDevice), + "an unreadable record should block the granted baseline, not vanish" + ); + assert!( + !state.storage_withdrawn(), + "and it fails closed by suppression, never destructively" + ); + assert!( + state.signals().is_empty(), + "and the string is not a valid signal, so nothing downstream sees it" + ); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_but("tcf")); + assert!( + state.is_set(Permission::StoreOnDevice), + "a scheme that does not run answers nothing, readable or not" + ); +} + +#[test] +fn an_unreadable_gpp_string_is_the_gpp_modules_opt_out_and_the_order_decides() { + // The GPP module reads its own unreadable string as the opt-out it + // may have carried. Asked after it, a readable TCF record consenting to + // storage amends that, because the order is the policy. Nothing in + // core answers ahead of the modules. + let consent = ConsentContext { + tcf: Some(tcf_with_purposes(&[1, 4])), + raw_tc_string: Some("CPreadable".to_owned()), + raw_gpp_string: Some("this is not a GPP string".to_owned()), + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + assert!( + !assembled(&consent, GeoStatus::Located(&geo), &all_but("tcf")) + .is_set(Permission::StoreOnDevice), + "without TCF the GPP module's reading of its unreadable string stands" + ); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + state.is_set(Permission::StoreOnDevice), + "asked last, the readable TCF record amends the GPP module's answer" + ); + let schemes: Vec<&str> = state.signals().iter().map(|s| s.scheme).collect(); + assert_eq!( + schemes, + vec!["tcf"], + "only the readable string is vouched for, so only it goes any further" + ); +} + +#[test] +fn an_expired_tcf_record_is_not_treated_as_malformed() { + let consent = ConsentContext { + raw_tc_string: Some("CPc-old-string".to_owned()), + expired: true, + ..ConsentContext::default() + }; + let geo = us_ca_geo(); + let state = assembled(&consent, GeoStatus::Located(&geo), &all_four()); + assert!( + state.is_set(Permission::StoreOnDevice), + "expiry is its own explicit state, deliberately distinct from malformed" + ); +} + +#[test] +fn a_visitor_in_the_eu_with_no_record_is_awaiting_what_tcf_could_grant() { + let geo = GeoInfo { + city: String::new(), + country: "FR".to_owned(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }; + let state = assembled( + &ConsentContext::default(), + GeoStatus::Located(&geo), + &all_four(), + ); + assert!( + state.is_awaited(Permission::StoreOnDevice) + && state.is_awaited(Permission::SelectPersonalisedAds), + "a prompt that has not run leaves the TCF-mapped Data Uses awaited" + ); + let email = Permission::all() + .find(|permission| permission.as_str() == "advertising_marketing.communications.email") + .expect("the taxonomy should carry the email channel"); + assert!( + !state.is_awaited(email), + "a channel no module can grant is not awaited" + ); + let state = assembled( + &ConsentContext { + tcf: Some(tcf_with_purposes(&[1])), + ..ConsentContext::default() + }, + GeoStatus::Located(&geo), + &all_four(), + ); + assert!( + state.is_set(Permission::StoreOnDevice) && !state.is_awaited(Permission::StoreOnDevice), + "an answered prompt settles what it granted" + ); + assert!( + !state.is_awaited(Permission::SelectPersonalisedAds) + && !state.is_set(Permission::SelectPersonalisedAds), + "a record refusing a purpose is a refusal, not an awaited answer" + ); +} + +#[test] +fn a_pmp_answer_in_the_eu_settles_what_the_model_terms_cover_and_declares_them() { + use trusted_server_permission_signal_mtm::COOKIE_MTM_PREF; + + let geo = GeoInfo { + city: String::new(), + country: "FR".to_owned(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + }; + let evidence = |word: &str| { + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + "cookie", + format!("{COOKIE_MTM_PREF}={word}") + .parse() + .expect("should build a cookie header"), + ); + OwnedRequestInfo::new(String::new(), headers) + }; + let standard = assembled_with( + &ConsentContext::default(), + &evidence("standard"), + GeoStatus::Located(&geo), + &all_five(), + ); + assert!( + standard.is_set(Permission::StoreOnDevice) + && standard.is_set(Permission::SelectBasicAds) + && !standard.is_set(Permission::SelectPersonalisedAds) + && !standard.is_awaited(Permission::SelectPersonalisedAds), + "standard grants storage and contextual advertising and refuses targeting" + ); + assert_eq!( + standard.signals(), + &[trusted_server_core::permissions::ValidSignal::new( + "mtm", "mtm", "standard" + )], + "the word is the valid signal, as received" + ); + assert_eq!( + standard + .tdls() + .iter() + .map(trusted_server_core::tdl::Tdl::as_str) + .collect::>(), + vec!["https://m4ow.uk/mtm/2.txt"], + "an answer is given under the versioned Model Terms" + ); + let personalized = assembled_with( + &ConsentContext::default(), + &evidence("personalized"), + GeoStatus::Located(&geo), + &all_five(), + ); + assert!( + personalized.is_set(Permission::SelectPersonalisedAds), + "personalized grants targeting too" + ); + let unanswered = assembled_with( + &ConsentContext::default(), + &no_evidence(), + GeoStatus::Located(&geo), + &all_five(), + ); + assert!( + unanswered.is_awaited(Permission::SelectPersonalisedAds) && unanswered.tdls().is_empty(), + "with no answer the question is still open and no terms are declared" + ); +} diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index bcb31b7e44..4e80a788b3 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -13,7 +13,7 @@ use trusted_server_adapter_axum::app::TrustedServerApp; const LEGACY_ADMIN_DENY_METHODS: &[&str] = &["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"]; -/// Build the full application router from explicit test settings. +/// Explicit test settings for the route tests. /// /// The settings baked into the binary contain placeholder secrets that /// `get_settings()` rejects by design, which would turn every route into a @@ -21,11 +21,6 @@ const LEGACY_ADMIN_DENY_METHODS: &[&str] = fn test_settings() -> trusted_server_core::settings::Settings { trusted_server_core::settings::Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -33,12 +28,19 @@ fn test_settings() -> trusted_server_core::settings::Settings { proxy_secret = "integration-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse route test settings") } +/// Build the full application router from explicit test settings. fn test_router() -> edgezero_core::router::RouterService { TrustedServerApp::routes_with_settings(test_settings()) .expect("should build router from test settings") @@ -65,24 +67,29 @@ fn assert_route_registered(method: &str, path: &str) { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn aps_profile_serves_renderer_through_adapter_fallback() { +async fn an_aps_demand_source_serves_the_renderer_through_adapter_fallback() { let mut settings = test_settings(); - settings.auction.providers.insert( - "aps-main".parse().expect("should parse APS provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: trusted_server_core::auction::NotificationConfig::default(), - profile_config: "{\"account_id\":\"example-account\"}" - .parse() - .expect("should parse APS profile config"), - }, + let table = serde_json::Map::from_iter([ + ( + "implementation".to_string(), + serde_json::json!("auction.aps"), + ), + ( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/pb/bid"), + ), + ("routing".to_string(), serde_json::json!("all_eligible")), + ( + "account_id".to_string(), + serde_json::json!("example-account"), + ), + ]); + settings.demand = trusted_server_core::provider_table::ProviderList::new( + vec!["aps_main".to_string()], + std::collections::BTreeMap::from([("aps_main".to_string(), table)]), ); let router = TrustedServerApp::routes_with_settings(settings) - .expect("should build router with APS profile"); + .expect("should build a router with an APS demand source"); let mut service = EdgeZeroAxumService::new(router); let request = Request::builder() .method("GET") @@ -110,13 +117,11 @@ fn all_explicit_routes_are_registered() { let expected: &[(&str, &str)] = &[ ("GET", "/.well-known/trusted-server.json"), ("POST", "/verify-signature"), - ("POST", "/_ts/admin/keys/rotate"), - ("POST", "/_ts/admin/keys/deactivate"), - ("GET", "/_ts/admin/ec"), - ("GET", "/_ts/admin/ec/{id}"), - ("GET", "/_ts/admin/eids"), - ("POST", "/admin/keys/rotate"), - ("POST", "/admin/keys/deactivate"), + ("GET", "/_ts/permissions"), + ("GET", "/_ts/permissions.json"), + ("GET", "/_ts/config"), + ("GET", "/_ts/config.json"), + ("GET", "/_ts/data"), ("POST", "/auction"), // SPA re-auction endpoint, plus its deprecated `/__ts/` alias. Both // paths are spelled out as literals rather than referencing @@ -141,29 +146,6 @@ fn all_explicit_routes_are_registered() { } } -/// Verify the legacy non-`/_ts` admin aliases ARE registered — to the local -/// deny handler — matching the Fastly and Cloudflare adapters. -/// -/// The production basic-auth handler regex (`^/_ts/admin`) does not match -/// `/admin/keys/*`, so these aliases are not auth-gated. Leaving them unrouted -/// would let them fall through to the publisher fallback, which forwards the -/// request (including `Authorization` and key body) to the origin, leaking admin -/// credentials. Registering them to a local 404 deny fails closed instead. This -/// guard pins the cross-adapter agreement so the divergence cannot silently -/// reappear. -#[test] -fn legacy_admin_aliases_are_registered_to_local_deny() { - let routes = registered_routes(); - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - for method in LEGACY_ADMIN_DENY_METHODS { - assert!( - routes.iter().any(|(m, p)| m == method && p == path), - "legacy {method} {path} must be registered (to the local deny) so it never reaches the publisher fallback; registered routes: {routes:?}" - ); - } - } -} - // --------------------------------------------------------------------------- // Route smoke tests — verify routing (not business logic correctness) // --------------------------------------------------------------------------- @@ -252,7 +234,9 @@ async fn tsjs_route_prefix_is_handled_not_5xx() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { let mut svc = make_service(); - let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); + let src = trusted_server_core::tsjs::tsjs_script_src( + &trusted_server_core::tsjs_bundle::compile_time_parts(&["creative"]), + ); let req = Request::builder() .method("GET") .uri(src) @@ -318,168 +302,16 @@ async fn finalize_middleware_sets_geo_unavailable_header() { } // --------------------------------------------------------------------------- -// Basic-auth parity tests +// Closed paths and the inspection endpoints // --------------------------------------------------------------------------- #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_without_credentials_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 401, - "admin route must return 401 without credentials" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_ec_routes_return_501() { - // The EC identity graph is Fastly KV backed, so the Axum dev server - // answers the admin EC lookup routes locally with 501 instead of letting - // them fall through to the publisher fallback. - let sample_ec_id = format!("{}.abc123", "a".repeat(64)); - for path in [ - "/_ts/admin/ec".to_owned(), - format!("/_ts/admin/ec/{sample_ec_id}"), - ] { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 501, - "{path} should report that Axum EC lookup is unsupported" - ); - assert_eq!( - resp.headers() - .get("content-type") - .and_then(|v| v.to_str().ok()), - Some("application/json") - ); - assert_eq!( - resp.headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_ec_route_without_credentials_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/_ts/admin/ec") - .body(AxumBody::empty()) - .expect("should build unauthenticated admin EC request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - - assert_eq!(resp.status().as_u16(), 401); - assert!( - resp.headers().contains_key("www-authenticate"), - "admin EC 401 should include the Basic authentication challenge" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_eids_route_returns_200() { - // The EIDs echo is pure request inspection (no KV), so the dev server - // serves the real handler. - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/_ts/admin/eids") - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 200, - "/_ts/admin/eids should serve the real EIDs echo handler" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { +async fn an_authenticated_request_to_a_closed_path_is_answered_here() { let ec_id = format!("{}.abc123", "a".repeat(64)); - let valid_paths = [ + for path in [ "/_ts/admin/ec".to_owned(), format!("/_ts/admin/ec/{ec_id}"), "/_ts/admin/eids".to_owned(), - ]; - - for path in valid_paths { - for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - let request = Request::builder() - .method(method) - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::from("sensitive-admin-body")) - .expect("should build authenticated admin request"); - let response = make_service() - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should respond"); - - assert_eq!(response.status().as_u16(), 405); - assert_eq!( - response - .headers() - .get("allow") - .and_then(|v| v.to_str().ok()), - Some("GET") - ); - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } - } - - for path in [ "/_ts/admin/ec/".to_owned(), format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), @@ -487,14 +319,13 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), - // Percent-encoded separators match the `^/_ts/admin` basic-auth - // handler but not a literal-slash namespace check, so they must be - // reserved before publisher fallback forwards credentials upstream. + // A check for a literal slash would miss a percent-encoded separator, + // so these are closed before the publisher fallback forwards + // credentials upstream. "/_ts/admin%2Fec".to_owned(), "/_ts/admin%2fec".to_owned(), - // Retired non-`/_ts` alias namespace: only the two exact paths are - // routed to a local deny, so descendants and encoded separators must - // be reserved at the shared fallback boundary. + // The alias outside `/_ts`, with its descendants and encoded + // separators. "/admin/keys".to_owned(), "/admin/keys/rotate/extra".to_owned(), "/admin/keys%2Frotate".to_owned(), @@ -532,15 +363,18 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { - // Regression for the credential-leak finding: the production basic-auth regex - // `^/_ts/admin` does not match `/admin/keys/*`, so those aliases are not - // auth-gated. Any publisher-fallback method carrying an `Authorization` - // header must be denied locally with 404, never proxied to the publisher - // origin (which would leak the admin credentials and key body). A - // publisher-fallback proxy without a backend would surface as a 5xx, so 404 - // proves the local deny ran. - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { +async fn key_administration_paths_are_answered_here_and_never_proxied() { + // A request to a key administration path carries an operator's + // credentials and a key payload. Every publisher-fallback method must be + // answered here with 404 and never proxied to the publisher origin, which + // would be handed both. A publisher-fallback proxy without a backend would + // surface as a 5xx, so a 404 proves the request was answered here. + for path in [ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/admin/keys/rotate", + "/admin/keys/deactivate", + ] { for method in LEGACY_ADMIN_DENY_METHODS { let mut svc = make_service(); let req = Request::builder() @@ -549,7 +383,7 @@ async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") .header("content-type", "application/json") .body(AxumBody::from("{\"key_id\":\"leak-me\"}")) - .expect("should build authorized legacy-alias request"); + .expect("should build authorized key administration request"); let resp = svc .ready() .await @@ -560,196 +394,132 @@ async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { assert_eq!( resp.status().as_u16(), 404, - "legacy {method} {path} with Authorization must be denied locally (404), not proxied to publisher" + "{method} {path} with Authorization must be answered here (404), not proxied to publisher" ); } } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_without_credentials_includes_www_authenticate_header() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 401, - "should be 401 before checking header" - ); - assert!( - resp.headers().contains_key("www-authenticate"), - "401 response must include WWW-Authenticate header" - ); - let www_auth = resp - .headers() - .get("www-authenticate") - .expect("should have www-authenticate header") - .to_str() - .expect("should be valid UTF-8"); - assert!( - www_auth.starts_with("Basic realm="), - "WWW-Authenticate must be Basic scheme, got: {www_auth}" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_with_wrong_credentials_returns_401() { - use base64::Engine as _; - let creds = base64::engine::general_purpose::STANDARD.encode("admin:wrong-password"); - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .header("authorization", format!("Basic {creds}")) - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 401, - "admin route must reject wrong credentials with 401" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn discovery_endpoint_does_not_require_auth() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_ne!( - resp.status().as_u16(), - 401, - "/.well-known/trusted-server.json must not require auth" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auction_endpoint_does_not_require_auth() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/auction") - .header("content-type", "application/json") - .body(AxumBody::from(r#"{"adUnits":[]}"#)) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_ne!( - resp.status().as_u16(), - 401, - "/auction must not apply admin basic-auth gate" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_returns_non_404_non_5xx() { - let mut svc = make_service(); - - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - let status = resp.status().as_u16(); - - assert_ne!(status, 404, "admin route must be routed"); - // The auth gate short-circuits with 401 before the handler; only an - // unhandled 500 indicates a panic or missing handler. - assert_ne!(status, 500, "admin route must not panic: got {status}"); +async fn permissions_endpoint_answers_anyone_as_data_and_as_a_page() { + for (path, content_type) in [ + ("/_ts/permissions.json", "application/json"), + ("/_ts/permissions", "text/html; charset=utf-8"), + ] { + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri(path) + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 200, + "{path} should answer with no credential" + ); + assert_eq!( + resp.headers()["content-type"], + content_type, + "{path} should answer in its own form" + ); + assert_eq!( + resp.headers()["cache-control"], + "no-store", + "{path} is one request's own answer" + ); + } } -// --------------------------------------------------------------------------- -// Admin key route full path coverage -// --------------------------------------------------------------------------- - -// Exercises the auth-fail path with a realistic key body (complements the -// generic `admin_route_without_credentials_returns_401` above). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_rotate_key_auth_fail_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 401, - "admin/keys/rotate without credentials must return 401" - ); +async fn config_endpoint_answers_anyone_and_masks_secrets() { + for (path, content_type) in [ + ("/_ts/config.json", "application/json"), + ("/_ts/config", "text/html; charset=utf-8"), + ] { + let mut svc = make_service(); + let req = Request::builder() + .method("GET") + .uri(path) + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + 200, + "{path} should answer with no credential" + ); + assert_eq!( + resp.headers()["content-type"], + content_type, + "{path} should answer in its own form" + ); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("should read the body"); + let text = String::from_utf8_lossy(&body); + assert!( + text.contains("XXXX"), + "{path} should mask what it does not show" + ); + assert!( + !text.contains("integration-test-proxy-secret"), + "{path} should not carry the proxy secret" + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_deactivate_key_auth_fail_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/deactivate") - .header("content-type", "application/json") - .body(AxumBody::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - assert_eq!( - resp.status().as_u16(), - 401, - "admin/keys/deactivate without credentials must return 401" - ); +async fn data_endpoint_answers_a_browser_opening_the_page_and_nothing_else() { + for (opened_as_a_page, expected) in [(false, 403), (true, 200)] { + let mut svc = make_service(); + let mut builder = Request::builder().method("GET").uri("/_ts/data"); + if opened_as_a_page { + builder = builder + .header("sec-fetch-mode", "navigate") + .header("sec-fetch-dest", "document"); + } + let req = builder + .body(AxumBody::empty()) + .expect("should build request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + assert_eq!( + resp.status().as_u16(), + expected, + "opened as a page: {opened_as_a_page}" + ); + assert_eq!( + resp.headers()["cache-control"], + "no-store, private", + "should never be stored" + ); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("should read the body"); + assert_eq!( + String::from_utf8_lossy(&body).contains("keeps no record"), + opened_as_a_page, + "should say nothing is held to a browser opening the page, and to nothing else" + ); + } } // --------------------------------------------------------------------------- @@ -868,6 +638,97 @@ async fn first_party_proxy_rebuild_is_routed() { ); } +// --------------------------------------------------------------------------- +// Edge Cookie module availability +// --------------------------------------------------------------------------- + +/// Test settings selecting a vendor Edge Cookie module this adapter does not +/// inject, with the `[ec.acme]` block that module's settings live in. +/// `acme` is a fictional vendor key. +const UNINJECTED_MODULE_TOML: &str = r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "integration-test-proxy-secret" + + [ec] + module = "acme" + + [ec.acme] + endpoint = "https://ec.acme.example.com" + + # An Edge Cookie module is configured, so single-jurisdiction operation + # is acknowledged because no geo module is selected. + [geo] + assume_single_jurisdiction = true +"#; + +/// A module selection this adapter can never supply must fail while the +/// application state is built, before any request is served. +/// +/// Configuration validation accepts this selection, because only the adapter +/// that injects a module knows what that module needs, and the Axum dev +/// server injects no vendor Edge Cookie module, so only the composition root +/// can catch it. Without the startup check the deployment would come up and +/// answer every request. +#[test] +fn selecting_a_module_this_adapter_cannot_supply_fails_at_startup() { + let settings = trusted_server_core::settings::Settings::from_toml(UNINJECTED_MODULE_TOML) + .expect("should parse settings selecting an uninjected module"); + + // `RouterService` is not `Debug`, so take the error side directly rather + // than through `expect_err`. + let error = trusted_server_adapter_axum::app::TrustedServerApp::routes_with_settings(settings) + .err() + .expect("building state with an uninjected module should fail"); + + assert!( + error.to_string().contains("acme"), + "the startup error should name the selected module, got: {error}" + ); +} + +/// Builds a registration claiming an id a stock module already owns. +fn duplicate_lockr_registration( + _settings: &trusted_server_core::settings::Settings, +) -> Result< + Option, + error_stack::Report, +> { + Ok(Some( + trusted_server_core::integrations::IntegrationRegistration::builder("lockr").build(), + )) +} + +fn validate_nothing( + _settings: &trusted_server_core::settings::Settings, +) -> Result> { + Ok(true) +} + +#[test] +fn routes_with_registrations_rejects_a_duplicate_integration_id_naming_both_sources() { + let extra = [trusted_server_core::integrations::IntegrationBuilder::new( + "lockr", + "seam-probe", + duplicate_lockr_registration, + validate_nothing, + )]; + + let error = TrustedServerApp::routes_with_registrations(test_settings(), &extra) + .err() + .expect("should reject a duplicate integration id supplied through the adapter"); + + let message = error.to_string(); + assert!( + message.contains("lockr") + && message.contains("trusted-server-identity-lockr") + && message.contains("seam-probe"), + "error should name the id and both sources: {message}" + ); +} + /// Regression test: a Next.js navigation with a pending auction must buffer to /// the structural body close. The Flight payload carries a literal ``, so /// a parser-blind seam would inject bids early and split the RSC data. @@ -939,8 +800,26 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { ); } +/// Every route registered beneath an underscore prefix is one of the +/// addresses an attestation endpoint may not take, so a new fixed route +/// cannot leave an endpoint free to register its address twice. +#[test] +fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() { + use trusted_server_core::attestation::RESERVED_PATHS; + + for route in test_router().routes() { + let path = route.path(); + if path.starts_with("/_") { + assert!( + RESERVED_PATHS.contains(&path), + "{path} should be listed in attestation::RESERVED_PATHS" + ); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn health_reports_git_version() { +async fn health_carries_no_version() { let mut service = make_service(); let request = Request::builder() .method("GET") @@ -961,12 +840,8 @@ async fn health_reports_git_version() { 200, "should return 200 on /health" ); - assert_eq!( - response - .headers() - .get("x-ts-version") - .and_then(|v| v.to_str().ok()), - trusted_server_core::constants::TS_GIT_VERSION, - "should report the compiled-in git version on /health" + assert!( + response.headers().get("x-ts-version").is_none(), + "should write no version on /health, which the service_version middleware writes on a page" ); } diff --git a/crates/trusted-server-adapter-axum/tests/seam_probe.rs b/crates/trusted-server-adapter-axum/tests/seam_probe.rs new file mode 100644 index 0000000000..6e7e75d20e --- /dev/null +++ b/crates/trusted-server-adapter-axum/tests/seam_probe.rs @@ -0,0 +1,1123 @@ +//! Round-trip tests for the integration seam, driven from outside +//! `trusted-server-core`. +//! +//! Every capability the seam exposes is registered here by +//! `trusted-server-integration-seam-probe`, a fixture crate core knows nothing +//! about, and asserted through the Axum adapter's real router. Each test +//! turns on an observable outcome (the bytes served, the JSON a route +//! returns, the error a startup or deploy check produces) rather than on a +//! function having been called. + +use std::sync::Arc; + +use axum::body::Body as AxumBody; +use axum::http::{HeaderMap, Request}; +use edgezero_adapter_axum::service::EdgeZeroAxumService; +use error_stack::Report; +use tower::{Service as _, ServiceExt as _}; +use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_core::auction::compile_auction_plan; +use trusted_server_core::config::validate_settings_for_deploy_with; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::OwnedRequestInfo; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistration, IntegrationRegistry, +}; +use trusted_server_core::module_context::{ModuleContext, ModuleRequest}; +use trusted_server_core::platform::{ + BackendNamingPolicy, ClientInfo, DisabledGeo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformSecretStore, RuntimeServices, StoreId, StoreName, + UnavailableHttpClient, UnavailableKvStore, +}; +use trusted_server_core::settings::Settings; +use trusted_server_core::tsjs::tsjs_script_src; +use trusted_server_core::tsjs_bundle::{JsModulePart, compile_time_parts}; +use trusted_server_integration_seam_probe as seam_probe; + +/// Largest response body these tests read, ample for the served script. +const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; + +/// Settings shaped like the route tests', with `extra` appended. +/// +/// The settings baked into the binary carry placeholder secrets that +/// `get_settings()` rejects by design, so every test states its own. +fn settings_with(extra: &str) -> Settings { + // A request the geo module leaves unmatched resolves at the top of the + // permissions.yaml rules tree, and a deployment that runs an Edge Cookie + // module with no geo module acknowledges that explicitly. Tests that + // select a geo module write their own `[geo]` table, so only supply one + // when the caller has not. + let geo = if extra.contains("[geo]") { + String::new() + } else { + " +[geo] +assume_single_jurisdiction = true +" + .to_owned() + }; + Settings::from_toml(&format!( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "seam-probe-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + + {extra} + {geo} + "# + )) + .expect("should parse seam probe test settings") +} + +/// The probe's own section and table, selecting it with the country the geo +/// assertions expect. +const PROBE_BLOCK: &str = r#" + [testing] + modules = ["seam-probe"] + + [testing.seam-probe] + country = "ZZ" +"#; + +/// Builds a service from the router composed with the supplied builders. +fn service_with(settings: Settings, integrations: &[IntegrationBuilder]) -> EdgeZeroAxumService { + let router = TrustedServerApp::routes_with_registrations(settings, integrations) + .expect("should build a router from the composed builders"); + EdgeZeroAxumService::new(router) +} + +/// Builds a registry with the probe registered, over the auction plan the +/// settings compile to, the way an adapter builds one at startup. +fn registry_with_probe(settings: &Settings) -> IntegrationRegistry { + let plan = Arc::new(compile_auction_plan(settings).expect("should compile the auction plan")); + IntegrationRegistry::with_plan_and_registrations(settings, plan, &[seam_probe::builder()]) + .expect("should build a registry with the probe registered") +} + +/// Sends one GET carrying the probe's counting header and returns the +/// response. Each caller passes its own `token` so tests running in parallel +/// count into separate entries. +async fn get_counted( + service: &mut EdgeZeroAxumService, + uri: &str, + token: &str, +) -> axum::response::Response { + let request = Request::builder() + .method("GET") + .uri(uri) + .header(seam_probe::SEAM_PROBE_COUNT_HEADER, token) + .body(AxumBody::empty()) + .expect("should build request"); + + service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond") +} + +/// Sends one GET and returns the response. +async fn get(service: &mut EdgeZeroAxumService, uri: &str) -> axum::response::Response { + let request = Request::builder() + .method("GET") + .uri(uri) + .body(AxumBody::empty()) + .expect("should build request"); + + service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond") +} + +/// Reads a response body as a string. +async fn body_text(response: axum::response::Response) -> String { + let bytes = axum::body::to_bytes(response.into_body(), MAX_BODY_BYTES) + .await + .expect("should read the response body"); + String::from_utf8(bytes.to_vec()).expect("should be UTF-8") +} + +/// The parts the unified bundle is expected to be composed from when the probe +/// is the only enabled module: the always-on `creative` module, then the +/// module the probe's registration carries. `compose` puts core first. +fn expected_bundle_parts() -> Vec { + let mut parts = compile_time_parts(&["creative"]); + parts.push(JsModulePart { + id: seam_probe::SEAM_PROBE_ID, + source: seam_probe::PROBE_JS, + sha256: seam_probe::PROBE_JS_SHA256, + }); + parts +} + +// --------------------------------------------------------------------------- +// 1. The carried module reaches the browser through the unified bundle +// --------------------------------------------------------------------------- + +/// A module a crate outside core carries is served in the unified bundle, +/// under the hash composed from its content, and marked immutable when the +/// request's `?v=` matches that hash. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn carried_module_is_served_in_the_unified_bundle_under_its_composed_hash() { + let mut service = service_with(settings_with(PROBE_BLOCK), &[seam_probe::builder()]); + let source = tsjs_script_src(&expected_bundle_parts()); + + let response = get(&mut service, &source).await; + + assert_eq!( + response.status().as_u16(), + 200, + "should serve the unified bundle" + ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, s-maxage=31536000, immutable"), + "a `?v=` matching the composed hash should be served immutable" + ); + + let body = body_text(response).await; + let core = JsModulePart::compile_time("core").expect("should compile core in"); + + assert!( + body.starts_with(core.source), + "the bundle should start with core" + ); + assert!( + body.contains(seam_probe::PROBE_JS), + "the bundle should carry the probe's module source" + ); +} + +// --------------------------------------------------------------------------- +// 2. A proxy route sees the module's geo and the preparer's marker +// --------------------------------------------------------------------------- + +/// The probe's proxy route reports the country its own geo module resolved, +/// selected by `[geo] module`, and the marker its request preparer left in +/// the request extensions. +/// +/// This is the first end-to-end proof that an adapter runs registry preparers +/// on the request path. +/// +/// The run count is asserted exactly, and the invariant is that a request +/// prepares exactly once: the adapter prepares at a single point that covers +/// every route, so a module's preparer sees each request one time and always +/// before routing. A preparer that appends a header, counts, or emits +/// telemetry can only be written against that guarantee. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() { + let settings = settings_with(&format!( + r#" + [geo] + module = "testing.seam-probe" + {PROBE_BLOCK} + "# + )); + let mut service = service_with(settings, &[seam_probe::builder()]); + + let response = get(&mut service, seam_probe::SEAM_PROBE_REPORT_PATH).await; + + assert_eq!( + response.status().as_u16(), + 200, + "the probe's proxy route should be dispatched" + ); + + let body = body_text(response).await; + let report: serde_json::Value = + serde_json::from_str(&body).expect("the route should return JSON"); + + assert_eq!( + report["module"], + seam_probe::module_name(), + "the route should identify the module: {body}" + ); + assert_eq!( + report["geo_country"], "ZZ", + "the route should see the country the module's own geo module resolved: {body}" + ); + assert_eq!( + report["request_preparer_runs"], 1, + "the adapter should have run the module's request preparer exactly once on the request path: {body}" + ); +} + +/// `[shared.seam-probe]` runs the builder the probe shares, whose route +/// answers the table's marker. Without the table the builder is not built, +/// and the table with no builder reading it is refused at startup. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_vendors_shared_table_runs_its_builder() { + let mut with = service_with( + settings_with(&format!( + "[shared.seam-probe]\nmarker = \"from the shared table\"\n{PROBE_BLOCK}" + )), + &[seam_probe::builder(), seam_probe::shared_builder()], + ); + let response = get(&mut with, seam_probe::SEAM_PROBE_SHARED_PATH).await; + assert_eq!( + response.status().as_u16(), + 200, + "the table runs the builder" + ); + assert_eq!(body_text(response).await, "from the shared table"); + + let mut without = service_with( + settings_with(PROBE_BLOCK), + &[seam_probe::builder(), seam_probe::shared_builder()], + ); + let response = get(&mut without, seam_probe::SEAM_PROBE_SHARED_PATH).await; + assert_ne!( + response.status().as_u16(), + 200, + "without the table the builder is not built, so the route is not this server's" + ); + + let refused = TrustedServerApp::routes_with_registrations( + settings_with("[shared.seam-probe]\nmarker = \"x\"\n"), + &[seam_probe::builder()], + ) + .err() + .expect("nothing reads the table"); + assert!( + format!("{refused:?}").contains("[shared.seam-probe] is read by no module"), + "{refused:?}" + ); +} + +/// With `refuse_all` selected every response this adapter finalizes says not +/// to index, and the file itself is the publisher path's, made by the +/// `robots_txt` middleware the entry names, so no route of this server names +/// the path. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { + let settings = || { + settings_with(&format!( + r#" + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + {PROBE_BLOCK} + "# + )) + }; + let registered: Vec = + TrustedServerApp::routes_with_registrations(settings(), &[seam_probe::builder()]) + .expect("should build the routes") + .routes() + .into_iter() + .map(|route| route.path().to_string()) + .collect(); + assert!( + !registered.iter().any(|path| path == "/robots.txt"), + "the file is the publisher path's: {registered:?}" + ); + + let mut service = service_with(settings(), &[seam_probe::builder()]); + let other = get(&mut service, seam_probe::SEAM_PROBE_REPORT_PATH).await; + + assert_eq!( + other + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()), + Some("noindex, nofollow"), + "no route may be the one page a crawler is allowed to index" + ); +} + +/// A name `[robots-txt] modules` gives that nothing the deployment runs +/// supplies stops the adapter at startup, so the file does not fail on the +/// first crawler to ask for it. +#[test] +fn robots_txt_selection_naming_a_module_that_is_not_running_fails_at_startup() { + let settings = settings_with( + r#" + [robots-txt] + modules = ["testing.seam-probe"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ); + + let error = TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()]) + .err() + .expect("should refuse a contributor whose module no section selects"); + let message = format!("{error:?}"); + + assert!( + message.contains("`[robots-txt] modules` names `testing.seam-probe`"), + "should name the selection: {message}" + ); + assert!( + message.contains("refuse_all") && message.contains("allow_all"), + "should say what can be named: {message}" + ); +} + +/// A request prepares exactly once whether it is served by a named route or by +/// the fallback, so the invariant holds across the whole route table rather +/// than only on the path the probe's own proxy route sits on. +/// +/// The count comes from the probe's counter rather than the request +/// extensions, because a named route returns a fixed response and reports +/// nothing about the request it was given. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn every_route_prepares_the_request_exactly_once() { + let settings = settings_with(PROBE_BLOCK); + let mut service = service_with(settings, &[seam_probe::builder()]); + + // `/first-party/proxy` is a named route, reached through + // `named_route_handler`, whose handler refuses a request naming no target, + // so the request reaches the preparer and no backend. + let named = get_counted(&mut service, "/first-party/proxy", "named-route").await; + + assert_eq!( + named.status().as_u16(), + 502, + "the named route should refuse a request that names no target" + ); + assert_eq!( + seam_probe::prepare_runs_for("named-route"), + 1, + "a request served by a named route should prepare exactly once" + ); + + // The probe's own proxy route is served by the fallback dispatcher. + let fallback = get_counted( + &mut service, + seam_probe::SEAM_PROBE_REPORT_PATH, + "fallback-route", + ) + .await; + + assert_eq!( + fallback.status().as_u16(), + 200, + "the probe's proxy route should be dispatched by the fallback" + ); + assert_eq!( + seam_probe::prepare_runs_for("fallback-route"), + 1, + "a request served by the fallback should prepare exactly once" + ); +} + +// --------------------------------------------------------------------------- +// 3. Deploy validation reaches a vendor's own rule +// --------------------------------------------------------------------------- + +/// `validate_settings_for_deploy_with` runs the probe's own validation, so a +/// country that breaks the probe's two-letter rule is rejected with the +/// probe's message. +#[test] +fn deploy_validation_rejects_a_violation_of_the_modules_own_rule() { + let settings = settings_with( + r#" + [testing] + modules = ["seam-probe"] + + [testing.seam-probe] + country = "ZZZ" + "#, + ); + + let error = validate_settings_for_deploy_with(&settings, &[seam_probe::builder()]) + .expect_err("should reject the probe's own rule violation"); + + assert!( + error + .to_string() + .contains(seam_probe::SEAM_PROBE_COUNTRY_MESSAGE), + "should reject with the module's own message: {error}" + ); + assert!( + validate_settings_for_deploy_with(&settings, &[]).is_ok(), + "the same settings should pass without the module's builder, so the rejection is the module's and not core's" + ); +} + +// --------------------------------------------------------------------------- +// 4. A geo selector naming a module that supplies no module fails at startup +// --------------------------------------------------------------------------- + +/// `[geo] module` naming a selected module that supplies no geo module is a +/// startup error, raised where the adapter builds its routes. +#[test] +fn geo_selector_naming_a_module_without_a_geo_module_fails_at_startup() { + let settings = settings_with( + r#" + [geo] + module = "testing.seam-probe" + + [testing] + modules = ["seam-probe"] + + [testing.seam-probe] + country = "ZZ" + declares_geo = false + "#, + ); + + let error = TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()]) + .err() + .expect("should refuse to start when the selected module supplies no geo module"); + + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `testing.seam-probe`") + && message.contains("is selected and supplies no geo module"), + "should name the module and the missing capability: {message}" + ); +} + +/// A `[[fetch]]` entry naming the probe's middleware starts the adapter, and +/// the registry runs that middleware on the paths the entry covers. +#[test] +fn fetch_entry_naming_a_modules_middleware_starts_the_adapter() { + use trusted_server_core::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; + + let settings = || { + settings_with(&format!( + r#" + {PROBE_BLOCK} + + [[fetch]] + media_type = "text/html" + path = "/news/" + middleware = ["testing.seam-probe"] + "# + )) + }; + + let registry = registry_with_probe(&settings()); + let chain_for = |path: &str| { + registry + .middleware_chain( + &settings().fetch, + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + path, + ) + .ids() + }; + assert_eq!( + chain_for("/news/today"), + [seam_probe::module_name()], + "should run the module's middleware under the entry's path" + ); + assert!( + chain_for("/sport/today").is_empty(), + "should run nothing on a path no entry covers" + ); + + TrustedServerApp::routes_with_registrations(settings(), &[seam_probe::builder()]) + .expect("should start with an entry naming a middleware the module supplies"); +} + +/// A `[[serve]]` entry starts the adapter when it names a middleware that +/// runs in that phase, and is a startup error when it names one that does +/// not. +#[test] +fn serve_entry_starts_the_adapter_only_with_a_middleware_of_that_phase() { + let settings = |name: &str| { + settings_with(&format!( + r#" + {PROBE_BLOCK} + + [[serve]] + media_type = "text/html" + middleware = ["{name}"] + "# + )) + }; + + TrustedServerApp::routes_with_registrations( + settings(seam_probe::reader_middleware_name()), + &[seam_probe::builder()], + ) + .expect("should start with a serve entry naming the module's serve middleware"); + + let error = TrustedServerApp::routes_with_registrations( + settings(seam_probe::module_name()), + &[seam_probe::builder()], + ) + .err() + .expect("should refuse to start when a serve entry names a fetch middleware"); + + let message = error.to_string(); + assert!( + message.contains("[[serve]] entry 1 names `testing.seam-probe`") + && message.contains("does not run in that phase. It runs in [[fetch]]"), + "should say which phase the middleware runs in: {message}" + ); +} + +/// A `[[fetch]]` entry naming the middleware of a module no section selects +/// is a startup error, raised where the adapter builds its routes. +#[test] +fn fetch_entry_naming_the_middleware_of_an_unselected_module_fails_at_startup() { + let settings = settings_with( + r#" + [[fetch]] + media_type = "text/html" + middleware = ["testing.seam-probe"] + "#, + ); + + let error = TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()]) + .err() + .expect("should refuse to start when an entry names a middleware that does not run"); + + let message = error.to_string(); + assert!( + message.contains("[[fetch]] entry 1 names `testing.seam-probe`") + && message.contains("`testing.seam-probe` is a module no section selects"), + "should name the entry and say why the middleware is not running: {message}" + ); +} + +// --------------------------------------------------------------------------- +// 5. Two builders claiming one id are rejected, naming both sources +// --------------------------------------------------------------------------- + +/// A second builder claiming the probe's id is rejected at startup, and the +/// error names the id and both sources so an operator can tell which two +/// crates collided. +#[test] +fn duplicate_integration_id_is_rejected_naming_both_sources() { + /// Source label of the builder that collides with the probe's. + const OTHER_SOURCE: &str = "another-vendor-crate"; + + fn register_nothing( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + fn validate_nothing(_settings: &Settings) -> Result> { + Ok(false) + } + + let extra = [ + seam_probe::builder(), + IntegrationBuilder::new( + seam_probe::SEAM_PROBE_ID, + OTHER_SOURCE, + register_nothing, + validate_nothing, + ), + ]; + + let error = TrustedServerApp::routes_with_registrations(settings_with(PROBE_BLOCK), &extra) + .err() + .expect("should reject two builders claiming one integration id"); + + let message = error.to_string(); + assert!( + message.contains(seam_probe::SEAM_PROBE_ID) + && message.contains(seam_probe::SEAM_PROBE_SOURCE) + && message.contains(OTHER_SOURCE), + "should name the id and both sources: {message}" + ); +} + +// --------------------------------------------------------------------------- +// 6. A demand source and an ad server the probe supplies are known to the +// plan and to validation +// --------------------------------------------------------------------------- + +/// `[demand]` selecting the probe's demand source under a label of the +/// operator's own, with nothing else of the probe's selected. +const PROBE_DEMAND_BLOCK: &str = r#" + [demand] + modules = ["probe"] + + [demand.probe] + implementation = "testing.seam-probe" + endpoint = "https://demand.example/openrtb2/auction" +"#; + +/// A demand source a crate outside core supplies starts the adapter, and the +/// same settings are refused without that crate's builder, by the adapter and +/// by deploy validation alike. +#[test] +fn demand_source_a_module_supplies_is_known_only_with_its_builder() { + let started = TrustedServerApp::routes_with_registrations( + settings_with(PROBE_DEMAND_BLOCK), + &[seam_probe::builder()], + ); + assert!( + started.is_ok(), + "the adapter should start with the probe's demand source selected: {:?}", + started.err() + ); + validate_settings_for_deploy_with(&settings_with(PROBE_DEMAND_BLOCK), &[seam_probe::builder()]) + .expect("should accept the probe's demand source when given the probe's builder"); + + let error = TrustedServerApp::routes_with_registrations(settings_with(PROBE_DEMAND_BLOCK), &[]) + .err() + .expect("should refuse a demand source no builder in the deployment supplies"); + let message = error.to_string(); + assert!( + message.contains("[demand] `probe` uses implementation `testing.seam-probe`"), + "should name the demand source and the implementation nothing supplies: {message}" + ); + let error = validate_settings_for_deploy_with(&settings_with(PROBE_DEMAND_BLOCK), &[]) + .expect_err("should refuse the probe's demand source without the probe's builder"); + assert!( + error.to_string().contains("testing.seam-probe"), + "should name the implementation nothing supplies: {error}" + ); +} + +/// `[ad-server]` selecting the probe's ad server under a label of the +/// operator's own, with nothing else of the probe's selected. +const PROBE_ADSERVER_BLOCK: &str = r#" + [ad-server] + module = "probe" + + [ad-server.probe] + implementation = "testing.seam-probe" +"#; + +/// The adapter compiles the auction plan with the builders it is given, so an +/// ad server a crate outside core supplies starts the adapter, and the same +/// settings are refused without that crate's builder. +#[test] +fn ad_server_a_module_supplies_starts_the_adapter_only_with_its_builder() { + let started = TrustedServerApp::routes_with_registrations( + settings_with(PROBE_ADSERVER_BLOCK), + &[seam_probe::builder()], + ); + assert!( + started.is_ok(), + "the adapter should start with the probe's ad server selected: {:?}", + started.err() + ); + + let error = + TrustedServerApp::routes_with_registrations(settings_with(PROBE_ADSERVER_BLOCK), &[]) + .err() + .expect("should refuse an ad server no builder in the deployment supplies"); + let message = error.to_string(); + assert!( + message.contains("[ad-server] `probe` uses implementation `testing.seam-probe`"), + "should name the ad server and the implementation nothing supplies: {message}" + ); +} + +/// Deploy validation compiles the plan with the builders it is given too, so +/// it agrees with the running adapter about the probe's ad server. +#[test] +fn deploy_validation_accepts_a_modules_ad_server_only_with_its_builder() { + let settings = settings_with(PROBE_ADSERVER_BLOCK); + + validate_settings_for_deploy_with(&settings, &[seam_probe::builder()]) + .expect("should accept the probe's ad server when given the probe's builder"); + + let error = validate_settings_for_deploy_with(&settings, &[]) + .expect_err("should refuse the probe's ad server without the probe's builder"); + assert!( + error.to_string().contains("testing.seam-probe"), + "should name the implementation nothing supplies: {error}" + ); +} + +/// Settings selecting a module-supplied module. +/// +/// These cannot use [`settings_with`], because that fixture carries the +/// deprecated `[ec] passphrase`, and configuration validation rejects the old +/// key and a module selection together. +fn settings_selecting_module(extra: &str) -> Settings { + Settings::from_toml(&format!( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "seam-probe-test-proxy-secret" + + [geo] + assume_single_jurisdiction = true + + {extra} + + {PROBE_BLOCK} + "# + )) + .expect("should parse settings selecting a module-supplied module") +} + +// --------------------------------------------------------------------------- +// Identity and device declared on the registration +// --------------------------------------------------------------------------- + +/// `[ec] module` naming a module resolves to that module's Edge Cookie +/// module, so identity reaches core through the integration registration +/// rather than through a second extension mechanism. +/// +/// This test proves the selection resolves the module's module by its name. +/// `ec_module_generates_an_identifier_with_the_modules_prefix` then drives +/// that resolved module through `generate` and asserts the `seam-probe-` +/// prefix the built-in HMAC module can never produce, so the round trip is +/// proven there. +#[test] +fn ec_selector_naming_a_module_resolves_the_edge_cookie_module_it_declares() { + let settings = settings_selecting_module( + r#" + [ec] + module = "testing.seam-probe" + "#, + ); + + let registry = registry_with_probe(&settings); + + let module = registry + .ec_module() + .expect("`[ec] module = \"testing.seam-probe\"` should resolve the module's module"); + + assert_eq!( + module.id(), + seam_probe::module_name(), + "the resolved module should be the one the module declared" + ); +} + +/// `[ec] module` may name a label of the operator's own, whose block names +/// the module, and the adapter starts with that module resolved. A label is +/// how a module named in full, such as the probe's, takes a settings block. +#[test] +fn ec_selector_naming_a_label_resolves_the_module_its_block_names() { + let settings = settings_selecting_module( + r#" + [ec] + module = "probe" + + [ec.probe] + implementation = "testing.seam-probe" + "#, + ); + + let registry = registry_with_probe(&settings); + assert_eq!( + registry + .ec_module() + .expect("should resolve the module the label's block names") + .id(), + seam_probe::module_name(), + "the resolved module should be the one the block names" + ); + + let started = TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()]); + assert!( + started.is_ok(), + "the adapter should start with the module selected under a label: {:?}", + started.err() + ); +} + +/// `[ec] module` naming a module starts the adapter, because the startup +/// check is handed the module the module declared. +/// +/// The selector tests around this one build the registry directly, so they +/// cannot see a startup check in the adapter that runs before the registry +/// exists and refuses a selection only a module can satisfy. +#[test] +fn ec_selector_naming_a_module_starts_the_adapter() { + let settings = settings_selecting_module( + r#" + [ec] + module = "testing.seam-probe" + "#, + ); + + let started = TrustedServerApp::routes_with_registrations(settings, &[seam_probe::builder()]); + + assert!( + started.is_ok(), + "the adapter should start with a module's Edge Cookie module selected: {:?}", + started.err() + ); +} + +/// `[device] module` naming a module resolves that module's device module +/// the same way, so all three capabilities travel one route. +#[test] +fn device_selector_naming_a_module_resolves_the_device_module_it_declares() { + let settings = settings_selecting_module( + r#" + [device] + module = "testing.seam-probe" + "#, + ); + + let registry = registry_with_probe(&settings); + + let module = registry + .device_module() + .expect("`[device] module = \"testing.seam-probe\"` should resolve the module's module"); + + assert_eq!( + module.id(), + seam_probe::module_name(), + "the resolved module should be the one the module declared" + ); +} + +/// `[prefix] module` naming a module resolves that module's prefix module +/// the same way, so the prefixes travel the route the other capabilities do. +#[test] +fn prefix_selector_naming_a_module_resolves_the_prefix_module_it_declares() { + let settings = settings_selecting_module( + r#" + [prefix] + module = "testing.seam-probe" + "#, + ); + + let registry = registry_with_probe(&settings); + + assert_eq!( + registry.prefix_module().name(), + seam_probe::module_name(), + "the resolved module should be the one the module declared" + ); +} + +/// A request's services carry what the selected prefix module gives, which +/// the probe's route answers, so the names the browser holds and the keys a +/// shared store holds are the module's and not core's own. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_requests_services_carry_the_selected_modules_prefixes() { + let settings = settings_with(&format!( + r#" + [prefix] + module = "testing.seam-probe" + {PROBE_BLOCK} + "# + )); + let mut service = service_with(settings, &[seam_probe::builder()]); + + let response = get(&mut service, seam_probe::SEAM_PROBE_REPORT_PATH).await; + + assert_eq!(response.status().as_u16(), 200); + let body = body_text(response).await; + let report: serde_json::Value = + serde_json::from_str(&body).expect("the route should return JSON"); + assert_eq!( + report["public_prefix"], + seam_probe::SEAM_PROBE_PREFIX_HEAD, + "the names the browser holds start with the module's head: {body}" + ); + assert_eq!( + report["server_key_prefix"], "probe.test-publisher.example.com", + "the keys a shared store holds start with the module's head and the publisher's domain: {body}" + ); +} + +/// The device module `[device] module` resolves answers the confidence that +/// a person is there and what a bidder is told about the device, through the +/// call it is handed, so both reach core from a module it does not know. +#[tokio::test] +async fn the_resolved_device_module_answers_a_confidence_and_the_devices_attributes() { + let settings = settings_selecting_module( + r#" + [device] + module = "testing.seam-probe" + "#, + ); + let registry = registry_with_probe(&settings); + let module = registry + .device_module() + .expect("`[device] module = \"testing.seam-probe\"` should resolve the module's module"); + let request = ModuleRequest::new( + &axum::http::Method::GET, + "www.publisher.example", + "https", + "/", + ); + let context = ModuleContext::new(request).with_settings(&settings); + + let signals = module + .detect(context.call(module.id(), module.required_permissions())) + .await; + let attributes = module + .advertising_attributes(context.call(module.id(), module.required_permissions())) + .await; + + assert_eq!( + signals.human_confidence, + trusted_server_core::ec::device::HumanConfidence::Medium, + "the scoring module's answer should reach core unchanged" + ); + assert_eq!( + attributes, + Some(seam_probe::SeamProbeDevice::attributes()), + "the attributes the module resolved should reach core unchanged" + ); + assert_eq!( + attributes.and_then(|found| found.device_type), + Some(5), + "a tablet is OpenRTB device type 5" + ); +} + +/// `[ec] module` naming a module resolves a module that, when driven +/// through `generate`, produces an identifier the built-in HMAC module can +/// never make. +/// +/// The selection tests above prove the resolved module is the module's by +/// its name. This one drives that module end to end: it hands the resolved +/// module a request and asserts the identifier it returns starts +/// `seam-probe-`, a prefix only the module's module produces, so the +/// module's module genuinely ran and read the evidence rather than core's +/// built-in one answering. +#[tokio::test] +async fn ec_module_generates_an_identifier_with_the_modules_prefix() { + let settings = settings_selecting_module( + r#" + [ec] + module = "testing.seam-probe" + "#, + ); + + let registry = registry_with_probe(&settings); + + let module = registry + .ec_module() + .expect("`[ec] module = \"testing.seam-probe\"` should resolve the module's module"); + + let request_info = OwnedRequestInfo::new("192.0.2.1".to_owned(), HeaderMap::new()); + let services = stub_services(); + let method = axum::http::Method::GET; + let context = ModuleContext::new(ModuleRequest::new( + &method, + "publisher.example", + "https", + "/", + )) + .with_evidence(&request_info) + .with_services(&services); + let generated = module + .generate(context.call(module.id(), module.required_permissions())) + .await + .expect("the module's module should generate an identifier"); + + let id = generated + .id + .expect("the module's module should return an identifier"); + assert!( + id.starts_with("seam-probe-"), + "the identifier should carry the module module's prefix, got `{id}`" + ); + assert_eq!( + id, "seam-probe-192.0.2.1", + "the module's module should derive the identifier from the request evidence" + ); +} + +/// Config store that answers nothing, so a test can build [`RuntimeServices`] +/// without a real platform. +/// +/// The probe's Edge Cookie module derives its identifier from the request +/// evidence and reads none of the services, so the store is never queried. +struct StubConfigStore; + +impl PlatformConfigStore for StubConfigStore { + fn get(&self, _store_name: &StoreName, _key: &str) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// Secret store that answers nothing, paired with [`StubConfigStore`]. +struct StubSecretStore; + +impl PlatformSecretStore for StubSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// Backend manager that registers nothing, paired with the stub stores. +struct StubBackend; + +impl PlatformBackend for StubBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// Builds a minimal [`RuntimeServices`] to hand a module under test. Every +/// service is a stub or a core-provided unavailable implementation because the +/// probe's Edge Cookie module reads only the request evidence. +fn stub_services() -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(StubConfigStore)) + .secret_store(Arc::new(StubSecretStore)) + .kv_store(Arc::new(UnavailableKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(Arc::new(UnavailableHttpClient)) + .geo(Arc::new(DisabledGeo)) + .client_info(ClientInfo::default()) + .build() +} diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 2ce969ef79..6fc2220097 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -31,6 +31,12 @@ futures = { workspace = true } log = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-modules = { workspace = true } +trusted-server-permission-signal-gpc = { workspace = true } +trusted-server-permission-signal-gpp = { workspace = true } +trusted-server-permission-signal-mtm = { workspace = true } +trusted-server-permission-signal-tcf = { workspace = true } +trusted-server-permission-signal-us-privacy = { workspace = true } trusted-server-js = { workspace = true } worker = { workspace = true, optional = true } diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 87b9567e7f..696d80985c 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -5,26 +5,31 @@ use std::sync::Arc; use edgezero_core::app::Hooks; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; +use edgezero_core::http::{HeaderValue, Method, Request, Response, header}; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::attestation::{PlatformIdentity, handle_prefix, prefix_routes}; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ - AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::closed_paths::closed_path_response; #[cfg(any(test, target_arch = "wasm32"))] use trusted_server_core::config_payload::CONFIG_BLOB_KEY; #[cfg(target_arch = "wasm32")] -use trusted_server_core::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob}; -use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{ - admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, - deny_admin_diagnostic_fallback, handle_admin_eids_lookup, +use trusted_server_core::config_payload::{ + DEFAULT_SECRET_STORE_ID, settings_from_config_blob_with, }; -use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::ec::EcContext; +use trusted_server_core::ec::module::{EdgeCookieModule, build_reusable_module}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; -use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; +use trusted_server_core::inspect::config::{CONFIG_PATHS, handle_config}; +use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; +use trusted_server_core::inspect::permissions::{PERMISSIONS_PATHS, handle_permissions}; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, +}; use trusted_server_core::platform::RuntimeServices; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::StoreName; @@ -33,7 +38,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; @@ -42,7 +47,7 @@ use trusted_server_core::request_signing::{ }; use trusted_server_core::settings::Settings; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, SanitizeRequestMiddleware}; +use crate::middleware::{FinalizeResponseMiddleware, SanitizeRequestMiddleware}; use crate::platform::build_runtime_services; // --------------------------------------------------------------------------- @@ -69,12 +74,37 @@ pub fn set_cloudflare_env(env: worker::Env) { }); } -/// Application state built once at startup and shared across all requests. +/// Application state shared by everything that serves one request. +/// +/// Not built once per Worker isolate, because +/// `edgezero_adapter_cloudflare::run_app` calls `build_app` inside the +/// per-request entry point, so this is rebuilt on every request. pub struct AppState { settings: Arc, orchestrator: Arc, registry: Arc, + /// The Edge Cookie module `[ec] module` selects, resolved once here. + /// + /// This adapter runs a fresh instance per request and resolving reads no + /// request data, so the selection is resolved when the state is built + /// and handed to every request through + /// [`RuntimeServices::resolved_ec_module`](trusted_server_core::platform::RuntimeServices::resolved_ec_module), + /// rather than resolved again on the request path. + /// `None` for a deployment that selects no module. + ec_module: Option>, + /// The permission signal modules `[permission-signal] modules` selects + /// from the scheme crates this adapter links, in the order they run. + /// Selected once here so a name no crate answers to fails startup rather + /// than the first request, and handed to every request's services. + permission_signal_modules: + Arc<[Arc]>, + /// Services a caller supplied for every request, rather than services built + /// from the request context. `None` in a deployment. services: Option, + /// The loading caches the `[cache.]` tables select from the linked + /// builders, taken when this state is built and handed to every request's + /// services. + loading_caches: Arc, } /// Build the application state, loading settings and constructing all per-application components. @@ -156,7 +186,14 @@ fn settings_from_cloudflare_config_json() -> Result Result, Report> { - build_state_with_services(settings, None) + build_state_with_registrations(settings, &[]) } +/// Build the application state from explicit settings, composing the modules +/// a stock build ships with the externally supplied builders in +/// `integrations`. +/// +/// A deployment that ships a vendor crate calls this to add that crate's +/// integration builder without the adapter naming the vendor. Auction +/// providers come from the compiled auction plan, as they do without any +/// external builders. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, when the auction plan does not compile or cannot run on this +/// adapter, or when the auction orchestrator or the integration registry fail +/// to initialize, which includes two builders claiming the same integration +/// id. +pub fn build_state_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], +) -> Result, Report> { + build_state_with_registrations_and_services(settings, integrations, None) +} + +/// Build the application state with the services every request will use, +/// rather than services built per request from the request context. fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { - let plan = Arc::new(compile_auction_plan(&settings)?); + build_state_with_registrations_and_services(settings, &[], services) +} + +fn build_state_with_registrations_and_services( + settings: Settings, + integrations: &[IntegrationBuilder], + services: Option, +) -> Result, Report> { + // The modules a stock build ships come first and the deployment's own + // follow, which is the order their hooks run in. + let integrations = trusted_server_modules::builders_with(integrations); + let integrations = integrations.as_slice(); + + // The plan is compiled with the integrations this adapter was given, so an + // `[ad-server]` or `[demand]` name one of their builders supplies resolves + // here. Compiling without them would drop the implementation and report the + // name as one no builder registers. + let plan = Arc::new(compile_auction_plan_with(&settings, integrations)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; - let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; - let registry = IntegrationRegistry::with_plan(&settings, plan)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan))?; + let registry = IntegrationRegistry::with_plan_and_registrations(&settings, plan, integrations)? + .with_platform(PlatformIdentity::named("cloudflare")); + + // Composition root: resolve the module selection once, before any request + // is served, so a selection this adapter can never supply fails here rather + // than on the first request. Keeping what the resolution produced is what + // stops the request path resolving the same settings again. The registry is + // built first because a module can supply the vendor Edge Cookie module + // the selector names, and resolving without it would reject a selection + // this deployment can in fact satisfy. A caller supplying its own + // `RuntimeServices` may have resolved one already, and that one comes first + // because it is what the request path will see. This adapter supplies no + // host signals, so that argument stays `None` until it does. + let ec_module = build_reusable_module( + &settings.ec, + None, + services + .as_ref() + .and_then(RuntimeServices::resolved_ec_module) + .or_else(|| registry.ec_module()), + )?; + let permission_signal_modules = + trusted_server_core::permission_signal::build_permission_signal_modules( + &settings, + &shipped_signal_modules(), + )?; + let loading_caches = + trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), + ec_module, + permission_signal_modules, services, + loading_caches, })) } +/// The permission signal modules this adapter links, in the order they run +/// when configuration names none. Global Privacy Control is first because it +/// is a browser setting with no interface of its own, and the three that +/// carry a choice someone made through an interface follow, so an answer +/// given at a prompt amends the header the visitor arrived with. +/// +/// Core supplies no module of its own, so this is where a deployment's +/// schemes are decided. A scheme is added by linking its crate here, and a +/// scheme core has never heard of plugs in the same way. +fn shipped_signal_modules() +-> Vec> { + vec![ + Arc::new(trusted_server_permission_signal_gpc::GpcModule::new()), + Arc::new(trusted_server_permission_signal_gpp::GppSaleOptOutModule::new()), + Arc::new(trusted_server_permission_signal_us_privacy::UsPrivacyModule::new()), + Arc::new(trusted_server_permission_signal_tcf::TcfModule::new()), + Arc::new(trusted_server_permission_signal_mtm::MtmModule::new()), + ] +} + impl AppState { + /// Build per-request [`RuntimeServices`], carrying the Edge Cookie module + /// the composition root already resolved and applying the module-supplied + /// geo, Edge Cookie and device modules selected by `[geo]`, `[ec]` and + /// `[device] module`. + /// + /// No Edge Cookie module is carried when the composition root found + /// nothing safe to keep, so the request path resolves the selection for + /// itself. + /// + /// For geo, unset and `"none"` both resolve nothing, so no client IP + /// reaches a host geo service. `"platform"` opts in to this adapter's own + /// lookup, and any other key names an integration module that declares a + /// geo module. Identity and device are applied the same way when a module + /// supplies them. fn services_for_request(&self, ctx: &RequestContext) -> RuntimeServices { - self.services + let mut services = self + .services .clone() - .unwrap_or_else(|| build_runtime_services(ctx)) + .unwrap_or_else(|| { + build_runtime_services(ctx, &self.settings, &self.permission_signal_modules) + .with_loading_caches(Arc::clone(&self.loading_caches)) + }) + .with_resolved_ec_module(self.ec_module.clone()); + if let Some(module) = self.registry.geo_module() { + services = services.with_geo(module); + } + if let Some(module) = self.registry.ec_module() { + services = services.with_ec_module(module); + } + if let Some(module) = self.registry.device_module() { + services = services.with_device_module(module); + } + // The publisher this request resolved to names every key in a shared + // store and every name the browser holds. + let prefixes = self.registry.prefixes( + &self.settings, + trusted_server_core::module_context::ModuleRequest::of( + ctx.request(), + services.client_info(), + ), + ); + services.with_prefixes(prefixes) } } @@ -231,24 +399,30 @@ impl AppState { /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, /// `/_ts/page-bids`, and the publisher fallback). /// -/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction -/// Unknown, which fails the auction consent gate closed even for consented -/// users. Geo comes from the Workers `cf` object when deployed. A malformed -/// consent string is logged and falls back to the default (fail-closed) context -/// rather than being silently swallowed. -fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) - .unwrap_or_else(|e| { - log::warn!("EC context read failed: {e:?}"); - EcContext::default() - }) +/// The geo lookup runs inside +/// [`EcContext::read_from_request_resolving_geo`], so every adapter reports the +/// same distinction: no location falls back to the top of the +/// `permissions.yaml` rules tree, while a failed lookup resolves every +/// permission at the requires-signal floor and is logged at error level. +/// Geo comes from the Workers `cf` object when deployed. +/// +/// Mirrors the Fastly entry point, which keeps the report and answers with an +/// error response: when the Edge Cookie context cannot be read the request +/// fails rather than continuing with `EcContext::default()`, which would serve +/// every request with no identity. A malformed cookie value, a bad consent +/// string and a failed geo lookup do not reach this error path at all, so +/// failing here does not fail requests for ordinary parse problems. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this request, or when the request's `Cookie` header is not valid UTF-8. +async fn build_ec_context( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> Result> { + EcContext::read_from_request_resolving_geo(settings, req, services).await } // --------------------------------------------------------------------------- @@ -277,10 +451,7 @@ where Box::pin(async move { let services = s.services_for_request(&ctx); let mut req = ctx.into_request(); - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &s.settings, - &mut req, - ) { + if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { return Ok(http_error(&error)); } Ok(f(s, services, req).await.unwrap_or_else(|e| http_error(&e))) @@ -338,55 +509,6 @@ pub(crate) fn http_error(report: &Report) -> Response { response } -fn admin_key_management_not_supported() -> Response { - let body = edgezero_core::body::Body::from( - "Admin key management is not supported on Cloudflare Workers.\n\ - Use the Fastly adapter (via Viceroy or deployed) to rotate or deactivate keys.\n", - ); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::NOT_IMPLEMENTED; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - -fn cache_purge_not_supported() -> Response { - let body = edgezero_core::body::Body::from( - "Template cache purge is not supported on Cloudflare Workers.\n\ - Use the Fastly adapter (via Viceroy or deployed) to purge.\n", - ); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::NOT_IMPLEMENTED; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - -fn admin_ec_lookup_not_supported() -> Response { - core_admin_ec_lookup_not_supported() -} - -/// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` -/// aliases on the Cloudflare adapter. -/// -/// These non-`/_ts` aliases are not matched by the `^/_ts/admin` basic-auth -/// handler, so they fail closed locally rather than fall through to the -/// publisher fallback — which would forward the caller's `Authorization` header -/// and key-management payload to the origin, leaking admin credentials. -fn legacy_admin_alias_denied() -> Response { - let mut response = Response::new(edgezero_core::body::Body::from("Not found\n")); - *response.status_mut() = edgezero_core::http::StatusCode::NOT_FOUND; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - // --------------------------------------------------------------------------- // Startup error fallback // --------------------------------------------------------------------------- @@ -477,6 +599,28 @@ impl TrustedServerApp { Ok(build_router(&state)) } + /// Build the full application router from explicit settings, composing the + /// built-in integrations with the externally supplied builders in + /// `integrations`. + /// + /// The route table is the one [`TrustedServerApp::routes_with_settings`] + /// builds, so a composed deployment routes exactly as the plain one does. + /// + /// # Errors + /// + /// Returns an error when the selected Edge Cookie module cannot be built + /// for this adapter, when the auction plan does not compile or cannot run + /// on this adapter, or when the auction orchestrator or the integration + /// registry fail to initialize, which includes two builders claiming the + /// same integration id. + pub fn routes_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], + ) -> Result> { + let state = build_state_with_registrations(settings, integrations)?; + Ok(build_router(&state)) + } + /// Build the full router with explicit settings and runtime services. /// /// Each request receives a clone of the supplied services, allowing callers @@ -513,13 +657,10 @@ fn build_router(state: &Arc) -> RouterService { ) -> Result { let services = state.services_for_request(&ctx); let mut req = ctx.into_request(); - if let Some(response) = deny_admin_diagnostic_fallback(&req) { + if let Some(response) = closed_path_response(&req) { return Ok(response); } - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); } let path = req.uri().path().to_owned(); @@ -534,7 +675,13 @@ fn build_router(state: &Arc) -> RouterService { EdgeCacheHeader::CloudflareCdnCacheControl, ) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); + // A module route is handed what was resolved for its request, + // the permissions among them, as the publisher path is. + let mut ec_context = match build_ec_context(&state.settings, &services, &req).await + { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; state .registry .handle_proxy(ProxyDispatchInput { @@ -553,14 +700,24 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - let mut ec_context = build_ec_context(&state.settings, &services, &req); + // Identity could not be established (for example the selected + // Edge Cookie module is unavailable). Answer with an error + // rather than serving the page with no identity. + let mut ec_context = match build_ec_context(&state.settings, &services, &req).await + { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, }; match handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, &services, None, &mut ec_context, @@ -603,7 +760,6 @@ fn build_router(state: &Arc) -> RouterService { // shared-secret authentication header. .middleware(SanitizeRequestMiddleware::new(Arc::clone(&state.settings))) .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) - .middleware(AuthMiddleware::new(Arc::clone(&state.settings))) .get( "/.well-known/trusted-server.json", make_handler(Arc::clone(&state), |s, services, req| async move { @@ -616,49 +772,13 @@ fn build_router(state: &Arc) -> RouterService { handle_verify_signature(&s.settings, &services, req) }), ) - // Canonical admin key routes. These match `Settings::ADMIN_ENDPOINTS` - // and the production basic-auth handler regex (`^/_ts/admin`), so they - // are auth-gated under a production-shaped config. - // - // The legacy non-`/_ts` aliases (`/admin/keys/*`) are registered - // below to a local 404 for every publisher-fallback method: the - // production handler regex `^/_ts/admin` does not match them, and - // letting them fall through would forward the caller's - // `Authorization` header and key-management payload to the origin, - // leaking admin credentials. - .post("/_ts/admin/keys/rotate", |_ctx: RequestContext| async { - Ok::(admin_key_management_not_supported()) - }) - .post("/_ts/admin/keys/deactivate", |_ctx: RequestContext| async { - Ok::(admin_key_management_not_supported()) - }) - // Admin EC lookup routes. Registered explicitly (like the key - // routes above) so they never fall through to the publisher - // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth - // coverage. The EC identity graph is Fastly KV backed, so this - // adapter has no store to read. - .get("/_ts/admin/ec", |_ctx: RequestContext| async { - Ok::(admin_ec_lookup_not_supported()) - }) - .get("/_ts/admin/ec/{id}", |_ctx: RequestContext| async { - Ok::(admin_ec_lookup_not_supported()) - }) - // Admin EIDs echo: pure request inspection (no KV), so this - // adapter serves the real handler. - .get( - "/_ts/admin/eids", - make_handler(Arc::clone(&state), |s, _services, req| async move { - let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; - handle_admin_eids_lookup(&partner_registry, &req) - }), - ) .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { // Build the geo-aware EC context so the auction consent gate // sees the caller's jurisdiction — `EcContext::default()` // fails it closed for consented users. - let mut ec_context = build_ec_context(&s.settings, &services, &req); + let mut ec_context = build_ec_context(&s.settings, &services, &req).await?; handle_auction( &s.settings, &s.orchestrator, @@ -722,7 +842,7 @@ fn build_router(state: &Arc) -> RouterService { // preflight fall through to a permissive origin would reopen exactly // the cross-site hole the canonical path closes. let page_bids = make_handler(Arc::clone(&state), |s, services, req| async move { - let mut ec_context = build_ec_context(&s.settings, &services, &req); + let mut ec_context = build_ec_context(&s.settings, &services, &req).await?; let auction = AuctionDispatch { orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), @@ -739,29 +859,47 @@ fn build_router(state: &Arc) -> RouterService { router = router.route(path, Method::OPTIONS, page_bids_preflight.clone()); } - let cache_purge_unsupported = - make_handler(Arc::clone(&state), |_s, _services, _req| async move { - Ok(cache_purge_not_supported()) - }); - for method in publisher_fallback_methods() { - router = router.route( - "/_ts/admin/cache/purge", - method, - cache_purge_unsupported.clone(), - ); + // What the deployment decided for the asking request, shown to anyone + // as a page and as data. Read only, so nothing is written for the + // reader. + let permissions = make_handler(Arc::clone(&state), |s, services, req| async move { + handle_permissions(&s.settings, &services, &req).await + }); + for path in PERMISSIONS_PATHS { + router = router.route(path, Method::GET, permissions.clone()); } - let legacy_admin_deny = - make_handler(Arc::clone(&state), |_s, _services, _req| async move { - Ok(legacy_admin_alias_denied()) + // The settings the deployment is running, masked, shown to anyone as + // a page and as data. + let config = make_handler(Arc::clone(&state), |s, _services, req| async move { + Ok(handle_config(&s.settings, &req)) + }); + for path in CONFIG_PATHS { + router = router.route(path, Method::GET, config.clone()); + } + + // What is held against the request's own Edge Cookie. This adapter + // keeps no identity graph, so the page says nothing is held. + let data = make_handler(Arc::clone(&state), |_s, services, req| async move { + handle_data(None, None, services.public_prefix(), &req) + }); + router = router.route(DATA_PAGE_PATH, Method::GET, data); + + // Routes chosen by the settings, so not in the fixed table. + let routes = prefix_routes(&state.settings); + if !routes.is_empty() { + let attestation = make_handler(Arc::clone(&state), |s, _services, req| async move { + Ok(handle_prefix(&s.settings, s.registry.platform(), &req)) }); - for method in publisher_fallback_methods() { - router = router.route( - "/admin/keys/rotate", - method.clone(), - legacy_admin_deny.clone(), - ); - router = router.route("/admin/keys/deactivate", method, legacy_admin_deny.clone()); + for route in &routes { + for method in publisher_fallback_methods() { + if matches!(method, Method::GET | Method::HEAD) || route.every_method { + router = router.route(&route.path, method, attestation.clone()); + } else { + router = router.route(&route.path, method, fallback.clone()); + } + } + } } for method in publisher_fallback_methods() { @@ -775,16 +913,71 @@ fn build_router(state: &Arc) -> RouterService { #[cfg(test)] mod tests { + use edgezero_core::http::request_builder; + use edgezero_core::params::PathParams; + use super::*; + /// Settings selecting a vendor Edge Cookie module this adapter does not + /// inject, with the `[ec.acme]` block that module's settings live in. + /// `acme` is a fictional vendor key. + const UNINJECTED_MODULE_TOML: &str = r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + module = "acme" + + [ec.acme] + endpoint = "https://ec.acme.example.com" + + # An Edge Cookie module is configured, so single-jurisdiction + # operation is acknowledged because no geo module is selected. + [geo] + assume_single_jurisdiction = true + "#; + + /// The per-request Edge Cookie read must return its error rather than a + /// default context. + /// + /// Continuing with `EcContext::default()` would serve every request with + /// no identity when the selected module cannot be built. The call sites + /// propagate the error to `http_error`, matching the Fastly adapter. The + /// settings are parsed directly, bypassing the composition root's startup + /// check, so the per-request behavior can be exercised with a selection + /// the adapter cannot supply. + #[tokio::test] + async fn build_ec_context_fails_when_the_selected_module_is_unavailable() { + let settings = Settings::from_toml(UNINJECTED_MODULE_TOML) + .expect("should parse settings selecting an uninjected module"); + let req = request_builder() + .method("POST") + .uri("https://test-publisher.example.com/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build test request"); + let ctx = RequestContext::new(req, PathParams::default()); + // No resolved module is threaded here, so the request path resolves + // the selection itself, which is what an embedder driving core + // directly does and where the loud failure has to stay. + let services = build_runtime_services(&ctx, &settings, &Arc::default()); + let req = ctx.into_request(); + + let error = build_ec_context(&settings, &services, &req) + .await + .expect_err("an unavailable Edge Cookie module must fail the request"); + + assert!( + error.to_string().contains("acme"), + "the error should name the selected module, got: {error}" + ); + } + fn aps_profile_settings() -> Settings { let mut settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-password" - [publisher] domain = "publisher.example" cookie_domain = ".publisher.example" @@ -793,24 +986,64 @@ mod tests { [ec] passphrase = "fictional-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse startup test settings"); - settings.auction.providers.insert( - "aps-main".parse().expect("should parse APS provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: trusted_server_core::auction::NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, + settings.demand = trusted_server_core::provider_table::ProviderList::new( + vec!["aps_main".to_string()], + std::collections::BTreeMap::from([( + "aps_main".to_string(), + serde_json::Map::from_iter([ + ( + "implementation".to_string(), + serde_json::json!("auction.aps"), + ), + ( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/pb/bid"), + ), + ("routing".to_string(), serde_json::json!("all_eligible")), + ( + "account_id".to_string(), + serde_json::json!("example-account"), + ), + ]), + )]), ); settings } + /// Two ordinary `OpenRTB` demand sources, which is the fanout Cloudflare + /// refuses because its HTTP client runs one request at a time. + fn two_source_demand() -> trusted_server_core::provider_table::ProviderList { + let names = ["provider_a", "provider_b"]; + trusted_server_core::provider_table::ProviderList::new( + names.iter().map(|name| (*name).to_string()).collect(), + names + .iter() + .map(|name| { + ( + (*name).to_string(), + serde_json::Map::from_iter([ + ( + "implementation".to_string(), + serde_json::json!("auction-protocol.openrtb"), + ), + ( + "endpoint".to_string(), + serde_json::json!(format!("https://{name}.example/openrtb")), + ), + ("routing".to_string(), serde_json::json!("all_eligible")), + ]), + ) + }) + .collect(), + ) + } + #[test] fn startup_registers_aps_renderer_route() { let state = build_state_with_settings(aps_profile_settings()) @@ -824,6 +1057,41 @@ mod tests { ); } + /// Every request's services carry the prefixes of the publisher its + /// settings describe, which start every shared store key and every name + /// the browser holds. + #[test] + fn a_requests_services_carry_the_publishers_prefixes() { + let settings = Settings::from_toml( + r#" + [publisher] + domain = "www.publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret-32-bytes-ok" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse the settings"); + let state = build_state_with_settings(settings).expect("should build the state"); + let request = request_builder() + .method("GET") + .uri("https://www.publisher.example/") + .body(edgezero_core::body::Body::empty()) + .expect("should build the request"); + let ctx = RequestContext::new(request, PathParams::default()); + + let services = state.services_for_request(&ctx); + + assert_eq!( + services.server_key_prefix().as_str(), + "ts.publisher.example" + ); + assert_eq!(services.public_prefix().as_str(), "ts"); + } + #[test] fn cloudflare_config_prefers_manifest_default_key() { let value = serde_json::json!({ @@ -842,11 +1110,6 @@ mod tests { fn disabled_startup_accepts_dormant_multi_provider_auction_plan() { let mut settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-password" - [publisher] domain = "publisher.example" cookie_domain = ".publisher.example" @@ -855,28 +1118,14 @@ mod tests { [ec] passphrase = "fictional-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse startup test settings"); settings.auction.enabled = false; - settings.auction.providers = - std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) - .map(|id| { - ( - id.parse().expect("should parse provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: format!("https://{id}.example/openrtb"), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: - trusted_server_core::auction::NotificationConfig::default(), - profile_config: serde_json::json!({}), - }, - ) - }) - .collect(); + settings.demand = two_source_demand(); build_state_with_settings(settings) .expect("disabled Cloudflare auction should accept dormant fanout"); @@ -886,11 +1135,6 @@ mod tests { fn startup_rejects_multi_provider_auction_plan() { let mut settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-password" - [publisher] domain = "publisher.example" cookie_domain = ".publisher.example" @@ -899,28 +1143,14 @@ mod tests { [ec] passphrase = "fictional-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse startup test settings"); settings.auction.enabled = true; - settings.auction.providers = - std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) - .map(|id| { - ( - id.parse().expect("should parse provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: format!("https://{id}.example/openrtb"), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: - trusted_server_core::auction::NotificationConfig::default(), - profile_config: serde_json::json!({}), - }, - ) - }) - .collect(); + settings.demand = two_source_demand(); let error = match build_state_with_settings(settings) { Ok(_) => panic!("Cloudflare startup should reject multi-provider fanout"), diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index a22b51982e..3fa9f06aaa 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -5,9 +5,10 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; -use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; -use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; +use trusted_server_core::http_util::{ + ROUTE_NAMESPACE, sanitize_trusted_client_ip_headers, strip_deployment_headers, +}; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -56,9 +57,8 @@ impl Middleware for SanitizeRequestMiddleware { /// (injected by the Cloudflare Workers runtime). On the native host target the /// header is absent, so `X-Geo-Info-Available: false` is emitted. /// -/// Registered directly inside [`SanitizeRequestMiddleware`] and ahead of -/// [`AuthMiddleware`] so that every outgoing response — including auth-rejected -/// ones — carries a consistent set of headers. +/// Registered directly inside [`SanitizeRequestMiddleware`], so that every +/// outgoing response carries a consistent set of headers. pub struct FinalizeResponseMiddleware { settings: Arc, } @@ -82,50 +82,15 @@ impl Middleware for FinalizeResponseMiddleware { .filter(|s| !s.is_empty() && *s != "XX") .is_some(); + let path = ctx.request().uri().path().to_owned(); let mut response = next.run(ctx).await?; + // Before finalizing, so a header finalizing writes is kept. + strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, geo_available, &mut response); Ok(response) } } -// --------------------------------------------------------------------------- -// AuthMiddleware -// --------------------------------------------------------------------------- - -/// Inner middleware: enforces basic-auth before the handler runs. -/// -/// - `Ok(Some(response))` from [`enforce_basic_auth`] → auth failed; return the -/// challenge response (bubbles through [`FinalizeResponseMiddleware`] for header injection). -/// - `Ok(None)` → no auth required or credentials accepted; continue the chain. -/// - `Err(report)` → internal error; log and convert to a 500 HTTP response. -pub struct AuthMiddleware { - settings: Arc, -} - -impl AuthMiddleware { - /// Creates a new [`AuthMiddleware`] with the given settings. - #[must_use] - pub fn new(settings: Arc) -> Self { - Self { settings } - } -} - -#[async_trait(?Send)] -impl Middleware for AuthMiddleware { - async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { - Ok(Some(response)) => return Ok(response), - Ok(None) => {} - Err(report) => { - log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); - } - } - - next.run(ctx).await - } -} - // --------------------------------------------------------------------------- // apply_finalize_headers — extracted for unit testing // --------------------------------------------------------------------------- @@ -146,14 +111,16 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - trusted_server_core::version_header::apply_git_version_header(response); - // Cloudflare is a real shared cache: cookie-bearing responses must stay // private and operator headers must not re-enable caching for uncacheable // per-user payloads. trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( settings, response, ); + + // Last, so that nothing an origin or an operator's header set can loosen + // a site taken out of search. + trusted_server_core::robots_txt::apply_response_robots_tag(settings, response); } // --------------------------------------------------------------------------- @@ -199,11 +166,6 @@ mod tests { // by design. let mut s = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -211,7 +173,13 @@ mod tests { proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should load test settings"); @@ -323,21 +291,58 @@ mod tests { ); } + fn ctx_for(path: &str) -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri(path) + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } + + /// A response as an origin behind a cache sends it, naming the node that + /// answered and how the cache treated the request. + fn response_naming_its_node() -> Response { + response_builder() + .header("x-served-by", "cache-node-1") + .header("x-cache", "HIT") + .header("x-cache-hits", "3") + .body(Body::empty()) + .expect("should build test response") + } + + /// What the finalize middleware makes of that response for a request to + /// `path`. + fn finalized(path: &str, settings: Settings) -> Response { + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let handler = Arc::new(|_ctx: RequestContext| async move { + Ok::(response_naming_its_node()) + }); + + block_on(middleware.handle(ctx_for(path), Next::new(&[], &*handler))) + .expect("should run middleware") + } + #[test] - fn emits_git_version_header() { - let mut response = empty_response(); - apply_finalize_headers( - &settings_with_response_headers(vec![]), - false, - &mut response, + fn finalize_middleware_strips_deployment_headers_before_it_writes_the_operators() { + let response = finalized( + "/articles/an-article", + settings_with_response_headers(vec![("X-Cache", "configured")]), ); + + for name in ["x-served-by", "x-cache-hits"] { + assert!( + !response.headers().contains_key(name), + "should strip {name} from a publisher's page" + ); + } assert_eq!( response .headers() - .get("x-ts-version") + .get("x-cache") .and_then(|v| v.to_str().ok()), - trusted_server_core::constants::TS_GIT_VERSION, - "should report the compiled-in git version as x-ts-version" + Some("configured"), + "should carry the operator's header and not the origin's" ); } } diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 94616b2035..e83ce1c584 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -594,7 +594,13 @@ impl PlatformSecretStore for CloudflareSecretStoreAdapter { /// Geo information is read from Cloudflare's injected request headers /// (`cf-ipcountry`, etc.) which are present on all plans; headers absent on /// the native host target simply produce empty/zero defaults. -pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { +pub fn build_runtime_services( + ctx: &edgezero_core::context::RequestContext, + settings: &trusted_server_core::settings::Settings, + permission_signal_modules: &Arc< + [Arc], + >, +) -> RuntimeServices { let client_ip = extract_client_ip(ctx); #[cfg(target_arch = "wasm32")] @@ -629,7 +635,9 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R // Geo: read Cloudflare-injected headers — no #[cfg] needed; headers are // simply absent on the native host target, producing Ok(None) from lookup(). - let geo = build_geo(ctx); + // Routed through the [geo] module selector like the Fastly adapter, so + // the selector behaves the same on every adapter. + let geo = trusted_server_core::platform::build_geo_module(settings, Arc::new(build_geo(ctx))); RuntimeServices::builder() .config_store(config_store) @@ -637,7 +645,11 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .kv_store(kv_store) .backend(Arc::new(NoopBackend)) .http_client(http_client) - .geo(Arc::new(geo)) + .geo(geo) + // The signal modules were selected once at startup from the scheme + // crates this adapter links, so every request asks exactly the ones + // configuration named, in that order. + .permission_signal_modules(Arc::clone(permission_signal_modules)) .client_info(ClientInfo { client_ip, tls_protocol: None, @@ -654,19 +666,32 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R /// Reads Cloudflare geo headers injected by the Workers runtime. /// /// `cf-ipcountry` is available on all plans. `cf-ipcity`, `cf-ipcontinent`, -/// `cf-iplatitude`, and `cf-iplongitude` require an Enterprise plan. Absent or -/// unparseable values default to empty strings or `0.0`. Country code `XX` -/// (Cloudflare's "unknown" sentinel) is treated as absent. +/// `cf-iplatitude`, `cf-iplongitude` and `cf-region-code` require an Enterprise +/// plan and the visitor-location managed transform. Absent or unparseable +/// values default to empty strings or `0.0`. Country code `XX` (Cloudflare's +/// "unknown" sentinel) is treated as absent. +/// +/// The region is the ISO 3166-2 subdivision code from `cf-region-code`, for +/// example `CA`, and not the subdivision name from `cf-region`, because the +/// region nodes of the `permissions.yaml` rules tree, including the US states +/// that carry `jurisdiction: us-state`, are written as two-letter codes and a +/// name would never match one. struct CloudflareGeo { country: String, city: String, continent: String, latitude: f64, longitude: f64, + region: Option, } +#[async_trait::async_trait(?Send)] impl PlatformGeo for CloudflareGeo { - fn lookup(&self, _client_ip: Option) -> Result, Report> { + async fn lookup( + &self, + _client_ip: Option, + _services: &trusted_server_core::platform::RuntimeServices, + ) -> Result, Report> { if self.country.is_empty() { return Ok(None); } @@ -677,7 +702,7 @@ impl PlatformGeo for CloudflareGeo { latitude: self.latitude, longitude: self.longitude, metro_code: 0, - region: None, + region: self.region.clone(), asn: None, })) } @@ -711,12 +736,19 @@ fn build_geo(ctx: &edgezero_core::context::RequestContext) -> CloudflareGeo { .and_then(|v| v.to_str().ok()) .and_then(|s| s.parse::().ok()) .unwrap_or(0.0); + let region = headers + .get("cf-region-code") + .and_then(|v| v.to_str().ok()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned); CloudflareGeo { country, city, continent, latitude, longitude, + region, } } @@ -780,6 +812,22 @@ mod tests { use edgezero_core::http::{HeaderValue, request_builder}; use edgezero_core::params::PathParams; + /// The services graph a module is handed, built the same way the request + /// path builds it so the tests exercise the production shape. + fn test_services() -> RuntimeServices { + let req = edgezero_core::http::request_builder() + .method("GET") + .uri("https://example.com/") + .body(edgezero_core::body::Body::empty()) + .expect("should build test request"); + let ctx = RequestContext::new(req, PathParams::default()); + build_runtime_services( + &ctx, + &trusted_server_core::settings::Settings::default(), + &Arc::default(), + ) + } + fn make_ctx_with_header(name: &str, value: &str) -> RequestContext { let req = request_builder() .method("GET") @@ -793,6 +841,79 @@ mod tests { RequestContext::new(req, PathParams::default()) } + /// Builds a request context carrying multiple headers at once. + fn make_ctx_with_headers(headers: &[(&str, &str)]) -> RequestContext { + let mut builder = request_builder().method("GET").uri("https://example.com/"); + for (name, value) in headers { + builder = builder.header( + *name, + HeaderValue::from_str(value).expect("should parse test header value"), + ); + } + let req = builder + .body(edgezero_core::body::Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::default()) + } + + #[tokio::test] + async fn a_us_state_visitor_reaches_the_us_state_jurisdiction_and_its_opt_out() { + // The region header has to reach the privacy outcome. + // `detect_jurisdiction` reaches a US state node of the policy tree + // only when the country is `US` and a region is present. With no + // region a US visitor falls through to the country node's + // `NonRegulated`, where `allows_ec_creation` returns true without + // reading `ctx.gpc`, so a Sec-GPC opt-out would be ignored. + let ctx = make_ctx_with_headers(&[("cf-ipcountry", "US"), ("cf-region-code", "CA")]); + let geo = build_geo(&ctx) + .lookup(None, &test_services()) + .await + .expect("should look up without failing") + .expect("a country header should resolve a location"); + assert_eq!( + geo.region.as_deref(), + Some("CA"), + "the ISO 3166-2 subdivision code should reach the geo info" + ); + + let jurisdiction = + trusted_server_core::consent::jurisdiction::detect_jurisdiction(Some(&geo)); + assert_eq!( + jurisdiction, + trusted_server_core::consent::jurisdiction::Jurisdiction::UsState("CA".to_owned()), + "a Californian visitor should reach the US state jurisdiction" + ); + + // Reaching that jurisdiction is what this adapter is responsible for, + // and it is the switch every downstream opt-out hangs off. A Sec-GPC + // signal, a GPP US sale opt-out and a US Privacy opt-out are all + // consulted on the US state branch and none of them are consulted on + // the unregulated one, so a visitor who never reaches the US state + // jurisdiction has every one of those signals ignored. Which gate reads + // the jurisdiction is core's business and changes across this stack, so + // it is core that tests the reading. + + // Without the region header nothing can place the visitor in a state, + // so the jurisdiction is the unregulated one and the same opt-out is + // ignored. That is the behavior this fix removes for any deployment + // whose plan supplies the header. + let ctx = make_ctx_with_headers(&[("cf-ipcountry", "US")]); + let geo = build_geo(&ctx) + .lookup(None, &test_services()) + .await + .expect("should look up without failing") + .expect("a country header should resolve a location"); + assert_eq!( + geo.region, None, + "no region header should mean no region, not an invented one" + ); + assert_eq!( + trusted_server_core::consent::jurisdiction::detect_jurisdiction(Some(&geo)), + trusted_server_core::consent::jurisdiction::Jurisdiction::NonRegulated, + "with no region a US visitor cannot be placed in a privacy state" + ); + } + fn make_ctx_without_header() -> RequestContext { let req = request_builder() .method("GET") @@ -845,20 +966,21 @@ mod tests { assert!(geo.country.is_empty(), "XX should be treated as absent"); } - #[test] - fn build_geo_lookup_returns_none_when_country_absent() { + #[tokio::test] + async fn build_geo_lookup_returns_none_when_country_absent() { let ctx = make_ctx_without_header(); let geo = build_geo(&ctx); assert!( - geo.lookup(None) + geo.lookup(None, &test_services()) + .await .expect("should perform geo lookup") .is_none(), "should return None when no country header" ); } - #[test] - fn build_geo_lookup_returns_some_with_populated_country() { + #[tokio::test] + async fn build_geo_lookup_returns_some_with_populated_country() { let req = request_builder() .method("GET") .uri("https://example.com/") @@ -872,7 +994,8 @@ mod tests { let ctx = RequestContext::new(req, PathParams::default()); let geo = build_geo(&ctx); let info = geo - .lookup(None) + .lookup(None, &test_services()) + .await .expect("should perform geo lookup") .expect("should return GeoInfo when country is set"); assert_eq!(info.country, "US", "should populate country"); diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index f385bf5295..824c8e9fa8 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -18,17 +18,9 @@ const LEGACY_ADMIN_DENY_METHODS: &[&str] = /// The settings baked into the binary contain placeholder secrets that /// `get_settings()` rejects by design, which would turn every route into a /// startup error page (and its route table into the fallback-only set). -/// The handler regex is the production-shaped `^/_ts/admin`, matching -/// `Settings::ADMIN_ENDPOINTS` and the default config, so the canonical -/// `/_ts/admin/keys/*` routes are auth-gated exactly as in production. fn test_router() -> RouterService { let settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -36,7 +28,13 @@ fn test_router() -> RouterService { proxy_secret = "route-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse route test settings"); @@ -73,18 +71,19 @@ fn assert_route_registered(method: &str, path: &str) { fn make_router() -> RouterService { let settings = trusted_server_core::settings::Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" origin_url = "https://origin.test-publisher.example.com" proxy_secret = "integration-test-proxy-secret" + [geo] + assume_single_jurisdiction = true + [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" "#, ) @@ -102,8 +101,8 @@ fn routes_build_without_panic() { } // --------------------------------------------------------------------------- -// Middleware regression tests — verify FinalizeResponseMiddleware and -// AuthMiddleware are wired so they cannot be removed silently. +// Middleware regression tests, which verify FinalizeResponseMiddleware is +// wired so it cannot be removed silently. // --------------------------------------------------------------------------- #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -127,42 +126,18 @@ async fn finalize_middleware_injects_geo_header() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auth_middleware_runs_in_chain_for_protected_routes() { - // Verifies that AuthMiddleware is wired by asserting the 401 + WWW-Authenticate - // challenge on a protected route (/_ts/admin/keys/rotate). Only AuthMiddleware - // short-circuits with this response — FinalizeResponseMiddleware alone would not. - let router = test_router(); - - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - - let resp = route(router, req).await; - - assert_eq!( - resp.status().as_u16(), - 401, - "AuthMiddleware must short-circuit with 401 on protected routes without credentials" - ); - assert!( - resp.headers().contains_key("www-authenticate"), - "AuthMiddleware must include WWW-Authenticate on 401 responses" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { - // Regression for the credential-leak finding: the production basic-auth regex - // `^/_ts/admin` does not match `/admin/keys/*`, so those aliases are not - // auth-gated. Any publisher-fallback method carrying an `Authorization` - // header must be denied locally with 404, never proxied to the publisher - // origin (which would leak the admin credentials and key body). A - // publisher-fallback proxy without a backend would surface as a 5xx, so 404 - // proves the local deny ran. - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { +async fn key_administration_paths_are_answered_here_and_never_proxied() { + // A request to a key administration path carries an operator's + // credentials and a key payload. Every publisher-fallback method must be + // answered here with 404 and never proxied to the publisher origin, which + // would be handed both. A publisher-fallback proxy without a backend would + // surface as a 5xx, so a 404 proves the request was answered here. + for path in [ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/admin/keys/rotate", + "/admin/keys/deactivate", + ] { for method in LEGACY_ADMIN_DENY_METHODS { let router = test_router(); let req = request_builder() @@ -171,14 +146,14 @@ async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") .header("content-type", "application/json") .body(edgezero_core::body::Body::from("{\"key_id\":\"leak-me\"}")) - .expect("should build authorized legacy-alias request"); + .expect("should build authorized key administration request"); let resp = route(router, req).await; assert_eq!( resp.status().as_u16(), 404, - "legacy {method} {path} with Authorization must be denied locally (404), not proxied to publisher" + "{method} {path} with Authorization must be answered here (404), not proxied to publisher" ); } } @@ -206,7 +181,9 @@ async fn tsjs_route_is_routed_not_5xx() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tsjs_route_emits_cloudflare_cache_header_for_matching_hash() { let router = test_router(); - let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); + let src = trusted_server_core::tsjs::tsjs_script_src( + &trusted_server_core::tsjs_bundle::compile_time_parts(&["creative"]), + ); let req = request_builder() .method("GET") .uri(src) @@ -250,11 +227,11 @@ fn all_explicit_routes_are_registered() { let expected: &[(&str, &str)] = &[ ("GET", "/.well-known/trusted-server.json"), ("POST", "/verify-signature"), - ("POST", "/_ts/admin/keys/rotate"), - ("POST", "/_ts/admin/keys/deactivate"), - ("GET", "/_ts/admin/ec"), - ("GET", "/_ts/admin/ec/{id}"), - ("GET", "/_ts/admin/eids"), + ("GET", "/_ts/permissions"), + ("GET", "/_ts/permissions.json"), + ("GET", "/_ts/config"), + ("GET", "/_ts/config.json"), + ("GET", "/_ts/data"), ("POST", "/auction"), // SPA re-auction endpoint, plus its deprecated `/__ts/` alias. Both // paths are spelled out as literals rather than referencing @@ -277,155 +254,19 @@ fn all_explicit_routes_are_registered() { for (method, path) in expected { assert_route_registered(method, path); } - - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - for method in LEGACY_ADMIN_DENY_METHODS { - assert_route_registered(method, path); - } - } } // --------------------------------------------------------------------------- -// Basic-auth parity tests +// Closed paths and the inspection endpoints // --------------------------------------------------------------------------- #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_routes_return_501() { - for (path, body) in [ - ("/_ts/admin/keys/rotate", "{}"), - ( - "/_ts/admin/keys/deactivate", - r#"{"kid":"test-key","delete":false}"#, - ), - ] { - let req = request_builder() - .method("POST") - .uri(path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(body)) - .expect("should build request"); - let resp = route(test_router(), req).await; - - assert_eq!( - resp.status().as_u16(), - 501, - "{path} should report that Cloudflare key management is unsupported" - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_ec_routes_return_501() { - // The EC identity graph is Fastly KV backed, so Cloudflare answers the - // admin EC lookup routes locally with 501 instead of letting them fall - // through to the publisher fallback. - let sample_ec_id = format!("{}.abc123", "a".repeat(64)); - for path in [ - "/_ts/admin/ec".to_owned(), - format!("/_ts/admin/ec/{sample_ec_id}"), - ] { - let req = request_builder() - .method("GET") - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = route(test_router(), req).await; - - assert_eq!( - resp.status().as_u16(), - 501, - "{path} should report that Cloudflare EC lookup is unsupported" - ); - assert_eq!( - resp.headers() - .get("content-type") - .and_then(|v| v.to_str().ok()), - Some("application/json") - ); - assert_eq!( - resp.headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_ec_route_without_credentials_returns_401() { - let req = request_builder() - .method("GET") - .uri("/_ts/admin/ec") - .body(edgezero_core::body::Body::empty()) - .expect("should build unauthenticated admin EC request"); - let resp = route(test_router(), req).await; - - assert_eq!(resp.status().as_u16(), 401); - assert!( - resp.headers().contains_key("www-authenticate"), - "admin EC 401 should include the Basic authentication challenge" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_eids_route_returns_200() { - // The EIDs echo is pure request inspection (no KV), so this adapter - // serves the real handler. - let req = request_builder() - .method("GET") - .uri("/_ts/admin/eids") - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = route(test_router(), req).await; - - assert_eq!( - resp.status().as_u16(), - 200, - "/_ts/admin/eids should serve the real EIDs echo handler" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { +async fn an_authenticated_request_to_a_closed_path_is_answered_here() { let ec_id = format!("{}.abc123", "a".repeat(64)); - let valid_paths = [ + for path in [ "/_ts/admin/ec".to_owned(), format!("/_ts/admin/ec/{ec_id}"), "/_ts/admin/eids".to_owned(), - ]; - - for path in valid_paths { - for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - let request = request_builder() - .method(method) - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(edgezero_core::body::Body::from("sensitive-admin-body")) - .expect("should build authenticated admin request"); - let response = route(test_router(), request).await; - - assert_eq!(response.status().as_u16(), 405); - assert_eq!( - response - .headers() - .get("allow") - .and_then(|v| v.to_str().ok()), - Some("GET") - ); - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } - } - - for path in [ "/_ts/admin/ec/".to_owned(), format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), @@ -433,14 +274,13 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), - // Percent-encoded separators match the `^/_ts/admin` basic-auth - // handler but not a literal-slash namespace check, so they must be - // reserved before publisher fallback forwards credentials upstream. + // A check for a literal slash would miss a percent-encoded separator, + // so these are closed before the publisher fallback forwards + // credentials upstream. "/_ts/admin%2Fec".to_owned(), "/_ts/admin%2fec".to_owned(), - // Retired non-`/_ts` alias namespace: only the two exact paths are - // routed to a local deny, so descendants and encoded separators must - // be reserved at the shared fallback boundary. + // The alias outside `/_ts`, with its descendants and encoded + // separators. "/admin/keys".to_owned(), "/admin/keys/rotate/extra".to_owned(), "/admin/keys%2Frotate".to_owned(), @@ -472,186 +312,100 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_without_credentials_returns_401() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin route must return 401 without credentials" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_without_credentials_includes_www_authenticate_header() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "should be 401 before checking header" - ); - assert!( - resp.headers().contains_key("www-authenticate"), - "401 response must include WWW-Authenticate header" - ); - let www_auth = resp - .headers() - .get("www-authenticate") - .expect("should have www-authenticate header") - .to_str() - .expect("should be valid UTF-8"); - assert!( - www_auth.starts_with("Basic realm="), - "WWW-Authenticate must be Basic scheme, got: {www_auth}" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_with_wrong_credentials_returns_401() { - use base64::Engine as _; - let creds = base64::engine::general_purpose::STANDARD.encode("admin:wrong-password"); - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .header("authorization", format!("Basic {creds}")) - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin route must reject wrong credentials with 401" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn discovery_endpoint_does_not_require_auth() { - let router = test_router(); - let req = request_builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = route(router, req).await; - assert_ne!( - resp.status().as_u16(), - 401, - "/.well-known/trusted-server.json must not require auth" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auction_endpoint_does_not_require_auth() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/auction") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(r#"{"adUnits":[]}"#)) - .expect("should build request"); - let resp = route(router, req).await; - assert_ne!( - resp.status().as_u16(), - 401, - "/auction must not apply admin basic-auth gate" - ); +async fn permissions_endpoint_answers_anyone_as_data_and_as_a_page() { + for (path, content_type) in [ + ("/_ts/permissions.json", "application/json"), + ("/_ts/permissions", "text/html; charset=utf-8"), + ] { + let req = request_builder() + .method("GET") + .uri(path) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + assert_eq!( + resp.status().as_u16(), + 200, + "{path} should answer with no credential" + ); + assert_eq!( + resp.headers()["content-type"], + content_type, + "{path} should answer in its own form" + ); + assert_eq!( + resp.headers()["cache-control"], + "no-store", + "{path} is one request's own answer" + ); + } } -// --------------------------------------------------------------------------- -// Admin key route full path coverage -// --------------------------------------------------------------------------- - -// Exercises the auth-fail path with a realistic key body (complements the -// generic `admin_route_without_credentials_returns_401` above). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_rotate_key_auth_fail_returns_401() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin/keys/rotate without credentials must return 401" - ); +async fn config_endpoint_answers_anyone_and_masks_secrets() { + for (path, content_type) in [ + ("/_ts/config.json", "application/json"), + ("/_ts/config", "text/html; charset=utf-8"), + ] { + let req = request_builder() + .method("GET") + .uri(path) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + assert_eq!( + resp.status().as_u16(), + 200, + "{path} should answer with no credential" + ); + assert_eq!( + resp.headers()["content-type"], + content_type, + "{path} should answer in its own form" + ); + let body = resp.into_body().into_bytes().unwrap_or_default(); + let text = String::from_utf8_lossy(&body); + assert!( + text.contains("XXXX"), + "{path} should mask what it does not show" + ); + assert!( + !text.contains("route-test-proxy-secret"), + "{path} should not carry the proxy secret" + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_deactivate_key_auth_fail_returns_401() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/deactivate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin/keys/deactivate without credentials must return 401" - ); -} - -#[tokio::test] -async fn legacy_admin_rotate_alias_returns_404() { - // The legacy non-`/_ts` alias is denied locally rather than routed to the - // admin handler or publisher fallback. - let router = make_router(); - - let req = request_builder() - .method("POST") - .uri("/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - - let resp = route(router, req).await; - - assert_eq!( - resp.status().as_u16(), - 404, - "legacy admin key rotation alias must return local 404" - ); -} - -#[tokio::test] -async fn legacy_admin_deactivate_alias_returns_404() { - let router = make_router(); - - let req = request_builder() - .method("POST") - .uri("/admin/keys/deactivate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - - let resp = route(router, req).await; - - assert_eq!( - resp.status().as_u16(), - 404, - "legacy admin key deactivation alias must return local 404" - ); +async fn data_endpoint_answers_a_browser_opening_the_page_and_nothing_else() { + for (opened_as_a_page, expected) in [(false, 403), (true, 200)] { + let mut builder = request_builder().method("GET").uri("/_ts/data"); + if opened_as_a_page { + builder = builder + .header("sec-fetch-mode", "navigate") + .header("sec-fetch-dest", "document"); + } + let req = builder + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + assert_eq!( + resp.status().as_u16(), + expected, + "opened as a page: {opened_as_a_page}" + ); + assert_eq!( + resp.headers()["cache-control"], + "no-store, private", + "should never be stored" + ); + let body = resp.into_body().into_bytes().unwrap_or_default(); + assert_eq!( + String::from_utf8_lossy(&body).contains("keeps no record"), + opened_as_a_page, + "should say nothing is held to a browser opening the page, and to nothing else" + ); + } } #[tokio::test] @@ -676,6 +430,57 @@ async fn tsjs_route_prefix_is_handled_not_5xx() { ); } +// --------------------------------------------------------------------------- +// Edge Cookie module availability +// --------------------------------------------------------------------------- + +/// Test settings selecting a vendor Edge Cookie module this adapter does not +/// inject, with the `[ec.acme]` block that module's settings live in. +/// `acme` is a fictional vendor key. +const UNINJECTED_MODULE_TOML: &str = r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "route-test-proxy-secret" + + [ec] + module = "acme" + + [ec.acme] + endpoint = "https://ec.acme.example.com" + + # An Edge Cookie module is configured, so single-jurisdiction operation + # is acknowledged because no geo module is selected. + [geo] + assume_single_jurisdiction = true +"#; + +/// A module selection this adapter can never supply must fail while the +/// application state is built, before any request is served. +/// +/// Configuration validation accepts this selection, because only the adapter +/// that injects a module knows what that module needs, and this adapter +/// injects no vendor Edge Cookie module, so only the composition root can +/// catch it. Without the startup check the deployment would come up and answer +/// every request. +#[test] +fn selecting_a_module_this_adapter_cannot_supply_fails_at_startup() { + let settings = Settings::from_toml(UNINJECTED_MODULE_TOML) + .expect("should parse settings selecting an uninjected module"); + + // `RouterService` is not `Debug`, so take the error side directly rather + // than through `expect_err`. + let error = TrustedServerApp::routes_with_settings(settings) + .err() + .expect("building state with an uninjected module should fail"); + + assert!( + error.to_string().contains("acme"), + "the startup error should name the selected module, got: {error}" + ); +} + /// Regression test: a Next.js navigation with a pending auction must buffer to /// the structural body close. The Flight payload carries a literal ``, so /// a parser-blind seam would inject bids early and split the RSC data. @@ -746,3 +551,115 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { "should not leak generated placeholders: {html}" ); } + +/// Every route registered beneath an underscore prefix is one of the +/// addresses an attestation endpoint may not take, so a new fixed route +/// cannot leave an endpoint free to register its address twice. +#[test] +fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() { + use trusted_server_core::attestation::RESERVED_PATHS; + + for route in test_router().routes() { + let path = route.path(); + if path.starts_with("/_") { + assert!( + RESERVED_PATHS.contains(&path), + "{path} should be listed in attestation::RESERVED_PATHS" + ); + } + } +} + +// --------------------------------------------------------------------------- +// robots.txt +// --------------------------------------------------------------------------- + +/// The route test settings with a `[robots-txt]` section that refuses every +/// crawler and keeps one path open. +fn router_refusing_every_crawler() -> RouterService { + let settings = Settings::from_toml( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "route-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ) + .expect("should parse route test settings"); + + TrustedServerApp::routes_with_settings(settings) + .expect("should build router from test settings") +} + +fn robots_request(method: &str, path: &str) -> Request { + request_builder() + .method(method) + .uri(path) + .body(edgezero_core::body::Body::empty()) + .expect("should build request") +} + +fn robots_tag(response: &Response) -> Option { + response + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) +} + +/// The file itself is the publisher path's, made by the `robots_txt` +/// middleware the entry names, so no route of this server names the path. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { + let registered: Vec = router_refusing_every_crawler() + .routes() + .into_iter() + .map(|route| route.path().to_string()) + .collect(); + assert!( + !registered.iter().any(|path| path == "/robots.txt"), + "the file is the publisher path's: {registered:?}" + ); + + let other = route( + router_refusing_every_crawler(), + robots_request("GET", "/.well-known/trusted-server.json"), + ) + .await; + + assert_eq!( + robots_tag(&other).as_deref(), + Some("noindex, nofollow"), + "no route may be the one page a crawler is allowed to index" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_response_carries_no_blanket_rule_without_the_section() { + let response = route( + test_router(), + robots_request("GET", "/.well-known/trusted-server.json"), + ) + .await; + + assert_eq!(robots_tag(&response), None); +} diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml index 9992db7123..c938045b8b 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml @@ -18,7 +18,6 @@ TRUSTED_SERVER_CONFIG = "{}" # Fictitious integration-only secret values. `worker::Env::secret` reads these # string bindings in local Wrangler runs; production values are provisioned with # `wrangler secret put` instead of being committed to a manifest. -integration_admin_password = "integration-admin-password-32-bytes-ok" integration_proxy_secret = "integration-test-proxy-secret-32-bytes-ok" integration_ec_passphrase = "integration-test-ec-secret-padded-32" integration_partner_token_alpha = "integration-test-token-alpha-32-bytes-ok" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 710523766e..d50065ee5d 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -19,7 +19,6 @@ reusable-sandbox = [] [dependencies] async-trait = { workspace = true } -base64 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } derive_more = { workspace = true } @@ -35,7 +34,14 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } -url = { workspace = true } +trusted-server-modules = { workspace = true } +trusted-server-permission-signal-gpc = { workspace = true } +trusted-server-permission-signal-gpp = { workspace = true } +trusted-server-permission-signal-mtm = { workspace = true } +trusted-server-permission-signal-tcf = { workspace = true } +trusted-server-permission-signal-us-privacy = { workspace = true } +trusted-server-device-fastly = { workspace = true } +trusted-server-geo-fastly = { workspace = true } urlencoding = { workspace = true } [dev-dependencies] @@ -43,3 +49,6 @@ bytes = { workspace = true } edgezero-core = { workspace = true, features = ["test-utils"] } toml = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } +# Test fixture only. It proves the integration seam on this adapter from +# outside core and must never be a regular dependency. +trusted-server-integration-seam-probe = { path = "../testing/seam-probe" } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d1da9b61fb..a1cb011dd7 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1,8 +1,8 @@ //! Full `EdgeZero` application wiring for Trusted Server. //! //! Registers all routes for Trusted Server into a -//! [`RouterService`]. On successful startup, attaches [`FinalizeResponseMiddleware`] -//! (outermost) and [`AuthMiddleware`] (inner). When startup fails, +//! [`RouterService`]. On successful startup, attaches +//! [`FinalizeResponseMiddleware`]. When startup fails, //! [`startup_error_router`] returns a bare router without middleware. //! Builds the [`AppState`] once per Wasm instance. //! @@ -20,16 +20,16 @@ //! |--------|-------------|---------| //! | GET | `/.well-known/trusted-server.json` | [`handle_trusted_server_discovery`] | //! | POST | `/verify-signature` | [`handle_verify_signature`] | -//! | POST | `/_ts/admin/keys/rotate` | [`handle_rotate_key`] | -//! | POST | `/_ts/admin/keys/deactivate` | [`handle_deactivate_key`] | -//! | GET | `/_ts/admin/ec` | [`handle_admin_ec_lookup`] | -//! | GET | `/_ts/admin/ec/{id}` | [`handle_admin_ec_lookup`] | -//! | GET | `/_ts/admin/eids` | [`handle_admin_eids_lookup`] | //! | POST | `/_ts/api/v1/batch-sync` | [`handle_batch_sync`] | //! | GET | `/_ts/api/v1/identify` | [`handle_identify`] | //! | GET | `/_ts/set-tester` | [`handle_set_tester`] | //! | GET | `/_ts/clear-tester` | [`handle_clear_tester`] | +//! | GET | `/_ts/permissions` and `/_ts/permissions.json` | [`handle_permissions`] | +//! | GET | `/_ts/config` and `/_ts/config.json` | [`handle_config`] | +//! | GET | `/_ts/data` | [`handle_data`] | +//! | GET, HEAD | The address `[attestation] endpoint` names, and the same with `.json` | [`handle_prefix`] | //! | OPTIONS | `/_ts/api/v1/identify` | [`cors_preflight_identify`] | +//! | POST | `/_ts/api/v1/ec/resolve` | [`handle_ec_resolve`] | //! | POST | `/auction` | [`handle_auction`] | //! | GET | `/first-party/proxy` | [`handle_first_party_proxy`] | //! | GET | `/first-party/click` | [`handle_first_party_click`] | @@ -44,7 +44,7 @@ //! > **Note:** Methods not in the list above (e.g. `TRACE`, `CONNECT`, WebDAV verbs) return a //! > router-level 405. Legacy routing proxied *every* method through to the publisher origin. //! > This is a known intentional restriction of the EdgeZero router; the entry-point -//! > `apply_finalize_headers` call in `main.rs` still adds TS headers to those 405 responses. +//! > `apply_finalize_headers` call in `lib.rs` still adds TS headers to those 405 responses. //! //! # EC identity lifecycle //! @@ -52,7 +52,7 @@ //! `route_request` (tracked in issue #495): //! //! - [`build_ec_request_state`] runs before every dispatched route (except -//! batch-sync, which uses Bearer auth, and the read-only admin diagnostics) +//! batch-sync, which uses Bearer auth) //! and reproduces the legacy //! pre-routing prelude: device signals, bot gate, `ts-eids`/`sharedid` //! cookie capture, geo lookup, [`EcContext`] creation, and KV-graph gating. @@ -65,11 +65,6 @@ //! //! ## Intentional deviations from legacy //! -//! - **401 auth challenges**: [`AuthMiddleware`] short-circuits before the -//! handler runs, so no EC state is built and `ec_finalize_response` does not -//! run on these responses. Legacy ran EC finalization on its own auth -//! challenges. Like the 401 geo-skip, this is privacy-conservative: no EC -//! cookies are issued to unauthenticated callers. //! - **Publisher responses** keep Fastly origin bodies streaming through the //! `EdgeZero` response body when the body is processable or pass-through. //! Adapters without streaming-body support still use the bounded buffered @@ -100,51 +95,62 @@ use edgezero_core::http::{ }; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::attestation::{PlatformIdentity, handle_prefix, prefix_routes}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ - AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::closed_paths::closed_path_response; use trusted_server_core::config_payload::DEFAULT_SECRET_STORE_ID; -use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; -use trusted_server_core::ec::admin::{ - deny_admin_diagnostic_fallback, handle_admin_ec_lookup, handle_admin_eids_lookup, -}; +use trusted_server_core::constants::{COOKIE_EIDS, COOKIE_SHAREDID}; use trusted_server_core::ec::batch_sync::handle_batch_sync; -use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::module::request_module; +use trusted_server_core::ec::module::{EdgeCookieModule, build_reusable_module}; use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::ec::resolve::handle_ec_resolve; use trusted_server_core::ec::{EcContext, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; +use trusted_server_core::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH, handle_config}; +use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; +use trusted_server_core::inspect::permissions::{ + PERMISSIONS_JSON_PATH, PERMISSIONS_PAGE_PATH, handle_permissions, +}; use trusted_server_core::integrations::{ - IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, RequestFilterRegistryInput, - RequestFilterRegistryOutcome, + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, + RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; +use trusted_server_core::module_context::ModuleRequest; +use trusted_server_core::permissions::PermissionState; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, LoadingCaches, PlatformKvStore, RuntimeServices, StoreName, + build_geo_module, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, handle_page_bids, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, publisher_response_into_streaming_response, }; use trusted_server_core::request_signing::{ - handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, - handle_verify_signature, + handle_trusted_server_discovery, handle_verify_signature, }; use trusted_server_core::settings::{ProxyAssetRoute, Settings}; -use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; +use trusted_server_core::settings_data::{ + DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store_with, +}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; +use trusted_server_device_fastly::FastlyHostSignals; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::FinalizeResponseMiddleware; use crate::platform::{ FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, FastlyPlatformHttpClient, FastlyPlatformSecretStore, UnavailableKvStore, @@ -154,6 +160,12 @@ use crate::platform::{ // AppState // --------------------------------------------------------------------------- +/// The stores a service reads and the key of its app-config blob. +/// +/// [`Self::from_env`] takes the key from the service's `__KEY` selector as it +/// is written. A selector that names a blob for each host is resolved for the +/// request before anything is built, by `stores_for_request` in the entry +/// point. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct RuntimeStoreConfig { pub(crate) config_store_name: StoreName, @@ -183,6 +195,25 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, + /// The Edge Cookie module `[ec] module` selects, resolved once here. + /// + /// This adapter runs a fresh instance per request and resolving reads no + /// request data, so the selection is resolved when the state is built + /// and handed to every request through + /// [`RuntimeServices::resolved_ec_module`](trusted_server_core::platform::RuntimeServices::resolved_ec_module), + /// rather than resolved again on the request path. + /// `None` for a deployment that selects no module. + pub(crate) ec_module: Option>, + /// The permission signal modules `[permission-signal] modules` selects + /// from the scheme crates this adapter links, in the order they run. + /// Selected once here so a name no crate answers to fails startup rather + /// than the first request, and handed to every request's services. + pub(crate) permission_signal_modules: + Arc<[Arc]>, + /// The loading caches the `[cache.]` tables select from the linked + /// builders, taken when this state is built and handed to every request's + /// services. + pub(crate) loading_caches: Arc, } /// Build the application state, loading settings and constructing all per-application components. @@ -200,27 +231,121 @@ pub(crate) fn build_state( pub(crate) fn load_settings_from_config_store( stores: &RuntimeStoreConfig, ) -> Result> { - get_settings_from_config_store( + // The settings are validated as they load, so the stock builders and the + // ones a deployment registered are supplied here as well as to the state + // build. Without them a `[demand]` or `[ad-server]` name one of them + // supplies is refused before the state that knows them is built. + get_settings_from_config_store_with( &FastlyPlatformConfigStore, &FastlyPlatformSecretStore, &stores.config_store_name, &stores.config_key, &stores.secret_store_name, + &trusted_server_modules::builders_with(registered_integrations()), ) } +/// Build the application state from explicit settings. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, or when the auction orchestrator or the integration registry +/// fail to initialize. pub(crate) fn build_state_from_settings( settings: Settings, +) -> Result, Report> { + build_state_with_registrations(settings, registered_integrations()) +} + +/// The integration builders a deployment offered through +/// [`crate::run_with`], set once before any request is served. +/// +/// Held here rather than threaded through the build, because the state is +/// built inside the `EdgeZero` application hooks, which take no arguments. +/// [`IntegrationBuilder`] is `Copy` and holds only function pointers and +/// static references, so nothing here can change once it is set. +static REGISTERED_INTEGRATIONS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +/// Records the builders a deployment offers. The first call wins, and +/// [`crate::run_with`] is the only caller, so nothing registers after serving +/// has begun. +pub(crate) fn register_integrations(builders: Vec) { + let _ = REGISTERED_INTEGRATIONS.set(builders); +} + +/// The builders a deployment registered, or none. +fn registered_integrations() -> &'static [IntegrationBuilder] { + REGISTERED_INTEGRATIONS.get().map_or(&[], Vec::as_slice) +} + +/// Build the application state from explicit settings, composing the modules +/// a stock build ships with the externally supplied builders in +/// `integrations`. +/// +/// A deployment that ships a vendor crate calls this to add that crate's +/// integration builder without the adapter naming the vendor. Auction +/// providers come from the compiled auction plan, as they do without any +/// external builders. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, when the auction plan does not compile or cannot run on this +/// adapter, or when the auction orchestrator or the integration registry fail +/// to initialize, which includes two builders claiming the same integration +/// id. +pub(crate) fn build_state_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], ) -> Result, Report> { warn_if_certificate_check_disabled(&settings); - let plan = Arc::new(compile_auction_plan(&settings)?); + // The modules a stock build ships come first and the deployment's own + // follow, which is the order their hooks run in. + let integrations = trusted_server_modules::builders_with(integrations); + let integrations = integrations.as_slice(); + + // The plan is compiled with the integrations this adapter was given, so an + // `[ad-server]` or `[demand]` name one of their builders supplies resolves + // here. Compiling without them would drop the implementation and report the + // name as one no builder registers. + let plan = Arc::new(compile_auction_plan_with(&settings, integrations)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; - let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; - let registry = IntegrationRegistry::with_plan(&settings, plan)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan))?; + let registry = IntegrationRegistry::with_plan_and_registrations(&settings, plan, integrations)? + .with_platform(fastly_platform_identity()); + + // Composition root: resolve the module selection once, before any request + // is served, so a selection this adapter can never supply fails here rather + // than on the first request, and keep what the resolution produced so the + // request path does not resolve the same settings again. The registry is + // built first because a module can supply the vendor Edge Cookie module + // the selector names, and resolving without it would reject a selection + // this deployment can in fact satisfy. + // + // This adapter injects host signals on every request, so a startup instance + // with no captured signals answers the only question the check asks, + // which is whether the service exists at all. That same emptiness is why + // `build_reusable_module` hands back nothing for a module built from + // those signals, leaving it to be resolved per request against the + // signals that request actually carried. + let ec_module = build_reusable_module( + &settings.ec, + Some(Arc::new(FastlyHostSignals::default())), + registry.ec_module(), + )?; + let permission_signal_modules = + trusted_server_core::permission_signal::build_permission_signal_modules( + &settings, + &shipped_signal_modules(), + )?; let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + let loading_caches = + trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; Ok(Arc::new(AppState { settings: Arc::new(settings), @@ -228,9 +353,32 @@ pub(crate) fn build_state_from_settings( registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, + ec_module, + permission_signal_modules, + loading_caches, })) } +/// The permission signal modules this adapter links, in the order they run +/// when configuration names none. Global Privacy Control is first because it +/// is a browser setting with no interface of its own, and the three that +/// carry a choice someone made through an interface follow, so an answer +/// given at a prompt amends the header the visitor arrived with. +/// +/// Core supplies no module of its own, so this is where a deployment's +/// schemes are decided. A scheme is added by linking its crate here, and a +/// scheme core has never heard of plugs in the same way. +fn shipped_signal_modules() +-> Vec> { + vec![ + Arc::new(trusted_server_permission_signal_gpc::GpcModule::new()), + Arc::new(trusted_server_permission_signal_gpp::GppSaleOptOutModule::new()), + Arc::new(trusted_server_permission_signal_us_privacy::UsPrivacyModule::new()), + Arc::new(trusted_server_permission_signal_tcf::TcfModule::new()), + Arc::new(trusted_server_permission_signal_mtm::MtmModule::new()), + ] +} + fn warn_if_certificate_check_disabled(settings: &Settings) { if !settings.proxy.certificate_check { log::warn!( @@ -254,6 +402,12 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { /// absent (e.g. tests that dispatch without the entry point). Scheme detection /// continues to rely on the trusted `fastly-ssl` header injected by /// `edgezero_main` after sanitization. +/// +/// Applies the module-supplied modules selected by `[geo]`, `[ec]` and +/// `[device] module` on top of the base services. Unset and `none` resolve +/// the disabled geo module, `platform` leaves the Fastly lookup standing, +/// and any other key names a module's module. Identity and device are +/// applied the same way when a module supplies them. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { let client_info = ctx .request() @@ -265,10 +419,29 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime ..ClientInfo::default() }); - RuntimeServices::builder() + // The TLS JA4 and HTTP/2 signals arrive as trusted internal headers + // injected by the entry point. They build the host-signal service a + // host-signal module reads. Fastly always supplies the capability, so the + // service is always set even when a request carried no signal. + let tls_ja4 = ctx + .request() + .headers() + .get("x-ts-tls-ja4") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let h2_fingerprint = ctx + .request() + .headers() + .get("x-ts-h2-fingerprint") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + + let builder = RuntimeServices::builder() .config_store(Arc::new(FastlyPlatformConfigStore)) .secret_store(Arc::new(FastlyPlatformSecretStore)) .kv_store(Arc::clone(&state.default_kv_store)) + .value_cache(Arc::new(crate::value_cache::FastlyValueCache)) + .loading_caches(Arc::clone(&state.loading_caches)) // Constructed unconditionally, but only read when the assembly mode is a // shared-template one — which defaults to Inline, so this is inert until an // operator opts in. @@ -276,9 +449,74 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) .http_client(Arc::new(FastlyPlatformHttpClient)) - .geo(Arc::new(FastlyPlatformGeo)) + .geo(build_geo_module( + &state.settings, + Arc::new(FastlyPlatformGeo), + )) .auction_telemetry_sink(Arc::clone(&state.auction_telemetry_sink)) .client_info(client_info) + // The signal modules were selected once at startup from the scheme + // crates this adapter links, so every request asks exactly the ones + // configuration named, in that order. + .permission_signal_modules(Arc::clone(&state.permission_signal_modules)) + .host_signals(Arc::new(FastlyHostSignals::new(tls_ja4, h2_fingerprint))); + + // Hand every request the module resolved at the composition root, so the + // request path reuses that instance instead of resolving `[ec] module` + // again. Nothing is set for a deployment that selects no module, or one + // whose module is built from this request's own host signals, and both + // are resolved on the request path instead. + let services = match state.ec_module.clone() { + Some(module) => builder.resolved_ec_module(module).build(), + None => builder.build(), + }; + + // Unset and `"none"` both resolve nothing, so no client IP reaches a host + // geo service. `"platform"` opts in to the Fastly lookup below, and any + // other key names an integration module that declares a geo module. + let mut services = services; + if let Some(module) = state.registry.geo_module() { + services = services.with_geo(module); + } + if let Some(module) = state.registry.ec_module() { + services = services.with_ec_module(module); + } + if let Some(module) = state.registry.device_module() { + services = services.with_device_module(module); + } + // The publisher this request resolved to, its tenant included, names + // every key in a shared store and every name the browser holds. + let prefixes = state.registry.prefixes( + &state.settings, + ModuleRequest::of(ctx.request(), services.client_info()), + ); + services.with_prefixes(prefixes) +} + +/// Builds the services graph handed to a module on a response-side finalize +/// path. +/// +/// The finalize paths run after the handler, where the per-request services +/// built by [`build_per_request_services`] are already out of scope, but a geo +/// module still needs real platform services to resolve a location. The +/// middleware builds this once and clones it per request with +/// [`RuntimeServices::with_client_info`], and the entry point rebuilds it per +/// call. +pub(crate) fn build_finalize_services( + settings: &Settings, + kv_store: Arc, + loading_caches: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(FastlyPlatformConfigStore)) + .secret_store(Arc::new(FastlyPlatformSecretStore)) + .kv_store(kv_store) + .value_cache(Arc::new(crate::value_cache::FastlyValueCache)) + .loading_caches(loading_caches) + .backend(Arc::new(FastlyPlatformBackend)) + .http_client(Arc::new(FastlyPlatformHttpClient)) + .geo(build_geo_module(settings, Arc::new(FastlyPlatformGeo))) + .client_info(ClientInfo::default()) .build() } @@ -302,7 +540,7 @@ fn uses_dynamic_tsjs_fallback(method: &Method, path: &str) -> bool { // EC request state // --------------------------------------------------------------------------- -/// EC state threaded from route handlers to the `main.rs` entry point via +/// EC state threaded from route handlers to the `lib.rs` entry point via /// response extensions. /// /// `edgezero_main` pops this from the response after dispatch and runs @@ -321,7 +559,9 @@ pub(crate) struct EcFinalizeState { pub(crate) sharedid_cookie: Option, pub(crate) is_real_browser: bool, /// Per-request services carried to the entry point so the pull-sync - /// dispatcher can reuse the same platform HTTP client. + /// dispatcher can reuse the same platform HTTP client, and so finalization + /// can hand them to the selected Edge Cookie module when it replaces an + /// orphaned identifier. pub(crate) services: RuntimeServices, } @@ -395,13 +635,13 @@ fn device_signals_for(req: &Request) -> DeviceSignals { /// Builds the per-request EC state, mirroring the pre-routing prelude of the /// legacy `route_request` step by step. -fn build_ec_request_state( +async fn build_ec_request_state( settings: &Settings, services: &RuntimeServices, req: &Request, ) -> EcRequestState { let device_signals = device_signals_for(req); - let is_real_browser = device_signals.looks_like_browser(); + let is_real_browser = device_signals.looks_like_browser; if !is_real_browser { log::info!( "Bot gate: blocking EC operations (ja4={:?}, platform={:?}, is_mobile={})", @@ -411,19 +651,12 @@ fn build_ec_request_state( ); } - let eids_cookie = crate::extract_cookie_value(req, COOKIE_TS_EIDS); + let eids_cookie = + crate::extract_cookie_value(req, &services.public_prefix().cookie_name(COOKIE_EIDS)); let sharedid_cookie = crate::extract_cookie_value(req, COOKIE_SHAREDID); - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed during EC setup: {e}"); - None - }); - let (ec_context, setup_error) = - match EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) { + match EcContext::read_from_request_resolving_geo(settings, req, services).await { Ok(mut context) => { context.set_device_signals(device_signals); // Orphan-recovery eligibility is intentionally left false here. @@ -437,18 +670,21 @@ fn build_ec_request_state( } Err(report) => (EcContext::default(), Some(report)), }; + let geo_info = ec_context.geo_info().cloned(); // Bot gate: suppress KV-backed EC writes for unrecognized clients, except - // consent withdrawals. Revocations keep the write path so tombstones stay - // authoritative even for privacy-extension-heavy clients. + // when the request carries an explicit withdrawal signal. The write path + // stays open for withdrawal so tombstones remain authoritative even for + // privacy-extension-heavy clients that do not look like known browsers. A + // merely not-permitted (pre-consent or fail-closed) request writes nothing, + // so it does not need the graph. let kv_graph = crate::maybe_identity_graph(settings); - let finalize_kv_graph = if setup_error.is_none() - && (is_real_browser || ec_consent_withdrawn(ec_context.consent())) - { - kv_graph.clone() - } else { - None - }; + let finalize_kv_graph = + if setup_error.is_none() && (is_real_browser || ec_context.storage_withdrawn()) { + kv_graph.clone() + } else { + None + }; let kv_graph = if is_real_browser { kv_graph } else { None }; EcRequestState { @@ -482,16 +718,20 @@ enum PreRoute { /// Runs the integration request-filter pipeline before route dispatch. /// -/// Mirrors the legacy `route_request` ordering: filters run after auth -/// (`AuthMiddleware` on this path) and before route matching. Request header +/// Mirrors the legacy `route_request` ordering: filters run before route +/// matching. Request header /// mutations are applied to `req` so the routed handler observes them; response /// effects are returned for the entry point to apply after EC finalization. A /// filter that responds (e.g. a `DataDome` challenge) short-circuits routing. +/// +/// `permissions` carries the state resolved when the EC context was built, so +/// every filter reads the same permissions as the rest of the request. async fn run_pre_route_filters( state: &AppState, services: &RuntimeServices, req: &mut Request, geo_info: Option<&GeoInfo>, + permissions: Option<&PermissionState>, ) -> PreRoute { match state .registry @@ -500,6 +740,7 @@ async fn run_pre_route_filters( services, req, geo_info, + permissions, }) .await { @@ -538,6 +779,11 @@ async fn execute_named( ctx: RequestContext, handler: NamedRouteHandler, ) -> Result { + // Read only, answered before the Edge Cookie lifecycle so nothing is written. + if matches!(handler, NamedRouteHandler::Config) { + return Ok(handle_config(&state.settings, &ctx.into_request())); + } + let services = build_per_request_services(&state, &ctx); let mut req = ctx.into_request(); @@ -549,51 +795,35 @@ async fn execute_named( return Ok(run_batch_sync(&state, &services, req)); } - // An operator cache purge is not a reader request: running the EC lifecycle would - // attach finalization state and could ingest the operator's cookies into KV. - if matches!(handler, NamedRouteHandler::AdminCachePurge) { - let principal = trusted_server_core::auth::authenticated_username(&req); - let response = trusted_server_core::cache_purge::handle_cache_purge( - &services, - req, - principal.as_deref(), - ) - .await - .unwrap_or_else(|error| http_error(&error)); + // Read only, answered before the Edge Cookie lifecycle so nothing is written. + if matches!(handler, NamedRouteHandler::Permissions) { + let response = handle_permissions(&state.settings, &services, &req) + .await + .unwrap_or_else(|error| http_error(&error)); return Ok(response); } - // These diagnostics are read-only. Running the normal EC lifecycle would - // attach finalization state and could ingest request cookies into KV after - // the handler returns, violating that contract. - if matches!( - handler, - NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup - ) { - let response = PartnerRegistry::from_config(&state.settings.ec.partners) - .and_then(|registry| match handler { - NamedRouteHandler::AdminEcLookup => { - // Deliberately do not use an EC request-state graph: that - // copy is bot-gated, while operators use curl for this - // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); - handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) - } - NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), - _ => unreachable!("admin diagnostics should use early dispatch"), + // Read only, answered before the Edge Cookie lifecycle so nothing is written. + if matches!(handler, NamedRouteHandler::Data) { + let kv = crate::maybe_identity_graph(&state.settings); + let response = request_module(&state.settings.ec, &services) + .and_then(|module| { + handle_data( + kv.as_ref(), + module.as_deref(), + services.public_prefix(), + &req, + ) }) .unwrap_or_else(|error| http_error(&error)); return Ok(response); } - if let Err(report) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(report) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&report)); } - let mut ec = build_ec_request_state(&state.settings, &services, &req); + let mut ec = build_ec_request_state(&state.settings, &services, &req).await; // EcContext creation errors short-circuit before filters, mirroring legacy: // the legacy path returns its error response before running filter_request. if let Some(report) = ec.setup_error.take() { @@ -605,13 +835,20 @@ async fn execute_named( )); } - let effects = - match run_pre_route_filters(&state, &services, &mut req, ec.geo_info.as_ref()).await { - PreRoute::ShortCircuit { response, effects } => { - return Ok(attach_dispatch_extensions(response, ec, effects)); - } - PreRoute::Continue { effects } => effects, - }; + let effects = match run_pre_route_filters( + &state, + &services, + &mut req, + ec.geo_info.as_ref(), + Some(ec.ec_context.permissions()), + ) + .await + { + PreRoute::ShortCircuit { response, effects } => { + return Ok(attach_dispatch_extensions(response, ec, effects)); + } + PreRoute::Continue { effects } => effects, + }; let response = run_named_route(&state, &services, req, handler, &mut ec) .await @@ -633,15 +870,6 @@ async fn run_named_route( NamedRouteHandler::VerifySignature => { handle_verify_signature(&state.settings, services, req) } - NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), - NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), - NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup => { - unreachable!("admin diagnostics should be handled before EC setup") - } - NamedRouteHandler::AdminCachePurge => { - unreachable!("cache purge should be handled before EC setup") - } - NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::BatchSync => { // Dispatched by execute_named before EC state is built. unreachable!("batch-sync should be handled by run_batch_sync") @@ -661,8 +889,28 @@ async fn run_named_route( ) } } - NamedRouteHandler::SetTester => handle_set_tester(&state.settings), - NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), + NamedRouteHandler::SetTester => { + handle_set_tester(&state.settings, services.public_prefix()) + } + NamedRouteHandler::ClearTester => { + handle_clear_tester(&state.settings, services.public_prefix()) + } + NamedRouteHandler::Permissions | NamedRouteHandler::Config | NamedRouteHandler::Data => { + unreachable!("the inspection endpoints are answered before EC setup") + } + NamedRouteHandler::EcResolve => { + // The resolve endpoint persists the identity-graph row before + // creating, so it takes the same bot-gated graph as generation: an + // unrecognized client gets no graph, and therefore no new Edge Cookie. + handle_ec_resolve( + &state.settings, + req, + &ec.ec_context, + ec.kv_graph.as_ref(), + services, + ) + .await + } NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; @@ -730,14 +978,19 @@ async fn run_named_route( /// response finalization. fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> Response { let device_signals = device_signals_for(&req); - let is_real_browser = device_signals.looks_like_browser(); - let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); + let is_real_browser = device_signals.looks_like_browser; + let eids_cookie = + crate::extract_cookie_value(&req, &services.public_prefix().cookie_name(COOKIE_EIDS)); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - handle_batch_sync(&kv, &partner_registry, &limiter, req) + // A partner echoes back an identifier the deployment's own module + // created, so validation and KV normalization are dispatched through + // that module rather than the built-in HMAC grammar. + let module = request_module(&state.settings.ec, services)?; + handle_batch_sync(&kv, &partner_registry, &limiter, module.as_deref(), req) }); let mut response = result.unwrap_or_else(|e| http_error(&e)); @@ -768,21 +1021,18 @@ async fn dispatch_fallback( services: &RuntimeServices, mut req: Request, ) -> Response { - if let Some(response) = deny_admin_diagnostic_fallback(&req) { + if let Some(response) = closed_path_response(&req) { return response; } let path = req.uri().path().to_string(); let method = req.method().clone(); - if let Err(report) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(report) = state.registry.prepare_request(&state.settings, &mut req) { return http_error(&report); } - let mut ec = build_ec_request_state(&state.settings, services, &req); + let mut ec = build_ec_request_state(&state.settings, services, &req).await; if let Some(report) = ec.setup_error.take() { let response = http_error(&report); return attach_dispatch_extensions(response, ec, RequestFilterEffects::default()); @@ -790,7 +1040,14 @@ async fn dispatch_fallback( // Pre-route integration request filters (DataDome protection, etc.) run // before the route-type decision, matching legacy `route_request` ordering. - let effects = match run_pre_route_filters(state, services, &mut req, ec.geo_info.as_ref()).await + let effects = match run_pre_route_filters( + state, + services, + &mut req, + ec.geo_info.as_ref(), + Some(ec.ec_context.permissions()), + ) + .await { PreRoute::ShortCircuit { response, effects } => { return attach_dispatch_extensions(response, ec, effects); @@ -845,9 +1102,10 @@ async fn dispatch_fallback( if is_publisher_navigation && let Err(err) = ec .ec_context - .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) + .generate_if_needed(&state.settings, ec.kv_graph.as_ref(), services) + .await { - log::warn!("EC generation failed for publisher proxy: {err:?}"); + log::error!("EC generation failed for publisher proxy: {err:?}"); } // Run the server-side auction with the configured creative- @@ -865,7 +1123,10 @@ async fn dispatch_fallback( registry: Some(&partner_registry), }; match handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integration_registry: state.registry.as_ref(), + }, services, ec.kv_graph.as_ref(), &mut ec.ec_context, @@ -992,7 +1253,7 @@ fn attach_request_filter_effects(response: &mut Response, effects: &RequestFilte /// Convert a [`Report`] into an HTTP [`Response`], /// mirroring [`crate::http_error_response`] exactly. /// -/// The near-identical function in `main.rs` is intentional: the legacy path +/// The near-identical function in `lib.rs` is intentional: the legacy path /// uses fastly HTTP types while this path uses `edgezero_core` types. pub(crate) fn http_error(report: &Report) -> Response { let root_error = report.current_context(); @@ -1008,23 +1269,6 @@ pub(crate) fn http_error(report: &Report) -> Response { response } -/// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` -/// aliases on the `EdgeZero` path. -/// -/// These non-`/_ts` aliases are not matched by the `^/_ts/admin` basic-auth -/// handler, so they fail closed locally rather than fall through to the -/// publisher fallback — which would forward the caller's `Authorization` header -/// and key-management payload to the origin, leaking admin credentials. -fn legacy_admin_alias_denied() -> Response { - let mut response = Response::new(edgezero_core::body::Body::from("Not found\n")); - *response.status_mut() = StatusCode::NOT_FOUND; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - // --------------------------------------------------------------------------- // Startup error fallback // --------------------------------------------------------------------------- @@ -1066,18 +1310,14 @@ fn startup_error_router(e: &Report) -> RouterService { enum NamedRouteHandler { TrustedServerDiscovery, VerifySignature, - RotateKey, - DeactivateKey, - AdminEcLookup, - AdminEidsLookup, - AdminCachePurge, - /// Legacy `/admin/keys/*` aliases — denied locally with 404 so they never - /// reach the publisher fallback (which would leak admin credentials). - LegacyAdminDenied, BatchSync, Identify, SetTester, ClearTester, + Permissions, + Config, + Data, + EcResolve, Auction, PageBids, FirstPartyProxy, @@ -1092,21 +1332,6 @@ struct NamedRoute { handler: NamedRouteHandler, } -/// Every method an admin route must claim to keep non-primary methods from falling -/// through to the publisher with the `Authorization` header still attached. -/// -/// Named for the legacy `/admin/*` aliases it was introduced for, and reused by every -/// route with the same requirement here and in the Axum and Spin adapters. -const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ - Method::GET, - Method::POST, - Method::HEAD, - Method::OPTIONS, - Method::PUT, - Method::PATCH, - Method::DELETE, -]; - const NAMED_ROUTES: &[NamedRoute] = &[ NamedRoute { path: "/.well-known/trusted-server.json", @@ -1119,78 +1344,58 @@ const NAMED_ROUTES: &[NamedRoute] = &[ handler: NamedRouteHandler::VerifySignature, }, NamedRoute { - path: "/_ts/admin/keys/rotate", - primary_methods: &[Method::POST], - handler: NamedRouteHandler::RotateKey, - }, - NamedRoute { - path: "/_ts/admin/keys/deactivate", + path: "/_ts/api/v1/batch-sync", primary_methods: &[Method::POST], - handler: NamedRouteHandler::DeactivateKey, + handler: NamedRouteHandler::BatchSync, }, - // Every method is claimed, not just POST. A method this route did not claim would - // fall through to the publisher, and `enforce_basic_auth` leaves the `Authorization` - // header in place, so a GET would ship the shared admin credential to the origin. - // The handler answers the non-POST methods with 405 itself. NamedRoute { - path: "/_ts/admin/cache/purge", - primary_methods: LEGACY_ADMIN_DENY_METHODS, - handler: NamedRouteHandler::AdminCachePurge, + path: "/_ts/api/v1/identify", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::Identify, }, - // Admin EC lookup: the bare route reads the EC ID from the caller's - // `ts-ec` cookie; the parameterized route takes an explicit EC ID. NamedRoute { - path: "/_ts/admin/ec", + path: "/_ts/set-tester", primary_methods: &[Method::GET], - handler: NamedRouteHandler::AdminEcLookup, + handler: NamedRouteHandler::SetTester, }, NamedRoute { - path: "/_ts/admin/ec/{id}", + path: "/_ts/clear-tester", primary_methods: &[Method::GET], - handler: NamedRouteHandler::AdminEcLookup, + handler: NamedRouteHandler::ClearTester, }, - // Admin EIDs echo: decodes the request's ts-eids/sharedId cookies with - // an ingestion preview. Pure request inspection — no KV access. + // What the deployment decided for the asking request, shown to anyone. NamedRoute { - path: "/_ts/admin/eids", + path: PERMISSIONS_PAGE_PATH, primary_methods: &[Method::GET], - handler: NamedRouteHandler::AdminEidsLookup, - }, - // The legacy non-`/_ts` aliases (`/admin/keys/*`) are denied locally with a - // 404 instead of executing key operations: the production basic-auth handler - // regex `^/_ts/admin` does not match them, and letting them fall through to - // publisher fallback for any fallback method would forward the caller's - // `Authorization` header and key-management payload to the origin, leaking - // admin credentials. - NamedRoute { - path: "/admin/keys/rotate", - primary_methods: LEGACY_ADMIN_DENY_METHODS, - handler: NamedRouteHandler::LegacyAdminDenied, + handler: NamedRouteHandler::Permissions, }, NamedRoute { - path: "/admin/keys/deactivate", - primary_methods: LEGACY_ADMIN_DENY_METHODS, - handler: NamedRouteHandler::LegacyAdminDenied, + path: PERMISSIONS_JSON_PATH, + primary_methods: &[Method::GET], + handler: NamedRouteHandler::Permissions, }, + // The settings the deployment is running, masked, shown to anyone. NamedRoute { - path: "/_ts/api/v1/batch-sync", - primary_methods: &[Method::POST], - handler: NamedRouteHandler::BatchSync, + path: CONFIG_PAGE_PATH, + primary_methods: &[Method::GET], + handler: NamedRouteHandler::Config, }, NamedRoute { - path: "/_ts/api/v1/identify", - primary_methods: &[Method::GET, Method::OPTIONS], - handler: NamedRouteHandler::Identify, + path: CONFIG_JSON_PATH, + primary_methods: &[Method::GET], + handler: NamedRouteHandler::Config, }, + // What is held against the request's own Edge Cookie, shown to a browser + // opening the address as a page. NamedRoute { - path: "/_ts/set-tester", + path: DATA_PAGE_PATH, primary_methods: &[Method::GET], - handler: NamedRouteHandler::SetTester, + handler: NamedRouteHandler::Data, }, NamedRoute { - path: "/_ts/clear-tester", - primary_methods: &[Method::GET], - handler: NamedRouteHandler::ClearTester, + path: "/_ts/api/v1/ec/resolve", + primary_methods: &[Method::POST], + handler: NamedRouteHandler::EcResolve, }, NamedRoute { path: "/auction", @@ -1249,6 +1454,42 @@ fn named_route_handler( } } +/// Answers the attestation pages ahead of the Edge Cookie lifecycle and the +/// request filters, because the pages are read only and a verifier fetching +/// one from a server must not be turned away as a bot. +fn attestation_route_handler( + state: Arc, +) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { + move |ctx: RequestContext| { + let state = Arc::clone(&state); + Box::pin(async move { + Ok(handle_prefix( + &state.settings, + state.registry.platform(), + &ctx.into_request(), + )) + }) + } +} + +/// What the evidence says about the Fastly service answering, from the +/// environment variables Fastly gives every Compute service. +fn fastly_platform_identity() -> PlatformIdentity { + let read = |name: &str| { + std::env::var(name) + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) + }; + PlatformIdentity { + name: "fastly".to_owned(), + service_id: read("FASTLY_SERVICE_ID"), + service_version: read("FASTLY_SERVICE_VERSION"), + staging: std::env::var("FASTLY_IS_STAGING").is_ok_and(|value| value == "1"), + pop: read("FASTLY_POP"), + } +} + fn fallback_route_handler( state: Arc, ) -> impl Fn(RequestContext) -> HandlerFuture + Clone + Send + Sync + 'static { @@ -1288,12 +1529,18 @@ impl TrustedServerApp { } fn routes_for_state(state: &Arc) -> RouterService { - let mut router = RouterService::builder() - .middleware(FinalizeResponseMiddleware::new( + let mut router = RouterService::builder().middleware( + FinalizeResponseMiddleware::new( Arc::clone(&state.settings), - Arc::new(FastlyPlatformGeo), - )) - .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); + build_geo_module(&state.settings, Arc::new(FastlyPlatformGeo)), + build_finalize_services( + &state.settings, + Arc::clone(&state.default_kv_store), + Arc::clone(&state.loading_caches), + ), + ) + .with_registry(Arc::clone(&state.registry)), + ); let fallback_handler = fallback_route_handler(Arc::clone(state)); @@ -1317,6 +1564,21 @@ impl TrustedServerApp { } } + // Routes chosen by the settings, so not in the fixed table. + let routes = prefix_routes(&state.settings); + if !routes.is_empty() { + let attestation = attestation_route_handler(Arc::clone(state)); + for route in &routes { + for method in publisher_fallback_methods() { + if matches!(method, Method::GET | Method::HEAD) || route.every_method { + router = router.route(&route.path, method, attestation.clone()); + } else { + router = router.route(&route.path, method, fallback_handler.clone()); + } + } + } + } + // matchit's `/{*rest}` does not match the bare root `/` — register // explicit root routes so `/` reaches the publisher fallback too. for method in publisher_fallback_methods() { @@ -1357,21 +1619,18 @@ impl Hooks for TrustedServerApp { } } +#[cfg(test)] +mod seam_probe_tests; + #[cfg(test)] mod tests { - use std::collections::HashMap; - use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; - use std::time::Duration; use super::{ - AppState, AuctionDispatch, EcContext, EdgeCacheHeader, EidSyncSource, HandlerFuture, - NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, - TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, - build_state_from_settings, compile_auction_plan, handle_publisher_request, - publisher_response_into_streaming_response, startup_error_router, + AppState, EidSyncSource, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, + PAGE_BIDS_PATH, RuntimeStoreConfig, TrustedServerApp, build_per_request_services, + build_state_from_settings, startup_error_router, }; - use base64::Engine as _; use bytes::Bytes; use edgezero_core::app::Hooks as _; use edgezero_core::body::Body; @@ -1384,6 +1643,7 @@ mod tests { use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; + use trusted_server_core::auction::compile_auction_plan_with; use error_stack::Report; use futures::executor::block_on; @@ -1397,9 +1657,7 @@ mod tests { use trusted_server_core::platform::{ ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, PlatformTemplateCache, PlatformTemplateCacheReservation, - RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, - TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, + PlatformSelectResult, RuntimeServices, }; use trusted_server_core::settings::Settings; @@ -1501,16 +1759,6 @@ mod tests { fn test_settings() -> Settings { Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [[handlers]] - path = "^/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" @@ -1521,22 +1769,29 @@ mod tests { allowed_domains = ["*.example", "*.example.com"] [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + [geo] + assume_single_jurisdiction = true + [request_signing] enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - [integrations.prebid] + [auction] enabled = true + modules = ["prebid"] + + [auction.prebid] external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" - [auction] - enabled = true - [auction.providers.prebid] - protocol = "openrtb-2.6" - profile = "prebid-server" + [demand] + modules = ["prebid"] + + [demand.prebid] + implementation = "auction.prebid-server" endpoint = "https://test-prebid.com/openrtb2/auction" timeout_ms = 2000 "#, @@ -1549,6 +1804,26 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + /// Every request's services carry the prefixes of the publisher it + /// resolved to, which start every shared store key and every name the + /// browser holds. + #[test] + fn per_request_services_carry_the_publishers_prefixes() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = RequestContext::new( + empty_request(Method::GET, "/article"), + PathParams::default(), + ); + + let services = build_per_request_services(&state, &context); + + assert_eq!( + services.server_key_prefix().as_str(), + "ts.test-publisher.com" + ); + assert_eq!(services.public_prefix().as_str(), "ts"); + } + #[test] fn per_request_services_register_the_fastly_template_assembler() { let state = build_state_from_settings(test_settings()).expect("should build test state"); @@ -1587,15 +1862,22 @@ mod tests { ) -> RouterService { let settings = test_settings(); let plan = Arc::new( - trusted_server_core::auction::compile_auction_plan(&settings) + compile_auction_plan_with(&settings, &trusted_server_modules::builders()) .expect("should compile auction plan"), ); - let orchestrator = - trusted_server_core::auction::build_orchestrator_with_plan(plan, &settings) - .expect("should build orchestrator"); + let orchestrator = trusted_server_core::auction::build_orchestrator_with_plan(plan) + .expect("should build orchestrator"); let registry = IntegrationRegistry::from_request_filters(filters); let default_kv_store = Arc::new(crate::platform::UnavailableKvStore) as Arc; + // Resolved the same way the composition root resolves it, so this + // router behaves like a served one. + let ec_module = trusted_server_core::ec::module::build_reusable_module( + &settings.ec, + None, + registry.ec_module(), + ) + .expect("should resolve the Edge Cookie module selection"); let state = Arc::new(super::AppState { auction_telemetry_sink: Arc::new( trusted_server_core::auction::NoopAuctionTelemetrySink, @@ -1604,6 +1886,9 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + ec_module, + permission_signal_modules: Arc::default(), + loading_caches: Arc::default(), }); TrustedServerApp::routes_for_state(&state) } @@ -1681,18 +1966,26 @@ mod tests { #[test] fn startup_registers_aps_renderer_route() { let mut settings = test_settings(); - settings.auction.providers.clear(); - settings.auction.providers.insert( - "aps-main".parse().expect("should parse APS provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: trusted_server_core::auction::NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, + settings.demand = trusted_server_core::provider_table::ProviderList::new( + vec!["aps_main".to_string()], + std::collections::BTreeMap::from([( + "aps_main".to_string(), + serde_json::Map::from_iter([ + ( + "implementation".to_string(), + serde_json::json!("auction.aps"), + ), + ( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/pb/bid"), + ), + ("routing".to_string(), serde_json::json!("all_eligible")), + ( + "account_id".to_string(), + serde_json::json!("example-account"), + ), + ]), + )]), ); let state = build_state_from_settings(settings) @@ -1789,232 +2082,57 @@ mod tests { // --------------------------------------------------------------------------- #[test] - fn dispatch_auth_rejected_401_carries_finalize_headers() { - // Verifies FinalizeResponseMiddleware is outermost: an auth-rejected 401 - // must still carry standard TS headers before reaching the client. - // - // Test settings protect `^/(_ts/)?admin` with basic-auth. Sending the - // request without an Authorization header causes AuthMiddleware to - // short-circuit with a 401, which then bubbles through - // FinalizeResponseMiddleware for header injection. + fn page_bids_serves_canonical_path_and_deprecated_alias() { + // The SPA re-auction endpoint lives at the canonical single-underscore + // `/_ts/page-bids`, matching every other internal route. The deprecated + // `/__ts/page-bids` alias must stay registered to the same handler with + // the same methods until pre-rename tsjs bundles age out of browser + // caches — dropping it would leave those clients without ads on SPA + // navigations. // - // This is safe to run without Viceroy: enforce_basic_auth is pure Rust - // (reads settings + request headers only) and FastlyPlatformGeo.lookup(None) - // short-circuits without calling any Fastly ABI. - let router = test_router(); - let req = empty_request(Method::POST, "/_ts/admin/keys/rotate"); - - let response = route(&router, req); - + // The paths are literals, not `PAGE_BIDS_PATH` / `PAGE_BIDS_LEGACY_PATH`. + // Looking a route up by the same const it was registered with is + // tautological: it keeps passing if the const's value changes, which is + // exactly the break that would silently desync the server from the tsjs + // client's hardcoded fetch path. Pin the consts to their literals too so + // a rename has to be deliberate. assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "request without credentials should be rejected" + PAGE_BIDS_PATH, "/_ts/page-bids", + "canonical page-bids path must match the path tsjs fetches" ); assert_eq!( - response - .headers() - .get(HEADER_X_GEO_INFO_AVAILABLE) - .and_then(|v| v.to_str().ok()), - Some("false"), - "FinalizeResponseMiddleware must run even for auth-rejected responses" + PAGE_BIDS_LEGACY_PATH, "/__ts/page-bids", + "legacy alias must match the path pre-rename tsjs bundles fetch" ); - } - #[test] - fn legacy_admin_aliases_route_to_local_deny_not_key_handlers() { - // Security guard for the legacy non-`/_ts` admin aliases. They must be - // registered to the local `LegacyAdminDenied` 404 handler — not the - // rotate/deactivate key handlers, and not left unrouted. Leaving them - // unrouted would fall through to the publisher fallback, which forwards - // the request (including the `Authorization` header and key-management - // payload) to the origin, leaking admin credentials. Mapping them to the - // key handlers would expose key operations, since the production - // basic-auth regex `^/_ts/admin` does not match `/admin/keys/*`. - let handler_for = |path: &str| { - NAMED_ROUTES - .iter() - .find(|route| route.path == path) - .map(|route| route.handler) - }; - let methods_for = |path: &str| { - NAMED_ROUTES + for path in ["/_ts/page-bids", "/__ts/page-bids"] { + let route = NAMED_ROUTES .iter() .find(|route| route.path == path) - .map(|route| route.primary_methods) - .unwrap_or(&[]) - }; - - assert!( - matches!( - handler_for("/_ts/admin/keys/rotate"), - Some(NamedRouteHandler::RotateKey) - ), - "canonical /_ts/admin/keys/rotate must map to the rotate handler" - ); - assert!( - matches!( - handler_for("/_ts/admin/keys/deactivate"), - Some(NamedRouteHandler::DeactivateKey) - ), - "canonical /_ts/admin/keys/deactivate must map to the deactivate handler" - ); - assert!( - matches!( - handler_for("/admin/keys/rotate"), - Some(NamedRouteHandler::LegacyAdminDenied) - ), - "legacy /admin/keys/rotate must map to the local deny handler, not the key handler" - ); - assert!( - matches!( - handler_for("/admin/keys/deactivate"), - Some(NamedRouteHandler::LegacyAdminDenied) - ), - "legacy /admin/keys/deactivate must map to the local deny handler, not the key handler" - ); - - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - for method in super::publisher_fallback_methods() { - assert!( - methods_for(path).contains(&method), - "legacy {method} {path} must route to the local deny handler, not publisher fallback" - ); - } - } - } - - #[test] - fn cache_purge_claims_every_method_that_could_reach_the_publisher() { - // The guard this route exists behind. `enforce_basic_auth` authenticates on the raw - // path and leaves the `Authorization` header attached, so any method this route does - // not claim falls through to the publisher fallback carrying the shared admin - // credential to the origin. Asserted against the fallback list itself rather than a - // copy of it, so a method added there cannot quietly open a hole here. - let route = NAMED_ROUTES - .iter() - .find(|route| route.path == "/_ts/admin/cache/purge") - .expect("cache purge must be a named route"); + .unwrap_or_else(|| panic!("{path} should be registered")); - for method in super::publisher_fallback_methods() { assert!( - route.primary_methods.contains(&method), - "{method} /_ts/admin/cache/purge must be claimed, or it reaches the publisher \ - with the admin credential attached" + matches!(route.handler, NamedRouteHandler::PageBids), + "{path} must map to the page-bids handler" + ); + assert_eq!( + route.primary_methods, + &[Method::GET, Method::OPTIONS], + "{path} must handle GET and OPTIONS directly, not fall through to the publisher" ); } - assert!(matches!(route.handler, NamedRouteHandler::AdminCachePurge)); } #[test] - fn cache_purge_has_no_legacy_unauthenticated_alias() { - // The production basic-auth regex is `^/_ts/admin`. An `/admin/...` spelling would - // not match it, so it must not exist at all. - assert!( - !NAMED_ROUTES - .iter() - .any(|route| route.path == "/admin/cache/purge"), - "an /admin-prefixed alias would sit outside the basic-auth regex" - ); - } - - #[test] - fn admin_ec_lookup_routes_are_registered() { - // Both lookup shapes must be explicitly routed to the admin EC - // handler: the bare cookie-based route and the parameterized route. - // Leaving either unrouted would fall through to the publisher - // fallback, forwarding the caller's `Authorization` header to the - // origin. - for path in ["/_ts/admin/ec", "/_ts/admin/ec/{id}"] { - let route = NAMED_ROUTES - .iter() - .find(|route| route.path == path) - .unwrap_or_else(|| panic!("{path} must be a named route")); - assert!( - matches!(route.handler, NamedRouteHandler::AdminEcLookup), - "{path} must map to the admin EC lookup handler" - ); - assert_eq!( - route.primary_methods, - &[Method::GET], - "{path} must have GET as its only primary method" - ); - } - - let eids_route = NAMED_ROUTES - .iter() - .find(|route| route.path == "/_ts/admin/eids") - .expect("should register /_ts/admin/eids as a named route"); - assert!( - matches!(eids_route.handler, NamedRouteHandler::AdminEidsLookup), - "/_ts/admin/eids must map to the admin EIDs lookup handler" - ); - assert_eq!( - eids_route.primary_methods, - &[Method::GET], - "/_ts/admin/eids must have GET as its only primary method" - ); - } - - #[test] - fn page_bids_serves_canonical_path_and_deprecated_alias() { - // The SPA re-auction endpoint lives at the canonical single-underscore - // `/_ts/page-bids`, matching every other internal route. The deprecated - // `/__ts/page-bids` alias must stay registered to the same handler with - // the same methods until pre-rename tsjs bundles age out of browser - // caches — dropping it would leave those clients without ads on SPA - // navigations. - // - // The paths are literals, not `PAGE_BIDS_PATH` / `PAGE_BIDS_LEGACY_PATH`. - // Looking a route up by the same const it was registered with is - // tautological: it keeps passing if the const's value changes, which is - // exactly the break that would silently desync the server from the tsjs - // client's hardcoded fetch path. Pin the consts to their literals too so - // a rename has to be deliberate. - assert_eq!( - PAGE_BIDS_PATH, "/_ts/page-bids", - "canonical page-bids path must match the path tsjs fetches" - ); - assert_eq!( - PAGE_BIDS_LEGACY_PATH, "/__ts/page-bids", - "legacy alias must match the path pre-rename tsjs bundles fetch" - ); - - for path in ["/_ts/page-bids", "/__ts/page-bids"] { - let route = NAMED_ROUTES - .iter() - .find(|route| route.path == path) - .unwrap_or_else(|| panic!("{path} should be registered")); - - assert!( - matches!(route.handler, NamedRouteHandler::PageBids), - "{path} must map to the page-bids handler" - ); - assert_eq!( - route.primary_methods, - &[Method::GET, Method::OPTIONS], - "{path} must handle GET and OPTIONS directly, not fall through to the publisher" - ); - } - } - - #[test] - fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { - // Regression for the credential-leak finding: with a production-shaped - // config (only `^/_ts/admin` is auth-gated, so `/admin/keys/*` is NOT - // matched by any handler), any publisher-fallback method to a legacy - // alias carrying an `Authorization` header must be denied locally with - // 404 — never proxied to the publisher origin (which would leak the - // admin credentials and the key-management body). A publisher-fallback - // proxy without a backend would surface as a 5xx, so a 404 proves the - // deny route ran instead. + fn key_administration_paths_are_answered_here_and_never_proxied() { + // A request to a key administration path carries an operator's + // credentials and a key payload. Every publisher-fallback method must + // be answered here with 404 and never proxied to the publisher origin, + // which would be handed both. A publisher-fallback proxy without a + // backend would surface as a 5xx, so a 404 proves the request was + // answered here. let settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" @@ -2022,79 +2140,52 @@ mod tests { proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse production-shaped settings"); let state = build_state_from_settings(settings).expect("should build state"); let router = TrustedServerApp::routes_for_state(&state); - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { + for path in [ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/admin/keys/rotate", + "/admin/keys/deactivate", + ] { for method in super::publisher_fallback_methods() { let req = request_builder() .method(method.clone()) .uri(format!("https://test-publisher.com{path}")) .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") .body(Body::from("{\"key_id\":\"leak-me\"}")) - .expect("should build authorized legacy-alias request"); + .expect("should build authorized key administration request"); let response = route(&router, req); assert_eq!( response.status(), StatusCode::NOT_FOUND, - "{method} {path} with Authorization must be denied locally (404), not proxied to publisher" + "{method} {path} with Authorization must be answered here (404), not proxied to publisher" ); } } } #[test] - fn authenticated_admin_diagnostic_fallback_is_denied_locally() { + fn an_authenticated_request_to_a_closed_path_is_answered_here() { let router = test_router(); let ec_id = format!("{}.abc123", "a".repeat(64)); - let valid_paths = [ + for path in [ "/_ts/admin/ec".to_owned(), format!("/_ts/admin/ec/{ec_id}"), "/_ts/admin/eids".to_owned(), - ]; - - for path in valid_paths { - for method in [ - Method::POST, - Method::HEAD, - Method::OPTIONS, - Method::PUT, - Method::PATCH, - Method::DELETE, - ] { - let request = request_builder() - .method(method.clone()) - .uri(format!("https://test-publisher.com{path}")) - .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(Body::from("sensitive-admin-body")) - .expect("should build authenticated admin request"); - let response = route(&router, request); - - assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); - assert_eq!( - response - .headers() - .get(header::ALLOW) - .and_then(|v| v.to_str().ok()), - Some("GET") - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } - } - - for path in [ "/_ts/admin/ec/".to_owned(), format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), @@ -2102,14 +2193,13 @@ mod tests { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), - // Percent-encoded separators match the `^/_ts/admin` basic-auth - // handler but not a literal-slash namespace check, so they must be - // reserved before publisher fallback forwards credentials upstream. + // A check for a literal slash would miss a percent-encoded separator, + // so these are closed before the publisher fallback forwards + // credentials upstream. "/_ts/admin%2Fec".to_owned(), "/_ts/admin%2fec".to_owned(), - // Retired non-`/_ts` alias namespace: only the two exact paths are - // routed to a local deny, so descendants and encoded separators must - // be reserved at the shared fallback boundary. + // The alias outside `/_ts`, with its descendants and encoded + // separators. "/admin/keys".to_owned(), "/admin/keys/rotate/extra".to_owned(), "/admin/keys%2Frotate".to_owned(), @@ -2233,6 +2323,123 @@ mod tests { ); } + #[test] + fn dispatch_permissions_answers_as_data_and_as_a_page_and_writes_nothing() { + let router = test_router(); + + let data = route(&router, empty_request(Method::GET, "/_ts/permissions.json")); + assert_eq!(data.status(), StatusCode::OK, "the data form should answer"); + assert_eq!(data.headers()[header::CONTENT_TYPE], "application/json"); + assert_eq!(data.headers()[header::CACHE_CONTROL], "no-store"); + assert!( + data.headers().get(header::SET_COOKIE).is_none(), + "should write nothing for the reader" + ); + assert!( + data.extensions().get::().is_none(), + "should not run the Edge Cookie lifecycle for a read" + ); + let payload: serde_json::Value = + serde_json::from_slice(&data.into_body().into_bytes().unwrap_or_default()) + .expect("the data form should be JSON"); + for key in ["set", "awaiting", "signals", "tdls", "modules"] { + assert!( + payload.get(key).is_some(), + "the resolution should carry `{key}`" + ); + } + + let page = route(&router, empty_request(Method::GET, "/_ts/permissions")); + assert_eq!(page.status(), StatusCode::OK, "the page should answer"); + assert_eq!( + page.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + } + + #[test] + fn dispatch_config_answers_the_masked_settings_as_data_and_as_a_page() { + let router = test_router(); + + let data = route(&router, empty_request(Method::GET, "/_ts/config.json")); + assert_eq!(data.status(), StatusCode::OK, "the data form should answer"); + assert_eq!(data.headers()[header::CONTENT_TYPE], "application/json"); + assert_eq!(data.headers()[header::CACHE_CONTROL], "no-store"); + assert!( + data.extensions().get::().is_none(), + "should not run the Edge Cookie lifecycle for a read" + ); + let body = data.into_body().into_bytes().unwrap_or_default(); + let payload: serde_json::Value = + serde_json::from_slice(&body).expect("the data form should be JSON"); + assert_eq!( + payload["settings"]["publisher"]["domain"], "test-publisher.com", + "should show an ordinary setting" + ); + assert_eq!( + payload["settings"]["publisher"]["proxy_secret"], "XXXX", + "should mask a secret" + ); + assert!( + !String::from_utf8_lossy(&body).contains("unit-test-proxy-secret"), + "should not carry the secret anywhere" + ); + + let page = route(&router, empty_request(Method::GET, "/_ts/config")); + assert_eq!(page.status(), StatusCode::OK, "the page should answer"); + assert_eq!( + page.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + } + + #[test] + fn dispatch_data_answers_a_browser_opening_the_page_and_nothing_else() { + let router = test_router(); + + let refused = route(&router, empty_request(Method::GET, "/_ts/data")); + assert_eq!( + refused.status(), + StatusCode::FORBIDDEN, + "should refuse a request that is not a browser opening the page" + ); + assert!( + refused + .extensions() + .get::() + .is_none(), + "should not run the Edge Cookie lifecycle for a refusal" + ); + + let mut opened = empty_request(Method::GET, "/_ts/data"); + opened + .headers_mut() + .insert("sec-fetch-mode", HeaderValue::from_static("navigate")); + opened + .headers_mut() + .insert("sec-fetch-dest", HeaderValue::from_static("document")); + let page = route(&router, opened); + assert_eq!(page.status(), StatusCode::OK, "the page should answer"); + assert_eq!( + page.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + assert_eq!(page.headers()[header::CACHE_CONTROL], "no-store, private"); + assert!( + page.headers().get(header::SET_COOKIE).is_none(), + "should write nothing for the reader" + ); + assert!( + page.extensions().get::().is_none(), + "should not run the Edge Cookie lifecycle for a read" + ); + let body = page.into_body().into_bytes().unwrap_or_default(); + assert!( + String::from_utf8_lossy(&body).contains("keeps no record"), + "should say a deployment with no identity graph keeps no record" + ); + } + #[test] fn dispatch_clear_tester_is_disabled_by_default() { let router = test_router(); @@ -2249,6 +2456,230 @@ mod tests { ); } + /// Settings whose attestation pages sit at `endpoint`. + fn settings_with_attestation_at(endpoint: &str) -> Settings { + let mut settings = test_settings(); + settings.attestation = Some(trusted_server_core::attestation::AttestationConfig { + endpoint: endpoint.to_owned(), + operator: "Example Operator".to_owned(), + context: "example-attestation:v1".to_owned(), + verify_url: "https://verifier.example/verify".to_owned(), + }); + settings + } + + /// The router of a deployment whose attestation pages sit at `endpoint`. + fn router_with_attestation_at(endpoint: &str) -> RouterService { + let state = build_state_from_settings(settings_with_attestation_at(endpoint)) + .expect("should build test state"); + TrustedServerApp::routes_for_state(&state) + } + + fn header_text(response: &Response, name: header::HeaderName) -> String { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned() + } + + /// A deployment whose endpoint sits under a prefix of its own answers + /// everything beneath that prefix here. Reads of the two attestation + /// addresses reach the handler, the prefix lists its pages, and anything + /// else is not found rather than sent to the publisher's origin. Test + /// builds carry no signing key, so the handler answers 503 in JSON. + #[test] + fn an_owned_prefix_is_answered_here_and_never_by_the_origin() { + let router = router_with_attestation_at("/_ex/attestation"); + + let cases = [ + (Method::GET, "/_ex", 200, "text/html"), + (Method::GET, "/_ex/", 200, "text/html"), + ( + Method::GET, + "/_ex/attestation.json", + 503, + "application/json", + ), + ( + Method::HEAD, + "/_ex/attestation.json", + 503, + "application/json", + ), + (Method::GET, "/_ex/attestation", 503, "application/json"), + (Method::POST, "/_ex/attestation", 404, "text/html"), + (Method::PUT, "/_ex/attestation.json", 404, "text/html"), + (Method::DELETE, "/_ex", 404, "text/html"), + (Method::GET, "/_ex/unknown/page", 404, "text/html"), + ]; + for (method, path, status, content_type) in cases { + let case = format!("{method} {path}"); + let response = route(&router, empty_request(method, path)); + assert_eq!( + response.status().as_u16(), + status, + "should answer {case} here" + ); + assert!( + header_text(&response, header::CONTENT_TYPE).starts_with(content_type), + "should answer {case} as {content_type}" + ); + assert_eq!( + header_text(&response, header::CACHE_CONTROL), + "no-store", + "should keep the answer to {case} out of every cache" + ); + } + } + + /// When the endpoint is not the default, reads of the default address are + /// sent to the endpoint, keeping the nonce. With the default endpoint the + /// default address is answered directly, and no other address redirects. + #[test] + fn the_default_address_redirects_to_the_endpoint_the_settings_name() { + use trusted_server_core::attestation::DEFAULT_ENDPOINT; + + let router = router_with_attestation_at("/_ex/attestation"); + for (method, path, location) in [ + (Method::GET, "/_ts/attestation", "/_ex/attestation"), + (Method::HEAD, "/_ts/attestation", "/_ex/attestation"), + ( + Method::GET, + "/_ts/attestation.json?nonce=abc123", + "/_ex/attestation.json?nonce=abc123", + ), + ] { + let case = format!("{method} {path}"); + let response = route(&router, empty_request(method, path)); + assert_eq!(response.status().as_u16(), 301, "should redirect {case}"); + assert_eq!( + header_text(&response, header::LOCATION), + location, + "should send {case} to the endpoint with its query kept" + ); + assert_eq!( + header_text(&response, header::CACHE_CONTROL), + "no-store", + "should keep the redirect of {case} out of every cache" + ); + } + + let router = router_with_attestation_at(DEFAULT_ENDPOINT); + for path in ["/_ts/attestation", "/_ts/attestation.json"] { + let response = route(&router, empty_request(Method::GET, path)); + assert_eq!( + response.status().as_u16(), + 503, + "{path} should reach the handler, which has no key built in" + ); + assert!( + response.headers().get(header::LOCATION).is_none(), + "{path} should not redirect under the default endpoint" + ); + } + } + + /// Without the section the two addresses belong to the publisher, so a + /// deployment that serves no attestation takes nothing from its origin. + #[test] + fn without_the_section_no_attestation_route_is_registered() { + let registered: Vec = test_router() + .routes() + .iter() + .map(|route| route.path().to_owned()) + .filter(|path| path.contains("attestation")) + .collect(); + + assert!( + registered.is_empty(), + "should register no attestation route, found {registered:?}" + ); + } + + /// Every route this router registers beneath an underscore prefix is one + /// of the addresses an attestation endpoint may not take, so a new fixed + /// route cannot leave an endpoint free to register its address twice. + #[test] + fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() { + use trusted_server_core::attestation::RESERVED_PATHS; + + for route in test_router().routes() { + let path = route.path(); + if path.starts_with("/_") { + assert!( + RESERVED_PATHS.contains(&path), + "{path} should be listed in attestation::RESERVED_PATHS" + ); + } + } + } + + /// Every endpoint the settings accept builds a router whose attestation + /// pages answer, because a second route at one address panics when the + /// router is built. + #[test] + fn every_endpoint_the_settings_accept_builds_a_router() { + use trusted_server_core::attestation::{DEFAULT_ENDPOINT, validate_endpoint}; + + let mut candidates: Vec = test_router() + .routes() + .iter() + .map(edgezero_core::router::RouteInfo::path) + .filter(|path| path.starts_with("/_")) + .map(|path| { + let page = path.strip_suffix(".json").unwrap_or(path); + page.split('/') + .map(|segment| { + if segment.starts_with('{') { + "proof" + } else { + segment + } + }) + .collect::>() + .join("/") + }) + .collect(); + candidates.extend( + [ + DEFAULT_ENDPOINT, + "/_ts/proof", + "/_ts/api/v1/proof", + "/__ts/proof", + ] + .map(str::to_owned), + ); + candidates.sort(); + candidates.dedup(); + + let accepted: Vec = candidates + .into_iter() + .filter(|endpoint| validate_endpoint(endpoint).is_ok()) + .collect(); + for endpoint in &accepted { + let router = router_with_attestation_at(endpoint); + let json = format!("{endpoint}.json"); + let response = route(&router, empty_request(Method::GET, &json)); + assert_eq!( + response.status().as_u16(), + 503, + "{json} should reach the handler, which has no key built in" + ); + } + assert_eq!( + accepted, + [ + "/__ts/proof", + "/_ts/api/v1/proof", + DEFAULT_ENDPOINT, + "/_ts/proof" + ], + "should accept only the endpoints no fixed route answers" + ); + } + #[test] fn dispatch_clear_tester_clears_cookie_on_configured_domain() { let mut settings = test_settings(); @@ -2275,6 +2706,25 @@ mod tests { ); } + #[test] + fn dispatch_ec_resolve_routes_to_resolve_handler() { + // Parity guard: POST /_ts/api/v1/ec/resolve must reach the resolve + // handler, not the publisher fallback or a router-level 405. The test + // request carries no Origin, so the handler's own origin guard answers + // 403, proving the request was handled here rather than proxied to + // the publisher origin (which would error without a live backend). + let router = test_router(); + let response = + block_on(router.oneshot(empty_request(Method::POST, "/_ts/api/v1/ec/resolve"))) + .expect("should route request"); + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "POST ec/resolve should reach the resolve handler's origin guard, not the publisher" + ); + } + #[test] fn dispatch_fallback_attaches_ec_finalize_state() { // The publisher fallback must thread EC finalize state to the entry @@ -2525,103 +2975,6 @@ mod tests { ); } - #[test] - fn admin_eids_diagnostic_skips_ec_finalization() { - let router = test_router(); - let ec_id = format!("{}.abc123", "a".repeat(64)); - let eids = serde_json::json!([{ - "source": "example.com", - "uids": [{ "id": "example-uid", "atype": 1 }] - }]); - let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); - let mut request = request_builder() - .method(Method::GET) - .uri("https://test-publisher.com/_ts/admin/eids") - .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header( - header::COOKIE, - format!("ts-ec={ec_id}; ts-eids={eids_cookie}; sharedId=example-shared-id"), - ) - .body(Body::empty()) - .expect("should build authenticated EIDs diagnostic request"); - request.extensions_mut().insert(DeviceSignals::derive( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", - Some("t13d1516h2_8daaf6152771_b186095e22b6"), - Some("1:65536;2:0;4:6291456;6:262144"), - )); - - let response = route(&router, request); - - assert_eq!(response.status(), StatusCode::OK); - assert!( - response - .extensions() - .get::() - .is_none(), - "admin EIDs diagnostics should not attach EC finalization state" - ); - } - - #[test] - fn admin_ec_diagnostic_skips_ec_finalization() { - let router = test_router(); - let ec_id = format!("{}.abc123", "a".repeat(64)); - let eids = serde_json::json!([{ - "source": "example.com", - "uids": [{ "id": "example-uid", "atype": 1 }] - }]); - let eids_cookie = base64::engine::general_purpose::STANDARD.encode(eids.to_string()); - let mut request = request_builder() - .method(Method::GET) - .uri(format!("https://test-publisher.com/_ts/admin/ec/{ec_id}")) - .header(header::AUTHORIZATION, "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header( - header::COOKIE, - format!("ts-ec={ec_id}; ts-eids={eids_cookie}; sharedId=example-shared-id"), - ) - .body(Body::empty()) - .expect("should build authenticated EC diagnostic request"); - request.extensions_mut().insert(DeviceSignals::derive( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", - Some("t13d1516h2_8daaf6152771_b186095e22b6"), - Some("1:65536;2:0;4:6291456;6:262144"), - )); - - let response = route(&router, request); - - assert_eq!( - response.status(), - StatusCode::NOT_IMPLEMENTED, - "configured admin EC handler should run and report the unavailable test KV graph" - ); - assert!( - response - .extensions() - .get::() - .is_none(), - "admin EC diagnostics should not attach EC finalization state" - ); - assert!( - response.headers().get(header::SET_COOKIE).is_none(), - "admin EC diagnostics should not mutate the EC cookie" - ); - } - - #[test] - fn admin_ec_route_without_credentials_returns_401() { - let router = test_router(); - - let response = route(&router, empty_request(Method::GET, "/_ts/admin/ec")); - - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - response.headers().contains_key(header::WWW_AUTHENTICATE), - "admin EC 401 should include the Basic authentication challenge" - ); - } - #[test] fn dispatch_head_on_named_get_route_falls_through_to_publisher_fallback() { // Regression guard: HEAD /first-party/proxy must reach the publisher @@ -2651,7 +3004,7 @@ mod tests { // does not inject TS headers at this layer. // // The full-system guarantee (TS headers on ALL responses including these 405s) - // is maintained by the entry-point apply_finalize_headers call in main.rs. + // is maintained by the entry-point apply_finalize_headers call in lib.rs. let router = test_router(); let req = empty_request( Method::from_bytes(b"TRACE").expect("should parse TRACE"), @@ -2670,7 +3023,7 @@ mod tests { .headers() .get(HEADER_X_GEO_INFO_AVAILABLE) .is_none(), - "router-level 405 bypasses FinalizeResponseMiddleware; main.rs entry-point covers this" + "router-level 405 bypasses FinalizeResponseMiddleware; lib.rs entry-point covers this" ); } @@ -2684,11 +3037,6 @@ mod tests { // ran instead of the publisher fallback (which always attaches one). let settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" @@ -2696,12 +3044,16 @@ mod tests { proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + [geo] + assume_single_jurisdiction = true + [request_signing] enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" [proxy] @@ -2804,304 +3156,6 @@ mod tests { .build() } - #[derive(Default)] - struct DispatchTemplateCache { - entries: Arc>>, - } - - struct DispatchTemplateReservation { - entries: Arc>>, - key: TemplateCacheKey, - } - - impl PlatformTemplateCacheReservation for DispatchTemplateReservation { - fn insert( - self: Box, - metadata: &TemplateMetadata, - body: Vec, - _max_age: Duration, - ) -> Result<(), TemplateCacheError> { - self.entries.lock().expect("should lock entries").insert( - self.key.to_cache_key(), - TemplateEntry { - metadata: metadata.clone(), - body, - }, - ); - Ok(()) - } - - fn cancel(self: Box) -> Result<(), TemplateCacheError> { - Ok(()) - } - } - - #[async_trait::async_trait(?Send)] - impl PlatformTemplateCache for DispatchTemplateCache { - async fn lookup_or_reserve( - &self, - key: &TemplateCacheKey, - ) -> Result { - if let Some(entry) = self - .entries - .lock() - .expect("should lock entries") - .get(&key.to_cache_key()) - .cloned() - { - return Ok(TemplateCacheLookup::Hit(entry)); - } - Ok(TemplateCacheLookup::Reserved( - TemplateCacheReservation::new(Box::new(DispatchTemplateReservation { - entries: Arc::clone(&self.entries), - key: key.clone(), - })), - )) - } - - async fn get(&self, key: &TemplateCacheKey) -> Result { - self.entries - .lock() - .expect("should lock entries") - .get(&key.to_cache_key()) - .cloned() - .ok_or(TemplateCacheMiss::NotFound) - } - - async fn put( - &self, - key: &TemplateCacheKey, - metadata: &TemplateMetadata, - body: Vec, - _max_age: Duration, - ) -> Result<(), TemplateCacheError> { - self.entries.lock().expect("should lock entries").insert( - key.to_cache_key(), - TemplateEntry { - metadata: metadata.clone(), - body, - }, - ); - Ok(()) - } - - async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { - self.entries - .lock() - .expect("should lock entries") - .remove(&key.to_cache_key()); - Ok(()) - } - - /// A no-op beyond succeeding: this double stores by cache key, so it cannot - /// resolve a surrogate key to entries the way the platform does. - async fn purge_url_surrogate_key(&self, _key: &str) -> Result<(), TemplateCacheError> { - Ok(()) - } - - async fn purge_all(&self) -> Result<(), TemplateCacheError> { - self.entries.lock().expect("should lock entries").clear(); - Ok(()) - } - } - - #[derive(Default)] - struct DispatchOriginClient { - calls: AtomicUsize, - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for DispatchOriginClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - self.calls.fetch_add(1, Ordering::Relaxed); - let response = edgezero_core::http::response_builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "text/html; charset=utf-8") - .header(header::CACHE_CONTROL, "public, max-age=300") - .body(Body::from( - b"origin".as_ref(), - )) - .map_err(|_| Report::new(PlatformError::HttpClient))?; - Ok(PlatformResponse::new(response)) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - #[test] - fn dispatch_edge_authenticated_esi_request_stores_then_hits_template() { - let settings = Arc::new( - Settings::from_toml( - r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [auction] - enabled = true - providers = {} - - [creative_opportunities] - gam_network_id = "99999" - assembly_mode = "esi" - - [[creative_opportunities.slot]] - id = "test-slot" - page_patterns = ["/secure/article"] - formats = [{ width = 728, height = 90 }] - "#, - ) - .expect("should parse dispatch cache settings"), - ); - let cache = Arc::new(DispatchTemplateCache::default()); - let origin = Arc::new(DispatchOriginClient::default()); - let services = RuntimeServices::builder() - .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) - .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) - .kv_store(Arc::new(NoopKvStore) as Arc) - .template_cache(Arc::clone(&cache) as Arc) - .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) - .backend(Arc::new(FixedBackend)) - .http_client(Arc::clone(&origin) as Arc) - .geo(Arc::new(crate::platform::FastlyPlatformGeo)) - .client_info(ClientInfo::default()) - .build(); - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); - let registry = Arc::new( - IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) - .expect("should build integration registry"), - ); - let orchestrator = Arc::new( - build_orchestrator_with_plan(plan, &settings) - .expect("should build auction orchestrator"), - ); - - let handler = { - let settings = Arc::clone(&settings); - let services = services.clone(); - let registry = Arc::clone(®istry); - let orchestrator = Arc::clone(&orchestrator); - move |ctx: RequestContext| { - let settings = Arc::clone(&settings); - let services = services.clone(); - let registry = Arc::clone(®istry); - let orchestrator = Arc::clone(&orchestrator); - Box::pin(async move { - let request = ctx.into_request(); - let method = request.method().clone(); - let mut ec_context = - match EcContext::read_from_request(&settings, &request, &services) { - Ok(context) => context, - Err(report) => return Ok(super::http_error(&report)), - }; - let response = match handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: settings.creative_opportunity_slots(), - registry: None, - }, - request, - EdgeCacheHeader::SurrogateControl, - ) - .await - { - Ok(response) => response, - Err(report) => return Ok(super::http_error(&report)), - }; - match publisher_response_into_streaming_response( - response, - &method, - Arc::clone(&settings), - ®istry, - orchestrator, - services, - ) - .await - { - Ok(response) => Ok(response), - Err(report) => Ok(super::http_error(&report)), - } - }) as HandlerFuture - } - }; - let router = RouterService::builder() - .middleware(crate::middleware::AuthMiddleware::new(Arc::clone( - &settings, - ))) - .route("/secure/article", Method::GET, handler) - .build(); - let authorized_request = || { - request_builder() - .method(Method::GET) - .uri("https://test-publisher.com/secure/article") - .header(header::HOST, "test-publisher.com") - .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") - .header("sec-fetch-dest", "document") - .header("sec-fetch-mode", "navigate") - .body(Body::empty()) - .expect("should build authorized navigation") - }; - - let cold = route(&router, authorized_request()); - assert_eq!( - cold.headers() - .get("x-ts-template-cache") - .and_then(|value| value.to_str().ok()), - Some("miss-stored") - ); - block_on(cold.into_body().into_bytes_bounded(1024 * 1024)) - .expect("should drain cold response"); - - let warm = route(&router, authorized_request()); - assert_eq!( - warm.headers() - .get("x-ts-template-cache") - .and_then(|value| value.to_str().ok()), - Some("hit") - ); - block_on(warm.into_body().into_bytes_bounded(1024 * 1024)) - .expect("should drain warm response"); - assert_eq!( - origin.calls.load(Ordering::Relaxed), - 1, - "the warm dispatch must not fetch the publisher origin" - ); - } - #[test] fn dispatch_asset_fallback_streams_origin_body_without_buffering() { // Regression guard for the EdgeZero asset streaming cutover: a successful @@ -3111,11 +3165,6 @@ mod tests { // pipe straight to the client via `stream_to_client`. let settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" @@ -3123,12 +3172,16 @@ mod tests { proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + [geo] + assume_single_jurisdiction = true + [request_signing] enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" [proxy] diff --git a/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs new file mode 100644 index 0000000000..d900fe7524 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/app/seam_probe_tests.rs @@ -0,0 +1,669 @@ +//! Round-trip tests for the integration seam on the Fastly adapter, driven by +//! a crate `trusted-server-core` knows nothing about. +//! +//! `trusted-server-integration-seam-probe` registers every capability the seam +//! exposes, and each test asserts an observable outcome through this adapter's +//! own router: the bytes served, the JSON a route returns, or the error a +//! startup check produces. The Axum adapter's seam tests assert the same +//! outcomes on the dev server. + +use std::sync::Arc; + +use edgezero_core::body::Body; +use edgezero_core::http::{HeaderMap, Method, Response, request_builder}; +use edgezero_core::router::RouterService; +use error_stack::Report; +use futures::executor::block_on; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::OwnedRequestInfo; +use trusted_server_core::integrations::{IntegrationBuilder, IntegrationRegistration}; +use trusted_server_core::module_context::{ModuleContext, ModuleRequest}; +use trusted_server_core::settings::Settings; +use trusted_server_core::tsjs::tsjs_script_src; +use trusted_server_core::tsjs_bundle::{JsModulePart, compile_time_parts}; +use trusted_server_integration_seam_probe as seam_probe; + +use super::{ + AppState, TrustedServerApp, build_finalize_services, build_state_from_settings, + build_state_with_registrations, register_integrations, +}; + +/// Largest response body these tests read, ample for the served script. +const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; + +/// The probe's own section and table, selecting it with the country the geo +/// assertions expect. +const PROBE_BLOCK: &str = r#" + [testing] + modules = ["seam-probe"] + + [testing.seam-probe] + country = "ZZ" +"#; + +/// The built-in Edge Cookie module, for the tests that do not select the +/// probe's. +const HMAC_BLOCK: &str = r#" + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" +"#; + +/// Settings shaped like this adapter's route tests', with `extra` appended. +fn settings_with(extra: &str) -> Settings { + // A deployment that runs an Edge Cookie module with no geo module + // acknowledges that every request resolves at the top of the rules tree. + // Tests that select a geo module write their own `[geo]` table. + let geo = if extra.contains("[geo]") { + "" + } else { + " + [geo] + assume_single_jurisdiction = true +" + }; + Settings::from_toml(&format!( + r#" + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "seam-probe-test-proxy-secret" + + {extra} + {geo} + "# + )) + .expect("should parse seam probe test settings") +} + +/// Builds this adapter's application state with the supplied builders +/// composed in. +fn state_with(settings: Settings, integrations: &[IntegrationBuilder]) -> Arc { + build_state_with_registrations(settings, integrations) + .expect("should build state from the composed builders") +} + +/// Builds this adapter's router with the supplied builders composed in. +fn router_with(settings: Settings, integrations: &[IntegrationBuilder]) -> RouterService { + TrustedServerApp::routes_for_state(&state_with(settings, integrations)) +} + +/// Sends one GET and returns the response. A `token` carries the probe's +/// counting header, so each test counts into an entry of its own. +fn get(router: &RouterService, path: &str, token: Option<&str>) -> Response { + let mut builder = request_builder() + .method(Method::GET) + .uri(format!("https://test-publisher.com{path}")); + if let Some(token) = token { + builder = builder.header(seam_probe::SEAM_PROBE_COUNT_HEADER, token); + } + let request = builder.body(Body::empty()).expect("should build request"); + block_on(router.oneshot(request)).expect("should route request") +} + +/// Reads a response body as a string. +fn body_text(response: Response) -> String { + let bytes = block_on(response.into_body().into_bytes_bounded(MAX_BODY_BYTES)) + .expect("should read the response body"); + String::from_utf8(bytes.to_vec()).expect("should be UTF-8") +} + +/// The parts the unified bundle is composed from when the probe is the only +/// selected module: the always-on `creative` module, then the module the +/// probe's registration carries. Composition puts core first. +fn expected_bundle_parts() -> Vec { + let mut parts = compile_time_parts(&["creative"]); + parts.push(JsModulePart { + id: seam_probe::SEAM_PROBE_ID, + source: seam_probe::PROBE_JS, + sha256: seam_probe::PROBE_JS_SHA256, + }); + parts +} + +/// A module a crate outside core carries is served in the unified bundle, +/// under the hash composed from its content, and marked immutable when the +/// request's `?v=` matches that hash. +#[test] +fn carried_module_is_served_in_the_unified_bundle_under_its_composed_hash() { + let router = router_with( + settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")), + &[seam_probe::builder()], + ); + let source = tsjs_script_src(&expected_bundle_parts()); + + let response = get(&router, &source, None); + + assert_eq!( + response.status().as_u16(), + 200, + "should serve the unified bundle" + ); + let cache_control = response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned(); + assert!( + cache_control.contains("immutable"), + "a `?v=` matching the composed hash should be served immutable, got `{cache_control}`" + ); + + let body = body_text(response); + let core = JsModulePart::compile_time("core").expect("should compile core in"); + + assert!( + body.starts_with(core.source), + "the bundle should start with core" + ); + assert!( + body.contains(seam_probe::PROBE_JS), + "the bundle should carry the probe's module source" + ); +} + +/// The probe's proxy route reports the country its own geo module resolved, +/// selected by `[geo] module`, and that its request preparer ran exactly once +/// before the route was dispatched. +#[test] +fn proxy_route_reports_the_modules_geo_and_that_the_preparer_ran() { + let settings = settings_with(&format!( + r#" + [geo] + module = "testing.seam-probe" + {HMAC_BLOCK} + {PROBE_BLOCK} + "# + )); + let router = router_with(settings, &[seam_probe::builder()]); + + let response = get(&router, seam_probe::SEAM_PROBE_REPORT_PATH, None); + + assert_eq!( + response.status().as_u16(), + 200, + "the probe's proxy route should be dispatched" + ); + + let body = body_text(response); + let report: serde_json::Value = + serde_json::from_str(&body).expect("the route should return JSON"); + + assert_eq!( + report["module"], + seam_probe::module_name(), + "the route should identify the module: {body}" + ); + assert_eq!( + report["geo_country"], "ZZ", + "the route should see the country the module's own geo module resolved: {body}" + ); + assert_eq!( + report["request_preparer_runs"], 1, + "the adapter should run the module's request preparer exactly once: {body}" + ); +} + +/// With `refuse_all` selected every response this adapter finalizes says +/// not to index, the response of another route included. The file itself is +/// the publisher path's, made by the `robots_txt` middleware the entry +/// names, so no route of this server names the path. +#[test] +fn refusing_every_crawler_tags_every_response() { + let settings = settings_with(&format!( + r#" + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + {HMAC_BLOCK} + {PROBE_BLOCK} + "# + )); + let router = router_with(settings, &[seam_probe::builder()]); + let tag = |response: &Response| { + response + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + + assert!( + !router + .routes() + .into_iter() + .any(|route| route.path() == "/robots.txt"), + "the file is the publisher path's" + ); + + let other = get(&router, seam_probe::SEAM_PROBE_REPORT_PATH, None); + + assert_eq!( + tag(&other).as_deref(), + Some("noindex, nofollow"), + "no route may be the one page a crawler is allowed to index" + ); +} + +/// Without the section no blanket rule is placed on responses. +#[test] +fn no_blanket_rule_is_placed_on_responses_without_the_section() { + let without = router_with( + settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")), + &[seam_probe::builder()], + ); + + let other = get(&without, seam_probe::SEAM_PROBE_REPORT_PATH, None); + + assert!( + other.headers().get("x-robots-tag").is_none(), + "no blanket rule is placed on responses without the section" + ); +} + +/// A request prepares exactly once whether a named route or the fallback +/// serves it. +#[test] +fn a_named_route_and_the_fallback_each_prepare_the_request_exactly_once() { + let router = router_with( + settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")), + &[seam_probe::builder()], + ); + + // `/first-party/proxy` is a named route whose handler refuses a request + // naming no target, so the request reaches the preparer and no backend. + let named = get(&router, "/first-party/proxy", Some("fastly-named-route")); + + assert_eq!( + named.status().as_u16(), + 502, + "the named route should refuse a request that names no target" + ); + assert_eq!( + seam_probe::prepare_runs_for("fastly-named-route"), + 1, + "a request served by a named route should prepare exactly once" + ); + + let fallback = get( + &router, + seam_probe::SEAM_PROBE_REPORT_PATH, + Some("fastly-fallback-route"), + ); + + assert_eq!( + fallback.status().as_u16(), + 200, + "the probe's proxy route should be dispatched by the fallback" + ); + assert_eq!( + seam_probe::prepare_runs_for("fastly-fallback-route"), + 1, + "a request served by the fallback should prepare exactly once" + ); +} + +/// A `[[fetch]]` entry naming the probe's middleware builds this adapter's +/// state, and one naming the middleware of a module no section selects is a +/// startup error. +#[test] +fn fetch_entry_builds_state_only_when_its_middleware_is_running() { + const ENTRY: &str = r#" + [[fetch]] + media_type = "text/html" + middleware = ["testing.seam-probe"] +"#; + + let selected = settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}{ENTRY}")); + assert!( + build_state_with_registrations(selected, &[seam_probe::builder()]).is_ok(), + "should build state with an entry naming a middleware the module supplies" + ); + + let unselected = settings_with(&format!("{HMAC_BLOCK}{ENTRY}")); + let error = build_state_with_registrations(unselected, &[seam_probe::builder()]) + .err() + .expect("should refuse to start when an entry names a middleware that does not run"); + + let message = error.to_string(); + assert!( + message.contains("[[fetch]] entry 1 names `testing.seam-probe`") + && message.contains("`testing.seam-probe` is a module no section selects"), + "should name the entry and say why the middleware is not running: {message}" + ); +} + +/// A `[[serve]]` entry builds this adapter's state when it names a +/// middleware that runs in that phase, and is a startup error when it names +/// one that does not. +#[test] +fn serve_entry_builds_state_only_with_a_middleware_of_that_phase() { + let settings = |name: &str| { + settings_with(&format!( + r#"{HMAC_BLOCK}{PROBE_BLOCK} + [[serve]] + media_type = "text/html" + middleware = ["{name}"] +"# + )) + }; + + assert!( + build_state_with_registrations( + settings(seam_probe::reader_middleware_name()), + &[seam_probe::builder()] + ) + .is_ok(), + "should build state with a serve entry naming the module's serve middleware" + ); + + let error = build_state_with_registrations( + settings(seam_probe::module_name()), + &[seam_probe::builder()], + ) + .err() + .expect("should refuse to start when a serve entry names a fetch middleware"); + + let message = error.to_string(); + assert!( + message.contains("[[serve]] entry 1 names `testing.seam-probe`") + && message.contains("does not run in that phase. It runs in [[fetch]]"), + "should say which phase the middleware runs in: {message}" + ); +} + +/// `[geo] module` naming a selected module that supplies no geo module is a +/// startup error, raised where this adapter builds its state. +#[test] +fn geo_selector_naming_a_module_without_a_geo_module_fails_at_startup() { + let settings = settings_with(&format!( + r#" + [geo] + module = "testing.seam-probe" + {HMAC_BLOCK} + + [testing] + modules = ["seam-probe"] + + [testing.seam-probe] + country = "ZZ" + declares_geo = false + "# + )); + + let error = build_state_with_registrations(settings, &[seam_probe::builder()]) + .err() + .expect("should refuse to start when the selected module supplies no geo module"); + + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `testing.seam-probe`") + && message.contains("is selected and supplies no geo module"), + "should name the module and the missing capability: {message}" + ); +} + +/// A second builder claiming the probe's id is rejected at startup, and the +/// error names the id and both sources. +#[test] +fn duplicate_integration_id_is_rejected_naming_both_sources() { + /// Source label of the builder that collides with the probe's. + const OTHER_SOURCE: &str = "another-vendor-crate"; + + fn register_nothing( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + fn validate_nothing(_settings: &Settings) -> Result> { + Ok(false) + } + + let extra = [ + seam_probe::builder(), + IntegrationBuilder::new( + seam_probe::SEAM_PROBE_ID, + OTHER_SOURCE, + register_nothing, + validate_nothing, + ), + ]; + + let error = build_state_with_registrations( + settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")), + &extra, + ) + .err() + .expect("should reject two builders claiming one integration id"); + + let message = error.to_string(); + assert!( + message.contains(seam_probe::SEAM_PROBE_ID) + && message.contains(seam_probe::SEAM_PROBE_SOURCE) + && message.contains(OTHER_SOURCE), + "should name the id and both sources: {message}" + ); +} + +/// A demand source a crate outside core supplies builds this adapter's state, +/// and the same settings are refused without that crate's builder. +#[test] +fn demand_source_a_module_supplies_builds_state_only_with_its_builder() { + let settings = || { + settings_with(&format!( + r#" + [demand] + modules = ["probe"] + + [demand.probe] + implementation = "testing.seam-probe" + endpoint = "https://demand.example/openrtb2/auction" + {HMAC_BLOCK} + "# + )) + }; + + let built = build_state_with_registrations(settings(), &[seam_probe::builder()]); + assert!( + built.is_ok(), + "the state should build with the probe's demand source selected: {:?}", + built.err() + ); + + let error = build_state_with_registrations(settings(), &[]) + .err() + .expect("should refuse a demand source no builder in the deployment supplies"); + let message = error.to_string(); + assert!( + message.contains("[demand] `probe` uses implementation `testing.seam-probe`"), + "should name the demand source and the implementation nothing supplies: {message}" + ); +} + +/// This adapter compiles the auction plan with the builders it is given, so +/// an ad server a crate outside core supplies builds its state, and the same +/// settings are refused without that crate's builder. +#[test] +fn ad_server_a_module_supplies_builds_state_only_with_its_builder() { + let settings = || { + settings_with(&format!( + r#" + [ad-server] + module = "probe" + + [ad-server.probe] + implementation = "testing.seam-probe" + {HMAC_BLOCK} + "# + )) + }; + + let built = build_state_with_registrations(settings(), &[seam_probe::builder()]); + assert!( + built.is_ok(), + "the state should build with the probe's ad server selected: {:?}", + built.err() + ); + + let error = build_state_with_registrations(settings(), &[]) + .err() + .expect("should refuse an ad server no builder in the deployment supplies"); + let message = error.to_string(); + assert!( + message.contains("[ad-server] `probe` uses implementation `testing.seam-probe`"), + "should name the ad server and the implementation nothing supplies: {message}" + ); +} + +/// With `[ec] module` and `[device] module` naming the probe, this adapter +/// starts and resolves the Edge Cookie and device modules the probe declares, +/// and the Edge Cookie module creates an identifier the built-in module +/// cannot make. +#[test] +fn ec_and_device_selectors_naming_a_module_resolve_the_modules_it_declares() { + let settings = settings_with(&format!( + r#" + [ec] + module = "testing.seam-probe" + + [device] + module = "testing.seam-probe" + {PROBE_BLOCK} + "# + )); + + let state = state_with(settings, &[seam_probe::builder()]); + + let device = state.registry.device_module().expect( + "`[device] module = \"testing.seam-probe\"` should resolve the module's device module", + ); + assert_eq!( + device.id(), + seam_probe::module_name(), + "the device module should be the one the module declared" + ); + + let module = state.registry.ec_module().expect( + "`[ec] module = \"testing.seam-probe\"` should resolve the module's Edge Cookie module", + ); + assert_eq!( + module.id(), + seam_probe::module_name(), + "the Edge Cookie module should be the one the module declared" + ); + + let services = build_finalize_services( + &state.settings, + Arc::clone(&state.default_kv_store), + Arc::default(), + ); + let request_info = OwnedRequestInfo::new("192.0.2.1".to_owned(), HeaderMap::new()); + let method = Method::GET; + let context = ModuleContext::new(ModuleRequest::new( + &method, + "publisher.example", + "https", + "/", + )) + .with_evidence(&request_info) + .with_services(&services); + let generated = + block_on(module.generate(context.call(module.id(), module.required_permissions()))) + .expect("the module's Edge Cookie module should generate an identifier"); + + assert_eq!( + generated.id.as_deref(), + Some("seam-probe-192.0.2.1"), + "the module's Edge Cookie module should derive the identifier from the request evidence" + ); +} + +/// With `[device] module` naming the probe, the function this adapter's entry +/// point calls for every request classifies the request with the probe's +/// device module, and with the built-in module when the selector is unset. +#[test] +fn device_selector_naming_a_module_classifies_a_request_with_it() { + let request = || { + let mut request = fastly::Request::get("https://test-publisher.com/"); + request.set_header("user-agent", "Mozilla/5.0 (X11; Linux x86_64) Chrome/140.0"); + request + }; + + let selected = state_with( + settings_with(&format!( + r#" + [device] + module = "testing.seam-probe" + {HMAC_BLOCK} + {PROBE_BLOCK} + "# + )), + &[seam_probe::builder()], + ); + let services = build_finalize_services( + &selected.settings, + Arc::clone(&selected.default_kv_store), + Arc::default(), + ); + let signals = block_on(crate::derive_device_signals( + &selected.settings, + selected.registry.device_module(), + &request(), + &services, + )); + assert_eq!( + signals.platform_class.as_deref(), + Some("seam-probe"), + "the probe's device module should classify the request" + ); + + let unset = state_with( + settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}")), + &[seam_probe::builder()], + ); + let services = build_finalize_services( + &unset.settings, + Arc::clone(&unset.default_kv_store), + Arc::default(), + ); + let signals = block_on(crate::derive_device_signals( + &unset.settings, + unset.registry.device_module(), + &request(), + &services, + )); + assert_eq!( + signals.platform_class.as_deref(), + Some("linux"), + "the built-in module should classify the request when the selector is unset" + ); +} + +/// The builders a deployment registers through `run_with` reach the state the +/// application hooks build, which takes no builders of its own. +/// +/// Registration is set once for the process, so the probe stays on offer to +/// the tests that run after this one. An offered builder runs only when the +/// settings select its module, which none of them do. +#[test] +fn registered_integrations_are_composed_into_the_state_the_hooks_build() { + register_integrations(vec![seam_probe::builder()]); + + let state = build_state_from_settings(settings_with(&format!("{HMAC_BLOCK}{PROBE_BLOCK}"))) + .expect("should build state with the registered builder composed in"); + + assert!( + state.registry.integration_runs(seam_probe::SEAM_PROBE_ID), + "the registered module should run when the settings select it" + ); +} diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index 0151cbb165..d61d090ca8 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -2,7 +2,6 @@ use std::time::Duration; use error_stack::{Report, ResultExt as _}; use fastly::backend::Backend; -use url::Url; use trusted_server_core::error::TrustedServerError; use trusted_server_core::platform::{BackendNamingPolicy, PlatformBackendSpec, PredictedBackend}; @@ -273,93 +272,6 @@ impl<'a> BackendConfig<'a> { } } } - - /// Parse an origin URL into its (scheme, host, port) components. - /// - /// Centralises URL parsing so that [`from_url`](Self::from_url) and - /// [`from_url_with_first_byte_timeout`](Self::from_url_with_first_byte_timeout) - /// share one code-path. - fn parse_origin( - origin_url: &str, - ) -> Result<(String, String, Option), Report> { - let parsed_url = Url::parse(origin_url).change_context(TrustedServerError::Proxy { - message: format!("Invalid origin_url: {origin_url}"), - })?; - - let scheme = parsed_url.scheme().to_owned(); - let host = parsed_url - .host_str() - .ok_or_else(|| { - Report::new(TrustedServerError::Proxy { - message: "Missing host in origin_url".to_owned(), - }) - })? - .to_owned(); - let port = parsed_url.port(); - - Ok((scheme, host, port)) - } - - /// Parse an origin URL and ensure a dynamic backend exists for it. - /// - /// This is a convenience constructor that parses the URL, extracts scheme, - /// host, and port, then calls [`ensure`](Self::ensure) with the default - /// 15 s first-byte timeout. - /// - /// # Errors - /// - /// Returns an error if the URL cannot be parsed or lacks a host, or if - /// backend creation fails. - pub fn from_url( - origin_url: &str, - certificate_check: bool, - ) -> Result> { - Self::from_url_with_first_byte_timeout( - origin_url, - certificate_check, - DEFAULT_FIRST_BYTE_TIMEOUT, - ) - } - - /// Parse an origin URL and ensure a dynamic backend with a custom - /// first-byte timeout. - /// - /// For latency-sensitive paths (e.g. auction bid requests) callers should - /// pass the remaining auction budget so that individual requests don't hang - /// longer than the overall deadline allows. - /// - /// # Errors - /// - /// Returns an error if the URL cannot be parsed or lacks a host, or if - /// backend creation fails. - pub fn from_url_with_first_byte_timeout( - origin_url: &str, - certificate_check: bool, - first_byte_timeout: Duration, - ) -> Result> { - Self::from_url_with_first_byte_timeout_and_host_header_override( - origin_url, - certificate_check, - first_byte_timeout, - None, - ) - } - - fn from_url_with_first_byte_timeout_and_host_header_override( - origin_url: &str, - certificate_check: bool, - first_byte_timeout: Duration, - host_header_override: Option<&str>, - ) -> Result> { - let (scheme, host, port) = Self::parse_origin(origin_url)?; - - BackendConfig::new(&scheme, &host) - .port(port) - .certificate_check(certificate_check) - .first_byte_timeout(first_byte_timeout) - .host_header_override(host_header_override) - .ensure() - } } #[cfg(test)] @@ -426,14 +338,10 @@ mod tests { #[test] fn url_derived_ipv6_host_uses_bare_tls_identity_without_sni() { - let (scheme, url_host, port) = - BackendConfig::parse_origin("https://[2001:db8::7]:8443/openrtb") - .expect("should parse IPv6 provider URL"); - assert_eq!(scheme, "https"); - assert_eq!(url_host, "[2001:db8::7]"); - assert_eq!(port, Some(8443)); - - let normalized = normalize_backend_host(&url_host); + // The bracketed form `url::Url::host_str` gives for an IPv6 literal. + let (scheme, url_host, port) = ("https", "[2001:db8::7]", Some(8443)); + + let normalized = normalize_backend_host(url_host); assert_eq!(normalized.identity, "2001:db8::7"); assert_eq!(normalized.authority, "[2001:db8::7]"); assert!( @@ -441,11 +349,11 @@ mod tests { "IP literals must omit TLS SNI while retaining a bare certificate identity" ); - let from_url_name = BackendConfig::new(&scheme, &url_host) + let from_url_name = BackendConfig::new(scheme, url_host) .port(port) .predict_name() .expect("should predict URL-derived IPv6 backend name"); - let from_bare_name = BackendConfig::new(&scheme, "2001:db8::7") + let from_bare_name = BackendConfig::new(scheme, "2001:db8::7") .port(port) .predict_name() .expect("should predict bare IPv6 backend name"); diff --git a/crates/trusted-server-adapter-fastly/src/config_selection.rs b/crates/trusted-server-adapter-fastly/src/config_selection.rs new file mode 100644 index 0000000000..b3dfa2050e --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/config_selection.rs @@ -0,0 +1,299 @@ +//! Which app-config blob serves a request. +//! +//! A service reads one blob, at the key its `__KEY` selector names or at the +//! logical store ID. A selector that carries [`HOST_PLACEHOLDER`] names a blob +//! for each host instead. The placeholder is replaced by the host a request +//! names, so one service serves several publishers, each from a blob of its +//! own. +//! +//! A host that has no blob is refused. It is never answered from another key. + +use core::fmt::Display; + +/// The part of a `__KEY` selector that a request's host replaces. +pub(crate) const HOST_PLACEHOLDER: &str = "{host}"; + +/// The longest name DNS carries. +const MAX_HOST_LEN: usize = 253; + +/// Why a request is not served. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Refusal { + /// The request names no host, or a host the store holds no blob for. + NoConfigForHost, + /// Whether the store holds the host's blob could not be read. + StoreUnavailable(String), +} + +/// The host as a blob is keyed by it, or `None` when `host` is not a host +/// name. +/// +/// Lowercased, with a port and one trailing dot removed. Only letters, digits +/// and hyphens pass, in labels joined by single dots, and no label starts or +/// ends with a hyphen. +pub(crate) fn normalized_host(host: &str) -> Option { + let without_port = match host.rsplit_once(':') { + Some((name, port)) if !port.is_empty() && port.bytes().all(|b| b.is_ascii_digit()) => name, + _ => host, + }; + let name = without_port.strip_suffix('.').unwrap_or(without_port); + let is_host_name = !name.is_empty() + && name.len() <= MAX_HOST_LEN + && name.split('.').all(|label| { + !label.is_empty() + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + }); + is_host_name.then(|| name.to_ascii_lowercase()) +} + +/// Resolves the key of the blob that serves a request. +/// +/// `selector` is the service's `__KEY` selector, or the logical store ID +/// where none is set. A selector without the placeholder is the key itself +/// and nothing is looked up, which is every service that serves one +/// publisher. Otherwise the key is the selector with the request's host in +/// the placeholder's place, and `exists` is asked whether the store holds it. +/// +/// # Errors +/// +/// Returns [`Refusal::NoConfigForHost`] when the request names no host name +/// or the store holds no blob at the host's key. Returns +/// [`Refusal::StoreUnavailable`] when `exists` fails, because a request is +/// not served while it is unknown which blob is its own. +pub(crate) fn resolve_key( + selector: &str, + host: Option<&str>, + exists: impl FnOnce(&str) -> Result, +) -> Result { + if !selector.contains(HOST_PLACEHOLDER) { + return Ok(selector.to_owned()); + } + let key = host + .and_then(normalized_host) + .map(|host| selector.replace(HOST_PLACEHOLDER, &host)) + .ok_or(Refusal::NoConfigForHost)?; + match exists(&key) { + Ok(true) => Ok(key), + Ok(false) => Err(Refusal::NoConfigForHost), + Err(error) => Err(Refusal::StoreUnavailable(error.to_string())), + } +} + +/// Whether the config store linked as `store_name` holds an entry at `key`. +/// +/// Reads the entry at the key alone and not the chunks a large blob is split +/// into, because the question is only whether the host has a blob. +/// +/// # Errors +/// +/// Returns a message naming the store when it cannot be opened or the lookup +/// fails. Uses [`fastly::ConfigStore::try_get`], because `get` panics on a +/// lookup error. +pub(crate) fn blob_exists(store_name: &str, key: &str) -> Result { + let store = fastly::ConfigStore::try_open(store_name) + .map_err(|error| format!("failed to open config store `{store_name}`: {error}"))?; + store + .try_get(key) + .map(|value| value.is_some()) + .map_err(|error| { + format!("failed to look up `{key}` in config store `{store_name}`: {error}") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Stands in for a store lookup that must not happen. + fn no_lookup(key: &str) -> Result { + panic!("the store must not be asked about `{key}`") + } + + /// A store holding exactly `keys`. + fn holding<'a>(keys: &'a [&'a str]) -> impl FnOnce(&str) -> Result + 'a { + move |key| Ok(keys.contains(&key)) + } + + #[test] + fn a_selector_without_the_placeholder_is_the_key_and_nothing_is_looked_up() { + for selector in ["trusted_server_config", "trusted_server_config_staging"] { + assert_eq!( + resolve_key(selector, Some("publisher.example"), no_lookup), + Ok(selector.to_owned()), + "a service that serves one publisher should read its one key" + ); + } + assert_eq!( + resolve_key("trusted_server_config", None, no_lookup), + Ok("trusted_server_config".to_owned()), + "one key should serve a request that names no host" + ); + } + + #[test] + fn the_host_takes_the_placeholder_s_place() { + for (selector, key) in [ + ("{host}", "publisher.example"), + ("{host}_staging", "publisher.example_staging"), + ("sites.{host}", "sites.publisher.example"), + ] { + assert_eq!( + resolve_key(selector, Some("publisher.example"), holding(&[key])), + Ok(key.to_owned()), + "should key the blob by the host under `{selector}`" + ); + } + } + + #[test] + fn each_host_reads_a_key_of_its_own() { + let stored = ["a.example", "b.example"]; + + assert_eq!( + resolve_key("{host}", Some("a.example"), holding(&stored)), + Ok("a.example".to_owned()), + "should read the first publisher's blob for its host" + ); + assert_eq!( + resolve_key("{host}", Some("b.example"), holding(&stored)), + Ok("b.example".to_owned()), + "should read the second publisher's blob for its host" + ); + } + + #[test] + fn a_host_is_keyed_without_case_port_or_trailing_dot() { + for host in [ + "publisher.example", + "Publisher.EXAMPLE", + "publisher.example:443", + "publisher.example.", + "PUBLISHER.example.:8443", + ] { + assert_eq!( + normalized_host(host).as_deref(), + Some("publisher.example"), + "`{host}` should name the same site" + ); + } + assert_eq!( + normalized_host("127.0.0.1:7676").as_deref(), + Some("127.0.0.1"), + "should key a local address without its port" + ); + assert_eq!( + normalized_host("xn--bcher-kva.example").as_deref(), + Some("xn--bcher-kva.example"), + "should keep an internationalized name as it is sent" + ); + } + + #[test] + fn two_names_of_one_publisher_are_two_keys() { + assert_ne!( + normalized_host("www.publisher.example"), + normalized_host("publisher.example"), + "a publisher decides what each of its names serves" + ); + } + + #[test] + fn what_is_not_a_host_name_is_refused_without_a_lookup() { + let too_long = "a".repeat(MAX_HOST_LEN + 1); + for host in [ + "", + ".", + "publisher..example", + ".publisher.example", + "publisher.example..", + "publisher.example:", + "publisher.example:80:80", + " publisher.example", + "publisher.example/path", + "[::1]", + "[::1]:7676", + "publisher_example.com", + "trusted_server_config", + "publisher.example.__edgezero_chunks.abc.0", + "{host}", + "b\u{fc}cher.example", + too_long.as_str(), + ] { + assert_eq!( + resolve_key("{host}", Some(host), no_lookup), + Err(Refusal::NoConfigForHost), + "`{host}` is not a host name and should be refused" + ); + } + assert_eq!( + resolve_key("{host}", None, no_lookup), + Err(Refusal::NoConfigForHost), + "a request that names no host should be refused" + ); + } + + #[test] + fn a_label_that_starts_or_ends_with_a_hyphen_is_refused() { + for host in [ + "-a.example", + "a-.example", + "publisher.-example", + "publisher.example-", + "-", + ] { + assert_eq!( + resolve_key("{host}", Some(host), no_lookup), + Err(Refusal::NoConfigForHost), + "`{host}` is not a host name and should be refused" + ); + } + assert_eq!( + normalized_host("pub-lisher.example").as_deref(), + Some("pub-lisher.example"), + "a hyphen inside a label should pass" + ); + } + + #[test] + fn the_longest_host_name_is_keyed() { + let label = "a".repeat(63); + let host = format!("{label}.{label}.{label}.{}", "a".repeat(61)); + assert_eq!(host.len(), MAX_HOST_LEN, "should build the longest name"); + + assert_eq!( + normalized_host(&host), + Some(host.clone()), + "should key a name of the greatest length DNS carries" + ); + } + + #[test] + fn a_host_with_no_blob_is_refused_and_not_served_from_another_key() { + // The store holds a blob at the logical store ID and one for another + // publisher. Neither may answer for a host that has none. + let stored = ["trusted_server_config", "a.example"]; + + assert_eq!( + resolve_key("{host}", Some("unknown.example"), holding(&stored)), + Err(Refusal::NoConfigForHost), + "a host with no blob should be refused" + ); + } + + #[test] + fn a_failed_lookup_refuses_and_does_not_serve() { + let refused = resolve_key("{host}", Some("publisher.example"), |_key| { + Err::("lookup failed") + }); + + assert_eq!( + refused, + Err(Refusal::StoreUnavailable("lookup failed".to_owned())), + "should not serve while the host's blob is unknown" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/lib.rs b/crates/trusted-server-adapter-fastly/src/lib.rs new file mode 100644 index 0000000000..ef8c4fd269 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/lib.rs @@ -0,0 +1,1723 @@ +//! The Fastly Compute adapter for Trusted Server. +//! +//! A library with a thin binary over it, so a deployment that ships a vendor +//! crate builds on this adapter rather than editing it. [`run`] serves the way +//! the binary does, and [`run_with`] serves with the integration builders the +//! deployment offers composed with the built-in ones. + +use std::sync::Arc; + +use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; +use edgezero_adapter_fastly::request::into_core_request; +use edgezero_adapter_fastly::runtime_env_config; +use edgezero_core::app::Hooks as _; +use edgezero_core::body::Body as EdgeBody; +use edgezero_core::config_store::ConfigStoreHandle; +use edgezero_core::error::EdgeError; +use edgezero_core::http::{ + HeaderMap, HeaderValue, Request as HttpRequest, Response as HttpResponse, header, +}; +use edgezero_core::response::IntoResponse; +use error_stack::Report; +use fastly::http::Method as FastlyMethod; +use fastly::{Request as FastlyRequest, Response as FastlyResponse}; + +use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; +use trusted_server_core::ec::device::{DeviceModule, DeviceSignals, build_device_module}; +use trusted_server_core::ec::finalize::ec_finalize_response; +use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::pull_sync::{ + PullSyncContext, build_pull_sync_context, dispatch_pull_sync, +}; +use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::evidence::{BorrowedRequestInfo, HostSignals}; +use trusted_server_core::integrations::RequestFilterEffects; +use trusted_server_core::module_context::{ModuleContext, ModuleRequest}; +use trusted_server_core::platform::build_geo_module; +use trusted_server_core::platform::{ClientInfo, RuntimeServices, UnavailableKvStore}; +use trusted_server_core::prefix::Prefixes; +use trusted_server_core::proxy::{AssetProxyCachePolicy, stream_asset_body}; +use trusted_server_core::response_privacy::TerminalPrivateResponse; +use trusted_server_core::settings::Settings; +use trusted_server_device_fastly::{FastlyDeviceModule, FastlyHostSignals}; + +mod app; +mod backend; +mod compat; +mod config_selection; +mod ec_kv; +mod esi_assembly; +mod logging; +mod middleware; +mod platform; +mod rate_limiter; +mod sandbox; +mod template_cache; +mod tinybird; +mod value_cache; + +use crate::app::{ + AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, build_finalize_services, + load_settings_from_config_store, +}; +use crate::ec_kv::FastlyEcKvStore; +use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, geo_allowed_for_response}; +use crate::platform::{FastlyPlatformGeo, client_info_from_request}; +use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; +use crate::sandbox::{RetainedApp, Sandbox, SandboxCounters, ServeMode, StartupDiagnostics}; +/// Re-exported so a crate registering an integration names the type it hands +/// over without depending on core directly. +pub use trusted_server_core::integrations::IntegrationBuilder; +// Only the reuse path builds a serving loop, so the retirement snapshots have +// no consumer in the default build. +#[cfg(feature = "reusable-sandbox")] +use crate::sandbox::RetirementCounters; + +/// Opens the Fastly Config Store used by the `EdgeZero` dispatcher. +/// +/// # Errors +/// +/// Returns [`fastly::Error`] if the config store cannot be opened. +fn open_trusted_server_config_store(store_name: &str) -> Result { + let store = EdgeZeroFastlyConfigStore::try_open(store_name).map_err(|e| { + fastly::Error::msg(format!("failed to open config store `{store_name}`: {e}")) + })?; + Ok(ConfigStoreHandle::new(Arc::new(store))) +} + +/// Chooses the app-config blob that serves this request. +/// +/// A `__KEY` selector that carries the host placeholder names a blob for each +/// host, so the key is the selector with this request's host in it. Any +/// other selector is the key as it stands. +/// +/// # Errors +/// +/// Returns why the request is not served, which [`refusal_response`] turns +/// into the answer. +fn stores_for_request( + stores: RuntimeStoreConfig, + req: &FastlyRequest, +) -> Result { + let host = req.get_header_str(header::HOST.as_str()); + let resolved = config_selection::resolve_key(&stores.config_key, host, |key| { + config_selection::blob_exists(stores.config_store_name.as_ref(), key) + }); + match resolved { + Ok(config_key) => Ok(RuntimeStoreConfig { + config_key, + ..stores + }), + Err(refusal) => { + let named = host.unwrap_or_default(); + match &refusal { + config_selection::Refusal::NoConfigForHost => log::warn!( + "no app-config blob for host {named:?} under key selector `{}`", + stores.config_key + ), + config_selection::Refusal::StoreUnavailable(reason) => { + log::error!( + "cannot tell whether host {named:?} has an app-config blob: {reason}" + ); + } + } + Err(refusal) + } + } +} + +/// The answer to a request no app-config blob serves. +/// +/// A host with no blob is answered `421 Misdirected Request`. The answer is +/// not to be stored, so a host is served as soon as its blob is pushed. A +/// store that could not say whether the host has a blob is answered `500`. +fn refusal_response(refusal: &config_selection::Refusal) -> FastlyResponse { + match refusal { + config_selection::Refusal::NoConfigForHost => { + FastlyResponse::from_status(fastly::http::StatusCode::MISDIRECTED_REQUEST) + .with_header("cache-control", "private, no-store") + .with_body_text_plain("Misdirected Request") + } + config_selection::Refusal::StoreUnavailable(_) => { + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + } + } +} + +fn health_response(req: &FastlyRequest) -> Option { + if req.get_method() == FastlyMethod::GET && req.get_path() == "/health" { + return Some(FastlyResponse::from_status(200).with_body_text_plain("ok")); + } + + None +} + +/// Runs the Fastly Compute program with the integrations a deployment offers. +/// +/// The builders are composed with the ones this adapter ships rather than +/// replacing them, and they are recorded before any request is served. +/// Offering an integration does not switch it on, because a builder runs only +/// when the settings select its module. +/// +/// ```no_run +/// fn main() { +/// trusted_server_adapter_fastly::run_with(vec![a_vendor_crate::builder()]); +/// } +/// # mod a_vendor_crate { +/// # pub fn builder() -> trusted_server_adapter_fastly::IntegrationBuilder { +/// # unimplemented!() +/// # } +/// # } +/// ``` +pub fn run_with(integrations: Vec) { + app::register_integrations(integrations); + run(); +} + +/// Runs the Fastly Compute program, which the binary's `main()` calls. +/// +/// Uses an undecorated entry with `FastlyRequest::from_client()` instead of +/// `#[fastly::main]` so the `EdgeZero` streaming publisher path can call +/// [`fastly::Response::stream_to_client`] explicitly. It owns the sandbox +/// lifecycle and delegates each request to [`handle_request`]. +/// +/// Without the `reusable-sandbox` feature this takes one request and returns, +/// which is the original behaviour. With the feature, the bounds still have to +/// come from the runtime environment before the SDK serving loop is entered; +/// an unconfigured or partially configured sandbox stays single-request. +pub fn run() { + let (mode, diagnostics) = serve_mode(); + + // Held outside the sandbox: `serve_custom` owns the `Sandbox` and exposes + // no slot for application state that is not the retained payload. + let mut startup = StartupDiagnostics::default(); + for message in diagnostics { + startup.push(message); + } + + match mode { + ServeMode::Single => { + // `run_custom` completes the callback's SDK result exactly once. + // The callback sends its own response and returns `()`, so there + // is no error for the SDK to turn into a second response. + let Ok(()) = edgezero_adapter_fastly::lifecycle::run_custom( + FastlyRequest::from_client(), + |request, sandbox: &mut Sandbox| handle_request(request, sandbox, &mut startup), + ); + } + ServeMode::Reuse(limits) => serve_loop(limits, startup), + } +} + +/// Serves requests from one sandbox under explicit bounds. +/// +/// Only compiled with the `reusable-sandbox` feature; [`serve_mode`] can never +/// return [`ServeMode::Reuse`] without it. +#[cfg(feature = "reusable-sandbox")] +fn serve_loop(limits: crate::sandbox::SandboxLimits, mut startup: StartupDiagnostics) { + // `serve_custom` owns the `Sandbox` and drops it when serving ends, so the + // retirement line reports snapshots taken while it was still borrowed. + // `RetirementCounters` reads the attempt count on the way out, so a build + // performed by the final callback is included. + let mut counters = RetirementCounters::default(); + + let summary = edgezero_adapter_fastly::lifecycle::serve_custom( + fastly::http::serve::Serve::new() + .with_max_requests(limits.max_requests) + .with_max_memory(limits.max_memory_mib) + .with_max_lifetime(limits.max_lifetime) + .with_timeout(limits.timeout), + |request, sandbox: &mut Sandbox| { + counters.observe(sandbox, |sandbox| { + handle_request(request, sandbox, &mut startup); + }); + }, + ); + + log::info!( + "sandbox retiring after {} attempted callback(s), {} observed, {} build attempt(s)", + summary.requests(), + counters.requests(), + counters.attempts() + ); +} + +#[cfg(not(feature = "reusable-sandbox"))] +fn serve_loop(_limits: crate::sandbox::SandboxLimits, _startup: StartupDiagnostics) { + unreachable!("serve_mode never selects reuse without the reusable-sandbox feature") +} + +/// Handles one request end to end, sending its own response. +/// +/// Returns `()` rather than a response so the streaming publisher path can +/// call [`fastly::Response::stream_to_client`] itself. The SDK's +/// `HandlerResult` impl for `()` treats that as already sent. +/// +/// Every non-panicking path through this function must send exactly once. In a +/// reused sandbox the SDK refuses to wait for the next request until the +/// current one is complete, so a missed send stalls the loop rather than +/// merely dropping one response. +fn handle_request(req: FastlyRequest, sandbox: &mut Sandbox, startup: &mut StartupDiagnostics) { + // The framework counts the callback before invoking it, including early + // returns, so this is already this request's 1-based ordinal. + let ordinal = sandbox.requests(); + + // Health probe bypasses logging, settings, and app construction as a cheap liveness signal. + if let Some(response) = health_response(&req) { + response.send_to_client(); + return; + } + + // Marked complete only once installation succeeds, so a failed install is + // retried on a later callback. `setup_once` rolls nothing back, so that is + // only correct because `init_logger` is harmless to repeat — see its docs. + match sandbox.setup_once(crate::logging::init_logger) { + Ok(()) => startup.flush(), + Err(error) => { + // Logger installation failed, so its own error cannot rely on log. + // Keep startup diagnostics pending until installation succeeds. + #[allow(clippy::print_stderr, reason = "logger installation failed")] + { + eprintln!("logger installation failed, retrying next callback: {error}"); + } + } + } + + // Correlation is request-local and never retained. `FASTLY_TRACE_ID` names + // the sandbox, not the request, so it is not used here. The id rides on the + // response only when metrics are enabled; the client request is left + // untouched so nothing new reaches origin in the default configuration. + let request_id = req + .get_client_request_id() + .map(str::to_owned) + .unwrap_or_else(|| format!("{}-{ordinal}", instance_id())); + + edgezero_main(req, sandbox, ordinal, &request_id); +} + +/// Builds the counters snapshot response. +/// +/// A snapshot only. It reports the sandbox that served *this* probe, which is +/// not necessarily the sandbox that served any preceding workload request, so +/// reuse is established from the counters attached to workload responses +/// rather than from polling this. +/// +/// `ordinal` is this probe's own 1-based position in the sandbox, which is +/// also the number of callbacks the sandbox has served including this one. +/// The lifetime count is therefore not reported separately. +#[cfg(feature = "reusable-sandbox")] +fn sandbox_metrics_response(sandbox: &Sandbox, ordinal: u64) -> FastlyResponse { + let body = serde_json::json!({ + "instance": instance_id(), + "ordinal": ordinal, + "builds": crate::sandbox::build_attempts(sandbox), + }); + + FastlyResponse::from_status(fastly::http::StatusCode::OK) + .with_header("cache-control", "private, no-store") + .with_body_json(&body) + .unwrap_or_else(|e| { + log::error!("failed to serialize sandbox metrics: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + }) +} + +/// Attaches sandbox counters only to private, no-store workload responses. +/// +/// Called before headers are committed, which on the streaming path means +/// before `stream_to_client`. The counters therefore describe the request up +/// to commitment and cannot report its eventual outcome; a failure after +/// commitment is recorded in logs instead and reconciled during analysis. +fn attach_sandbox_counters(response: &mut HttpResponse, counters: &SandboxCounters) { + // Per-request measurements must never be replayed from a cache. Requiring + // no-store also excludes browser-cacheable and field-qualified privacy. + if !cache_control_headers_have_directive(response.headers(), "private") + || !cache_control_headers_have_directive(response.headers(), "no-store") + { + return; + } + + let headers = response.headers_mut(); + for (name, value) in [ + (sandbox::HEADER_SANDBOX_INSTANCE, instance_id()), + ( + sandbox::HEADER_SANDBOX_ORDINAL, + counters.ordinal.to_string(), + ), + (sandbox::HEADER_SANDBOX_BUILDS, counters.builds.to_string()), + ( + sandbox::HEADER_SANDBOX_REQUEST_ID, + counters.request_id.clone(), + ), + (sandbox::HEADER_SANDBOX_VCPU_MS, vcpu_ms()), + (sandbox::HEADER_SANDBOX_HEAP_MIB, heap_mib()), + ] { + match edgezero_core::http::HeaderValue::from_str(&value) { + Ok(value) => { + headers.insert(name, value); + } + Err(e) => log::warn!("sandbox counter `{name}` is not a valid header value: {e}"), + } + } +} + +/// Resolves how this sandbox will serve requests. +/// +/// Without the `reusable-sandbox` feature this is unconditionally +/// [`ServeMode::Single`] and reads nothing, so the default build does no +/// startup work the original entry point did not do. +/// Returns the mode alongside diagnostics that must wait for the logger. +/// +/// This runs before any logger exists, so the reasons reuse was declined are +/// carried out rather than logged here, where they would be discarded. +#[cfg(feature = "reusable-sandbox")] +fn serve_mode() -> (ServeMode, Vec) { + let (raw, diagnostics) = crate::sandbox::read_raw_limits(); + (crate::sandbox::resolve_mode(raw), diagnostics) +} + +#[cfg(not(feature = "reusable-sandbox"))] +fn serve_mode() -> (ServeMode, Vec) { + (ServeMode::Single, Vec::new()) +} + +/// Guest-instance identifier used to attribute requests to a sandbox. +/// +/// `FASTLY_TRACE_ID` describes the sandbox, which is exactly what is wanted +/// here and exactly why it must not be used as a request id. An absent value +/// is reported rather than synthesized, so a measurement run cannot silently +/// claim reuse it never observed. +fn instance_id() -> String { + // The SDK documents this as the per-sandbox identifier; on wasm32-wasip1 + // it resolves to `FASTLY_TRACE_ID`, which is why that value must never be + // used as a request id. + let id = fastly::compute_runtime::sandbox_id(); + if id.is_empty() { + return sandbox::INSTANCE_ID_UNAVAILABLE.to_owned(); + } + id.to_owned() +} + +/// Cumulative guest vCPU milliseconds, or a marker when unsupported. +fn vcpu_ms() -> String { + fastly::compute_runtime::elapsed_vcpu_ms().map_or_else( + |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), + |v| v.to_string(), + ) +} + +/// Guest heap snapshot in MiB, or a marker when unsupported. +fn heap_mib() -> String { + fastly::compute_runtime::heap_memory_snapshot_mib().map_or_else( + |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), + |v| v.to_string(), + ) +} + +/// Handles a request through the `EdgeZero` router path. +fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, request_id: &str) { + let runtime_env = runtime_env_config(TrustedServerApp::stores()); + // Settled before anything reads settings, so every path below reads the + // blob of the publisher this request is for. + let runtime_stores = match stores_for_request(RuntimeStoreConfig::from_env(&runtime_env), &req) + { + Ok(stores) => stores, + Err(refusal) => { + refusal_response(&refusal).send_to_client(); + return; + } + }; + + // Short-circuit the sandbox counters probe before app construction. It must + // not build the application: polling it would otherwise increment the very + // build counter it reports. + #[cfg(feature = "reusable-sandbox")] + if req.get_method() == FastlyMethod::GET && req.get_path() == sandbox::SANDBOX_METRICS_PATH { + match load_settings_from_config_store(&runtime_stores) { + Ok(settings) if sandbox::metrics_enabled(&settings) => { + sandbox_metrics_response(sandbox, ordinal).send_to_client(); + } + Ok(_) => { + FastlyResponse::from_status(fastly::http::StatusCode::NOT_FOUND).send_to_client(); + } + Err(e) => { + log::warn!("sandbox metrics endpoint: failed to load settings: {e:?}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + } + } + return; + } + + // Short-circuit the JA4 debug probe before app construction. Must run here + // because TLS/JA4 accessors are only available on FastlyRequest before + // conversion to edgezero types. + if req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { + match load_settings_from_config_store(&runtime_stores) { + Ok(settings) if settings.debug.ja4_endpoint_enabled => { + build_ja4_debug_response(&req).send_to_client(); + } + Ok(_) => { + FastlyResponse::from_status(fastly::http::StatusCode::NOT_FOUND).send_to_client(); + } + Err(e) => { + log::warn!("EdgeZero JA4 endpoint: failed to load settings: {e:?}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + } + } + return; + } + + let config_store = + match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { + Ok(cs) => cs, + Err(e) => { + log::error!("failed to open config store: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + } + }; + + // Build lazily, once per sandbox for each app-config key it is asked for. + // Reached only past the health, JA4, and counters short-circuits, so none + // of those pays for construction. + crate::sandbox::hold_applications(sandbox); + let Some(apps) = sandbox.state() else { + log::error!("no application holder available after initialization"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + }; + let built = apps.get_or_build(&runtime_stores.config_key, || { + let (app, state) = TrustedServerApp::build_app_with_state(&runtime_stores); + match state { + Some(state) => Ok(RetainedApp { app, state }), + None => Err(app), + } + }); + + let (app, app_state): (&edgezero_core::app::App, Option>) = match &built { + Ok(retained) => (&retained.app, Some(Arc::clone(&retained.state))), + Err(error_app) => (error_app, None), + }; + + let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); + let counters = + SandboxCounters::capture(sandbox, ordinal, request_id, settings_snapshot.as_deref()); + let trusted_client_ip = settings_snapshot + .as_deref() + .and_then(|settings| settings.trusted_client_ip.as_ref()); + + // Resolve the trusted client IP, then strip client-spoofable forwarded + // headers before dispatch. One call keeps resolution ahead of the + // sanitization that removes the headers it reads. + let resolved_client_ip = compat::resolve_and_sanitize_client_ip(&mut req, trusted_client_ip); + + // Re-inject a trusted TLS scheme signal after sanitization has stripped any + // client-sent fastly-ssl header. Setting it from Fastly's native TLS + // metadata here is authoritative. detect_request_scheme in http_util checks + // this header so scheme-sensitive logic produces https URLs on HTTPS traffic. + if req.get_tls_protocol().ok().flatten().is_some() + || req.get_tls_cipher_openssl_name().ok().flatten().is_some() + { + req.set_header("fastly-ssl", "1"); + } + + // Strip any client-supplied x-ts-tls-* headers before injecting the trusted + // values from the Fastly SDK. Must run after sanitize_fastly_forwarded_headers. + req.remove_header("x-ts-tls-protocol"); + req.remove_header("x-ts-tls-cipher"); + if let Some(proto) = req.get_tls_protocol().ok().flatten().map(str::to_owned) { + req.set_header("x-ts-tls-protocol", proto); + } + if let Some(cipher) = req + .get_tls_cipher_openssl_name() + .ok() + .flatten() + .map(str::to_owned) + { + req.set_header("x-ts-tls-cipher", cipher); + } + + // Capture metadata from the original FastlyRequest before conversion. These + // accessors only return real values on the client request, so store them in + // request extensions for build_per_request_services and EC bot classification. + let client_info = client_info_from_request(&req, resolved_client_ip); + let client_ip = client_info.client_ip; + + // Strip and re-inject the TLS JA4 and HTTP/2 signals from the + // authoritative Fastly SDK values, under the same trust model, so the + // EdgeZero app path can build the host-signal service from these internal + // headers (the SDK accessors return real values only on the live client + // request, not on a request rebuilt from EdgeZero HTTP types). + req.remove_header("x-ts-tls-ja4"); + req.remove_header("x-ts-h2-fingerprint"); + // Take ownership before setting: unlike the static TLS protocol/cipher + // names, these accessors borrow the request, which would otherwise conflict + // with the mutable `set_header`. + if let Some(ja4) = req.get_tls_ja4().map(str::to_string) { + req.set_header("x-ts-tls-ja4", ja4); + } + if let Some(h2) = req.get_client_h2_fingerprint().map(str::to_string) { + req.set_header("x-ts-h2-fingerprint", h2); + } + + // Derive device signals from the original FastlyRequest before conversion. + // Fastly's `get_tls_ja4()` and `get_client_h2_fingerprint()` accessors only + // return real values on the client request; a synthetic request rebuilt from + // EdgeZero HTTP types cannot expose them, which would strip the JA4/H2 class + // the EC bot gate needs and misclassify real browsers as bots. Stored in the + // request extensions so `build_ec_request_state` reads the authoritative + // signals instead of re-deriving from the reconstructed request. + // Reuse the settings snapshot already loaded for the app state rather than + // fetching and validating the config-store blob a second time per request. + let device_signals = match settings_snapshot.as_deref() { + Some(settings) => { + // The entry point is synchronous host code, so it drives the async + // module seam at the same boundary it already drives the router. + let services = entry_point_finalize_services(settings, &app_state, entry_request(&req)); + let registered = app_state + .as_deref() + .and_then(|state| state.registry.device_module()); + futures::executor::block_on(derive_device_signals( + settings, registered, &req, &services, + )) + } + None => { + log::warn!( + "EdgeZero device signals: settings unavailable, using UA-only classification" + ); + DeviceSignals::derive_ua_only(req.get_header_str("user-agent").unwrap_or("")) + } + }; + + // Dispatch directly through the EdgeZero router without an intermediate + // fastly::Response conversion. That preserves duplicate header values such + // as multiple Set-Cookie headers. + // Kept for the finishing steps' module calls, which run after the + // request has been handed to the router. + let kept = EntryRequest::of(&req); + let mut response = match into_core_request(req) { + Ok(mut core_req) => { + core_req.extensions_mut().insert(config_store); + core_req.extensions_mut().insert(device_signals); + core_req.extensions_mut().insert(client_info); + match futures::executor::block_on(app.router().oneshot(core_req)) { + Ok(response) => response, + Err(error) => edge_error_response(error), + } + } + Err(e) => { + log::error!("EdgeZero request conversion failed: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + } + }; + + // Pop response extensions before the Fastly conversion, which drops them. + let ec_state = response.extensions_mut().remove::(); + let asset_cache_policy = response.extensions_mut().remove::(); + let request_filter_effects = response.extensions_mut().remove::(); + + if !take_finalize_sentinel(&mut response) { + if let Some(settings) = settings_snapshot.as_deref() { + futures::executor::block_on(apply_entry_point_finalize_headers( + settings, + &mut response, + client_ip, + entry_point_finalize_services(settings, &app_state, kept.module_request()), + )); + } else { + match load_settings_from_config_store(&runtime_stores) { + Ok(settings) => { + futures::executor::block_on(apply_entry_point_finalize_headers( + &settings, + &mut response, + client_ip, + entry_point_finalize_services(&settings, &app_state, kept.module_request()), + )); + } + Err(e) => { + log::warn!("entry-point finalize skipped: failed to reload settings: {e:?}"); + } + } + } + } + + if let Some(policy) = asset_cache_policy { + policy.apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + } + + if let Some(mut ec_state) = ec_state { + if let Some(settings) = settings_snapshot.as_deref() { + match futures::executor::block_on(apply_edgezero_ec_finalize( + settings, + &mut ec_state, + &mut response, + )) { + Ok(partner_registry) => { + send_edgezero_response( + response, + request_filter_effects.as_ref(), + counters.as_ref(), + ); + run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + return; + } + Err(e) => { + log::error!( + "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" + ); + } + } + } else { + match load_settings_from_config_store(&runtime_stores) { + Ok(settings) => { + match futures::executor::block_on(apply_edgezero_ec_finalize( + &settings, + &mut ec_state, + &mut response, + )) { + Ok(partner_registry) => { + send_edgezero_response( + response, + request_filter_effects.as_ref(), + counters.as_ref(), + ); + run_edgezero_pull_sync_after_send( + &settings, + &partner_registry, + &ec_state, + ); + return; + } + Err(e) => { + log::error!( + "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" + ); + } + } + } + Err(e) => { + log::warn!("EdgeZero EC finalize skipped: failed to reload settings: {e:?}"); + } + } + } + } + + send_edgezero_response(response, request_filter_effects.as_ref(), counters.as_ref()); +} + +fn edge_error_response(error: EdgeError) -> HttpResponse { + log::error!("EdgeZero router returned error: {error:?}"); + match error.into_response() { + Ok(response) => response, + Err(error) => { + log::error!("failed to convert EdgeZero error into response: {error:?}"); + edgezero_core::http::response_builder() + .status(edgezero_core::http::StatusCode::INTERNAL_SERVER_ERROR) + .body(EdgeBody::from("Internal Server Error")) + .expect("should build EdgeZero error response") + } + } +} + +fn take_finalize_sentinel(response: &mut HttpResponse) -> bool { + response + .headers_mut() + .remove(HEADER_X_TS_FINALIZED) + .is_some() +} + +/// The services a module called from the entry point is given, with the +/// state's key-value store and loading cache when the state was built. +/// +/// The entry-point paths run whether or not application state was built, so a +/// deployment whose state failed to build offers the unavailable store and a +/// cache that holds nothing rather than no services at all. A module that +/// needs the store then fails its own call instead of the entry point silently +/// skipping the module. +/// +/// The prefixes come from the selected prefix module where the state was +/// built. A deployment whose state failed to build has no registry to name +/// one, so core's own stand. +fn entry_point_finalize_services( + settings: &Settings, + app_state: &Option>, + request: ModuleRequest<'_>, +) -> RuntimeServices { + match app_state { + Some(state) => build_finalize_services( + settings, + Arc::clone(&state.default_kv_store), + Arc::clone(&state.loading_caches), + ) + .with_prefixes(state.registry.prefixes(settings, request)), + None => build_finalize_services(settings, Arc::new(UnavailableKvStore), Arc::default()) + .with_prefixes(Prefixes::trusted_server(settings)), + } +} + +async fn apply_entry_point_finalize_headers( + settings: &Settings, + response: &mut HttpResponse, + client_ip: Option, + services: RuntimeServices, +) { + // Route through the [geo] module selector, so a deployment that opts + // out of geolocation makes no host geo call on the entry-point finalize + // path either. + let geo = build_geo_module(settings, Arc::new(FastlyPlatformGeo)); + let geo_info = if geo_allowed_for_response(response) { + let services = services.with_client_info(ClientInfo { + client_ip, + ..ClientInfo::default() + }); + geo.lookup(client_ip, &services).await.unwrap_or_else(|e| { + log::warn!("entry-point geo lookup failed: {e}"); + None + }) + } else { + None + }; + apply_finalize_headers(settings, geo_info.as_ref(), response); +} + +async fn apply_edgezero_ec_finalize( + settings: &Settings, + ec_state: &mut EcFinalizeState, + response: &mut HttpResponse, +) -> Result> { + let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; + let finalize_kv_graph = if ec_state.use_finalize_kv { + maybe_identity_graph(settings) + } else { + None + }; + ec_finalize_response( + settings, + &mut ec_state.ec_context, + finalize_kv_graph.as_ref(), + &partner_registry, + ec_state.eids_cookie.as_deref(), + ec_state.sharedid_cookie.as_deref(), + response, + &ec_state.services, + ) + .await; + Ok(partner_registry) +} + +fn run_edgezero_pull_sync_after_send( + settings: &Settings, + partner_registry: &PartnerRegistry, + ec_state: &EcFinalizeState, +) { + if !ec_state.is_real_browser { + return; + } + + let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); + let Some((context, kv)) = + prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) + else { + return; + }; + + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + dispatch_pull_sync( + settings, + &kv, + partner_registry, + &limiter, + &context, + &ec_state.services, + ); +} + +fn prepare_pull_sync_after_send( + context: Option, + graph_factory: F, +) -> Option<(PullSyncContext, KvIdentityGraph)> +where + F: FnOnce() -> Result>, +{ + let context = context?; + let kv = match graph_factory() { + Ok(kv) => kv, + Err(err) => { + log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); + return None; + } + }; + Some((context, kv)) +} + +/// Sends a finalized `EdgeZero` response to the client. +/// +/// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's +/// client stream so large asset and publisher-origin responses do not +/// materialize in the Wasm heap. +fn send_edgezero_response( + mut response: HttpResponse, + request_filter_effects: Option<&RequestFilterEffects>, + counters: Option<&SandboxCounters>, +) { + apply_terminal_response_effects(&mut response, request_filter_effects); + + // Captured before the body is consumed so post-commitment failures can be + // matched back to the response that carried these counters. + let counter_context = counters.map_or_else(String::new, |counters| { + format!( + " [instance={} ordinal={} request={}]", + instance_id(), + counters.ordinal, + counters.request_id + ) + }); + + // Before headers commit, including before `stream_to_client` below. + if let Some(counters) = counters.as_ref() { + attach_sandbox_counters(&mut response, counters); + } + + let (parts, body) = response.into_parts(); + + match body { + EdgeBody::Stream(_) => { + let skeleton = compat::to_fastly_response_skeleton(HttpResponse::from_parts( + parts, + EdgeBody::empty(), + )); + let mut streaming_body = skeleton.stream_to_client(); + match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { + Ok(()) => { + if let Err(e) = streaming_body.finish() { + // Also post-commitment: same attribution as the + // streaming failure below. + log::error!( + "failed to finish EdgeZero streaming body{counter_context}: {e}" + ); + } + } + Err(e) => { + // After commitment: log and stop. Returning an error here + // would let the SDK attempt a second response. Counters + // already went out with the headers, so the failure is + // tagged with the same identity for reconciliation. + log::error!("EdgeZero streaming failed{counter_context}: {e:?}"); + drop(streaming_body); + } + } + } + once => { + compat::to_fastly_response(HttpResponse::from_parts(parts, once)).send_to_client(); + } + } +} + +/// Apply every late response mutation, then restore privacy invariants before headers commit. +fn apply_terminal_response_effects( + response: &mut HttpResponse, + request_filter_effects: Option<&RequestFilterEffects>, +) { + let must_remain_private = response + .extensions() + .get::() + .is_some(); + if let Some(effects) = request_filter_effects { + effects.apply_to_response(response); + } + if must_remain_private { + trusted_server_core::response_privacy::enforce_private_no_store(response); + } + + // Final cache guards: EC finalization and request-filter effects may have + // added a per-user Set-Cookie or a private/no-store directive after + // `apply_finalize_headers` and normalized asset policy reapplication ran. + crate::middleware::enforce_set_cookie_cache_privacy(response); + crate::middleware::enforce_uncacheable_cache_privacy(response); +} + +const FALLBACK_UNAVAILABLE: &str = "unavailable"; +const FALLBACK_NOT_SENT: &str = "not sent"; +const FALLBACK_NONE: &str = "none"; + +// TODO: remove after JA4 evaluation completes - see #645 +fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { + let ja4 = req.get_tls_ja4().unwrap_or(FALLBACK_UNAVAILABLE); + let h2 = req + .get_client_h2_fingerprint() + .unwrap_or(FALLBACK_UNAVAILABLE); + let cipher = req + .get_tls_cipher_openssl_name() + .ok() + .flatten() + .unwrap_or(FALLBACK_UNAVAILABLE); + let tls_version = req + .get_tls_protocol() + .ok() + .flatten() + .unwrap_or(FALLBACK_UNAVAILABLE); + let ua = req.get_header_str("user-agent").unwrap_or(FALLBACK_NONE); + let ch_mobile = req + .get_header_str("sec-ch-ua-mobile") + .unwrap_or(FALLBACK_NOT_SENT); + let ch_platform = req + .get_header_str("sec-ch-ua-platform") + .unwrap_or(FALLBACK_NOT_SENT); + + let body = format!( + "ja4: {ja4}\n\ + h2_fp: {h2}\n\ + cipher: {cipher}\n\ + tls_version: {tls_version}\n\ + user-agent: {ua}\n\ + ch-mobile: {ch_mobile}\n\ + ch-platform: {ch_platform}\n" + ); + + FastlyResponse::from_status(fastly::http::StatusCode::OK) + .with_header(fastly::http::header::CACHE_CONTROL, "no-store, private") + .with_header( + fastly::http::header::VARY, + "User-Agent, Sec-CH-UA-Mobile, Sec-CH-UA-Platform", + ) + .with_content_type(fastly::mime::TEXT_PLAIN_UTF_8) + .with_body(body) +} + +pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option { + settings + .ec + .ec_store + .as_ref() + .map(|store_name| KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) +} + +/// Constructs a `KvIdentityGraph` from settings, or returns an error if the +/// `ec_store` config is not set. +pub(crate) fn require_identity_graph( + settings: &Settings, +) -> Result> { + let store_name = settings.ec.ec_store.as_deref().ok_or_else(|| { + Report::new(TrustedServerError::KvStore { + store_name: "ec.ec_store".to_owned(), + message: "ec.ec_store is not configured".to_owned(), + }) + })?; + Ok(KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) +} + +/// Extracts a named cookie value from the request's `Cookie` header. +pub(crate) fn extract_cookie_value(req: &HttpRequest, name: &str) -> Option { + let cookie_header = req.headers().get("cookie").and_then(|v| v.to_str().ok())?; + for pair in cookie_header.split(';') { + let pair = pair.trim(); + if let Some((key, value)) = pair.split_once('=') + && key.trim() == name + { + return Some(value.trim().to_owned()); + } + } + None +} + +/// Derives device signals with the device module `[device]` selects. +/// +/// A module a registration supplies is the one the integration registry +/// resolved, passed in as `registered`, and it is shown the User-Agent and +/// the cookies, which carry what a page gathers in the browser. Otherwise +/// [`build_device_module`] gives the built-in module, which reads only the +/// User-Agent, or the Fastly module, which also reads the TLS/H2 signals +/// captured into a [`FastlyHostSignals`]. The Fastly module, and so the +/// signal capture, is built only when selected, so the default request path +/// makes no Fastly-specific signal call. The module is handed a module +/// context carrying the request, that evidence, the settings and the +/// services. +pub(crate) async fn derive_device_signals( + settings: &Settings, + registered: Option>, + req: &FastlyRequest, + services: &RuntimeServices, +) -> DeviceSignals { + let mut headers = HeaderMap::new(); + for (name, shown) in [ + (header::USER_AGENT, true), + (header::COOKIE, registered.is_some()), + ] { + if shown + && let Some(value) = req + .get_header_str(name.as_str()) + .and_then(|value| HeaderValue::from_str(value).ok()) + { + headers.insert(name, value); + } + } + let client_ip = req + .get_client_ip_addr() + .map(|ip| ip.to_string()) + .unwrap_or_default(); + let request_info = BorrowedRequestInfo::new(&client_ip, None).with_headers(&headers); + let built; + let module: &dyn DeviceModule = match registered.as_deref() { + Some(module) => module, + None => { + built = build_device_module(settings, || { + let host_signals: Arc = + Arc::new(FastlyHostSignals::from_request(req)); + Box::new(FastlyDeviceModule::new(host_signals)) as Box + }); + built.as_ref() + } + }; + let context = ModuleContext::new(entry_request(req)) + .with_evidence(&request_info) + .with_settings(settings) + .with_services(services); + module + .detect(context.call(module.id(), module.required_permissions())) + .await +} + +/// What the client's request resolved to, read off the Fastly request before +/// it is converted. +/// +/// The scheme is `https` when the entry point marked the request as arriving +/// over TLS, which it does from Fastly's own TLS metadata. +fn entry_request(req: &FastlyRequest) -> ModuleRequest<'_> { + ModuleRequest::new( + req.get_method(), + req.get_header_str(header::HOST.as_str()) + .unwrap_or_default(), + entry_scheme(req), + req.get_path(), + ) + .with_query(req.get_query_str().unwrap_or_default()) +} + +/// `https` when the entry point marked the request as arriving over TLS, +/// which it does from Fastly's own TLS metadata. +fn entry_scheme(req: &FastlyRequest) -> &'static str { + if req.get_header_str("fastly-ssl").is_some() { + "https" + } else { + "http" + } +} + +/// What the client's request resolved to, owned, for the module calls that +/// run after the request itself has been handed to the router. +struct EntryRequest { + method: fastly::http::Method, + host: String, + scheme: &'static str, + path: String, + query: String, +} + +impl EntryRequest { + fn of(req: &FastlyRequest) -> Self { + Self { + method: req.get_method().clone(), + host: req + .get_header_str(header::HOST.as_str()) + .unwrap_or_default() + .to_owned(), + scheme: entry_scheme(req), + path: req.get_path().to_owned(), + query: req.get_query_str().unwrap_or_default().to_owned(), + } + } + + fn module_request(&self) -> ModuleRequest<'_> { + ModuleRequest::new(&self.method, &self.host, self.scheme, &self.path) + .with_query(&self.query) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use edgezero_core::body::Body as EdgeBody; + use edgezero_core::http::HeaderValue; + use edgezero_core::http::response_builder; + use fastly::mime; + use trusted_server_core::integrations::HeaderMutation; + + fn test_settings() -> Settings { + Settings::from_toml( + r#" + [publisher] + domain = "test-publisher.com" + cookie_domain = ".test-publisher.com" + origin_url = "https://origin.test-publisher.com" + proxy_secret = "unit-test-proxy-secret" + + [geo] + assume_single_jurisdiction = true + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [request_signing] + enabled = false + "#, + ) + .expect("should parse test settings") + } + + /// A device module standing in for one a registration supplies. It marks + /// the signals it returns with the cookie header, the host, the scheme + /// and the address it was shown, so a test can tell both that it ran and + /// what it saw. + struct MarkedDeviceModule; + + impl MarkedDeviceModule { + fn mark( + &self, + request: ModuleRequest<'_>, + request_info: &dyn trusted_server_core::evidence::RequestInfo, + ) -> DeviceSignals { + let mut signals = DeviceSignals::derive_ua_only(request_info.user_agent()); + signals.platform_class = Some(format!( + "marked:{}:{}://{}{}?{}", + request_info.header("cookie").unwrap_or_default(), + request.scheme(), + request.host(), + request.path(), + request.query(), + )); + signals + } + } + + #[async_trait::async_trait(?Send)] + impl DeviceModule for MarkedDeviceModule { + fn id(&self) -> &'static str { + "device.marked" + } + + async fn detect( + &self, + call: trusted_server_core::module_context::ModuleCall<'_>, + ) -> DeviceSignals { + call.inject(self, Self::mark) + .unwrap_or_else(|_| DeviceSignals::unknown()) + } + } + + fn page_request() -> FastlyRequest { + let mut req = FastlyRequest::get("https://example.com/article?page=2"); + req.set_header("host", "example.com"); + req.set_header("user-agent", "Mozilla/5.0 (X11; Linux x86_64) Chrome/140.0"); + req.set_header("cookie", "example_width=1280"); + // The entry point marks a request that arrived over TLS this way. + req.set_header("fastly-ssl", "1"); + req + } + + #[test] + fn the_device_module_the_registry_resolved_classifies_the_request() { + let settings = test_settings(); + let services = + build_finalize_services(&settings, Arc::new(UnavailableKvStore), Arc::default()); + let registered: Arc = Arc::new(MarkedDeviceModule); + + let signals = futures::executor::block_on(derive_device_signals( + &settings, + Some(registered), + &page_request(), + &services, + )); + + assert_eq!( + signals.platform_class.as_deref(), + Some("marked:example_width=1280:https://example.com/article?page=2"), + "the registered module should run and see the page's cookies and the request \ + it classifies" + ); + } + + #[test] + fn with_no_registered_module_the_built_in_module_classifies_the_request() { + let settings = test_settings(); + let services = + build_finalize_services(&settings, Arc::new(UnavailableKvStore), Arc::default()); + let req = page_request(); + + let signals = + futures::executor::block_on(derive_device_signals(&settings, None, &req, &services)); + + assert_eq!( + signals, + DeviceSignals::derive_ua_only(req.get_header_str("user-agent").unwrap_or_default()), + "with nothing registered the built-in User-Agent module should answer" + ); + } + + #[test] + fn pull_sync_noop_states_skip_post_send_graph_factory() { + let calls = std::cell::Cell::new(0); + let result = prepare_pull_sync_after_send(None, || { + calls.set(calls.get() + 1); + Err(Report::new(TrustedServerError::KvStore { + store_name: "unexpected".to_owned(), + message: "graph factory should not run".to_owned(), + })) + }); + assert!( + result.is_none(), + "a skipped pull-sync plan should return none" + ); + assert_eq!(calls.get(), 0, "should not invoke the graph factory"); + } + + #[test] + fn health_response_short_circuits_get_health() { + let req = FastlyRequest::get("https://example.com/health"); + + let mut response = health_response(&req).expect("should build health response"); + + assert_eq!( + response.get_status(), + fastly::http::StatusCode::OK, + "should return 200 OK" + ); + assert_eq!( + response.take_body_str(), + "ok", + "should return the health body" + ); + } + + #[test] + fn health_response_carries_no_version() { + let req = FastlyRequest::get("https://example.com/health"); + + let response = health_response(&req).expect("should build health response"); + + assert!( + response.get_header("x-ts-version").is_none(), + "should write no version on /health, which the service_version middleware writes on a page" + ); + assert!( + response.get_header("x-ts-service-version").is_none(), + "should write no service version on /health" + ); + } + + #[test] + fn health_response_ignores_non_health_paths() { + let req = FastlyRequest::get("https://example.com/auction"); + + assert!( + health_response(&req).is_none(), + "should only short-circuit /health" + ); + } + + /// The stores a service with no runtime overrides resolves, with + /// `selector` as its `__KEY` selector. + fn stores_with_selector(selector: &str) -> RuntimeStoreConfig { + RuntimeStoreConfig { + config_key: selector.to_owned(), + ..RuntimeStoreConfig::from_env(&edgezero_core::env_config::EnvConfig::default()) + } + } + + /// A request carrying `host` as its `Host` header, or none. + fn request_for_host(host: Option<&str>) -> FastlyRequest { + let mut req = FastlyRequest::get("https://example.com/article"); + req.remove_header(header::HOST.as_str()); + if let Some(host) = host { + req.set_header(header::HOST.as_str(), host); + } + req + } + + #[test] + fn a_service_with_one_blob_reads_its_key_whatever_the_host() { + for selector in ["trusted_server_config", "trusted_server_config_staging"] { + let stores = stores_with_selector(selector); + + for host in [None, Some("publisher.example"), Some("not a host")] { + let resolved = stores_for_request(stores.clone(), &request_for_host(host)) + .expect("should serve from the one blob"); + + assert_eq!( + resolved, stores, + "should read `{selector}` for host {host:?}" + ); + } + } + } + + #[test] + fn a_host_selector_refuses_what_is_not_a_host_name() { + for host in [ + None, + Some(""), + Some("trusted_server_config"), + Some("publisher.example.__edgezero_chunks.abc.0"), + Some("[::1]:7676"), + ] { + let refusal = + stores_for_request(stores_with_selector("{host}"), &request_for_host(host)) + .expect_err("should refuse a request that names no host name"); + + assert_eq!( + refusal, + config_selection::Refusal::NoConfigForHost, + "should refuse host {host:?} as one with no app-config blob" + ); + } + } + + #[test] + fn a_host_with_no_app_config_blob_is_answered_421_and_the_answer_is_not_stored() { + let mut response = refusal_response(&config_selection::Refusal::NoConfigForHost); + + assert_eq!( + response.get_status(), + fastly::http::StatusCode::MISDIRECTED_REQUEST, + "should say this service does not answer for the host" + ); + assert_eq!( + response.get_header_str("cache-control"), + Some("private, no-store"), + "a refusal should not be stored, so the host is served once it has a blob" + ); + assert_eq!( + response.take_body_str(), + "Misdirected Request", + "should say only that the request was misdirected" + ); + } + + #[test] + fn an_unreadable_store_is_answered_500() { + let response = refusal_response(&config_selection::Refusal::StoreUnavailable( + "lookup failed".to_owned(), + )); + + assert_eq!( + response.get_status(), + fastly::http::StatusCode::INTERNAL_SERVER_ERROR, + "should not answer 421 while the host's blob is unknown" + ); + } + + #[test] + fn take_finalize_sentinel_strips_sentinel() { + let mut response = HttpResponse::new(EdgeBody::empty()); + response + .headers_mut() + .insert("x-ts-finalized", HeaderValue::from_static("1")); + + assert!( + take_finalize_sentinel(&mut response), + "should detect middleware-finalized responses" + ); + assert!( + response.headers().get("x-ts-finalized").is_none(), + "sentinel should not be sent to clients" + ); + } + + #[test] + fn sandbox_counters_never_appear_on_cacheable_responses() { + let counters = SandboxCounters { + ordinal: 2, + builds: 1, + request_id: "request-example".to_owned(), + }; + for policy in [ + None, + Some("public, s-maxage=3600"), + Some("private, max-age=60"), + Some("private=\"set-cookie\""), + Some("public, extension=\"private, no-store\""), + ] { + let mut response = HttpResponse::new(EdgeBody::empty()); + if let Some(policy) = policy { + response.headers_mut().insert( + "cache-control", + HeaderValue::from_str(policy).expect("should encode cache policy"), + ); + } + apply_terminal_response_effects(&mut response, None); + let original_headers = response.headers().clone(); + + attach_sandbox_counters(&mut response, &counters); + + assert_eq!( + response.headers(), + &original_headers, + "should preserve cache policy and omit all counters for {policy:?}" + ); + } + } + + #[test] + fn sandbox_counters_follow_final_private_no_store_policy() { + let counters = SandboxCounters { + ordinal: 2, + builds: 1, + request_id: "request-example".to_owned(), + }; + let mut response = response_builder() + .header("cache-control", "public, s-maxage=3600") + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(TerminalPrivateResponse); + apply_terminal_response_effects(&mut response, None); + + attach_sandbox_counters(&mut response, &counters); + + assert_eq!( + response.headers()[sandbox::HEADER_SANDBOX_REQUEST_ID], + "request-example", + "should identify this uncached request" + ); + assert_eq!( + response.headers()[sandbox::HEADER_SANDBOX_ORDINAL], + "2", + "should report the current ordinal" + ); + assert_eq!( + response.headers()["cache-control"], + "no-store, private", + "should retain terminal privacy" + ); + } + + #[test] + fn late_filter_effects_cannot_make_an_assembled_response_public() { + let mut response = response_builder() + .header("cache-control", "private, no-store") + .header("etag", "\"reader-document\"") + .body(EdgeBody::empty()) + .expect("should build response"); + response.extensions_mut().insert(TerminalPrivateResponse); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("no-store, private") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + assert!(response.headers().get("etag").is_none()); + } + + #[test] + fn late_filter_effects_cannot_make_a_page_bids_response_public() { + let mut response = trusted_server_core::publisher::page_bids_preflight_denied(); + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("no-store, private") + ); + assert!(response.headers().get("surrogate-control").is_none()); + assert!(response.headers().get("cdn-cache-control").is_none()); + } + + #[test] + fn late_filter_effects_cannot_make_a_response_a_module_made_private_public() { + // The narrowest hole: a module's response finalizer makes a response + // one reader's without setting a cookie, so the `Set-Cookie` privacy + // net never fires, and only the marker the finalizer leaves lets the + // terminal guard enforce the policy again. + let mut response = response_builder() + .header("cache-control", "public, max-age=600") + .body(EdgeBody::empty()) + .expect("should build response"); + trusted_server_core::response_privacy::enforce_terminal_private_cache_privacy( + &mut response, + ); + + let effects = RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("cache-control", "public, s-maxage=3600"), + HeaderMutation::set("surrogate-control", "max-age=3600"), + ], + }; + + apply_terminal_response_effects(&mut response, Some(&effects)); + + assert!( + response.headers().get("set-cookie").is_none(), + "the case under test is the one with no Set-Cookie to protect it" + ); + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a response made for one request must never become shared-cacheable" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip CDN cache directives a late filter added" + ); + } + + #[test] + fn terminal_response_preserves_unmarked_origin_private_policy() { + let mut response = response_builder() + .header("cache-control", "private, max-age=600") + .header("etag", "\"origin\"") + .header("last-modified", "Wed, 12 Aug 2026 00:00:00 GMT") + .body(EdgeBody::empty()) + .expect("should build response"); + + apply_terminal_response_effects(&mut response, None); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, max-age=600"), + "should preserve the origin browser-cache policy" + ); + assert_eq!( + response + .headers() + .get("etag") + .and_then(|value| value.to_str().ok()), + Some("\"origin\""), + "should preserve the origin validator" + ); + assert_eq!( + response + .headers() + .get("last-modified") + .and_then(|value| value.to_str().ok()), + Some("Wed, 12 Aug 2026 00:00:00 GMT"), + "should preserve the origin modification date" + ); + } + + #[test] + fn entry_point_finalize_skips_geo_lookup_for_401() { + let settings = test_settings(); + let mut response = response_builder() + .status(edgezero_core::http::StatusCode::UNAUTHORIZED) + .body(EdgeBody::empty()) + .expect("should build response"); + + // The predicate is what stops the lookup, so assert on it directly: + // a `true` here would send the client IP to the geo module on a + // response that never authenticated. + assert!( + !geo_allowed_for_response(&response), + "should skip entry-point geo lookup for 401 responses" + ); + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE) + .and_then(|v| v.to_str().ok()), + Some("false"), + "401 responses should still carry geo-unavailable headers" + ); + } + + #[test] + fn ja4_debug_response_uses_plain_text_and_fallback_values() { + let req = FastlyRequest::get("https://example.com/_ts/debug/ja4"); + + let mut response = build_ja4_debug_response(&req); + + assert_eq!( + response.get_status(), + fastly::http::StatusCode::OK, + "should return 200 OK" + ); + assert_eq!( + response.get_content_type(), + Some(mime::TEXT_PLAIN_UTF_8), + "should return plain text content" + ); + assert_eq!( + response.get_header_str(fastly::http::header::CACHE_CONTROL), + Some("no-store, private"), + "should disable caching for the debug response" + ); + + let body = response.take_body_str(); + + assert!( + body.contains("ja4: unavailable"), + "should include JA4 fallback" + ); + assert!( + body.contains("h2_fp: unavailable"), + "should include H2 probabilistic identifier fallback" + ); + assert!( + body.contains("cipher: unavailable"), + "should include cipher fallback" + ); + assert!( + body.contains("tls_version: unavailable"), + "should include TLS version fallback" + ); + assert!( + body.contains("user-agent: none"), + "should include user-agent fallback" + ); + assert!( + body.contains("ch-mobile: not sent"), + "should include sec-ch-ua-mobile fallback" + ); + assert!( + body.contains("ch-platform: not sent"), + "should include sec-ch-ua-platform fallback" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 1f2668b0a6..0d34ed941b 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,1287 +1,9 @@ -use std::sync::Arc; +//! The Fastly Compute binary. +//! +//! A shell over the library, which holds everything of substance. A +//! deployment that ships a vendor crate writes its own binary of this shape +//! and calls `run_with` with that crate's builder. -use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; -use edgezero_adapter_fastly::request::into_core_request; -use edgezero_adapter_fastly::runtime_env_config; -use edgezero_core::app::Hooks as _; -use edgezero_core::body::Body as EdgeBody; -use edgezero_core::config_store::ConfigStoreHandle; -use edgezero_core::error::EdgeError; -use edgezero_core::http::{Request as HttpRequest, Response as HttpResponse}; -use edgezero_core::response::IntoResponse; -use error_stack::Report; -use fastly::http::Method as FastlyMethod; -use fastly::{Request as FastlyRequest, Response as FastlyResponse}; - -use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; -use trusted_server_core::constants::HEADER_X_TS_VERSION; -use trusted_server_core::ec::device::DeviceSignals; -use trusted_server_core::ec::finalize::ec_finalize_response; -use trusted_server_core::ec::kv::KvIdentityGraph; -use trusted_server_core::ec::pull_sync::{ - PullSyncContext, build_pull_sync_context, dispatch_pull_sync, -}; -use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::error::TrustedServerError; -use trusted_server_core::integrations::RequestFilterEffects; -use trusted_server_core::platform::PlatformGeo as _; -use trusted_server_core::proxy::{AssetProxyCachePolicy, stream_asset_body}; -use trusted_server_core::response_privacy::TerminalPrivateResponse; -use trusted_server_core::settings::Settings; -use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; - -mod app; -mod backend; -mod compat; -mod ec_kv; -mod esi_assembly; -mod logging; -mod management_api; -mod middleware; -mod platform; -mod rate_limiter; -mod sandbox; -mod template_cache; -mod tinybird; - -use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, -}; -use crate::ec_kv::FastlyEcKvStore; -use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; -use crate::platform::{FastlyPlatformGeo, client_info_from_request}; -use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; -use crate::sandbox::{RetainedApp, Sandbox, SandboxCounters, ServeMode, StartupDiagnostics}; -// Only the reuse path builds a serving loop, so the retirement snapshots have -// no consumer in the default build. -#[cfg(feature = "reusable-sandbox")] -use crate::sandbox::RetirementCounters; - -/// Opens the Fastly Config Store used by the `EdgeZero` dispatcher. -/// -/// # Errors -/// -/// Returns [`fastly::Error`] if the config store cannot be opened. -fn open_trusted_server_config_store(store_name: &str) -> Result { - let store = EdgeZeroFastlyConfigStore::try_open(store_name).map_err(|e| { - fastly::Error::msg(format!("failed to open config store `{store_name}`: {e}")) - })?; - Ok(ConfigStoreHandle::new(Arc::new(store))) -} - -fn health_response(req: &FastlyRequest) -> Option { - if req.get_method() == FastlyMethod::GET && req.get_path() == "/health" { - let mut response = FastlyResponse::from_status(200).with_body_text_plain("ok"); - // Compiled-in constant: keeps the probe free of settings and app construction. - if let Some(version) = git_version_header_value() { - response.set_header(HEADER_X_TS_VERSION, version); - } - return Some(response); - } - - None -} - -/// Entry point for the Fastly Compute program. -/// -/// Uses an undecorated `main()` with `FastlyRequest::from_client()` instead of -/// `#[fastly::main]` so the `EdgeZero` streaming publisher path can call -/// [`fastly::Response::stream_to_client`] explicitly. It owns the sandbox -/// lifecycle and delegates each request to [`handle_request`]. -/// -/// Without the `reusable-sandbox` feature this takes one request and returns, -/// which is the original behaviour. With the feature, the bounds still have to -/// come from the runtime environment before the SDK serving loop is entered; -/// an unconfigured or partially configured sandbox stays single-request. fn main() { - let (mode, diagnostics) = serve_mode(); - - // Held outside the sandbox: `serve_custom` owns the `Sandbox` and exposes - // no slot for application state that is not the retained payload. - let mut startup = StartupDiagnostics::default(); - for message in diagnostics { - startup.push(message); - } - - match mode { - ServeMode::Single => { - // `run_custom` completes the callback's SDK result exactly once. - // The callback sends its own response and returns `()`, so there - // is no error for the SDK to turn into a second response. - let Ok(()) = edgezero_adapter_fastly::lifecycle::run_custom( - FastlyRequest::from_client(), - |request, sandbox: &mut Sandbox| handle_request(request, sandbox, &mut startup), - ); - } - ServeMode::Reuse(limits) => serve_loop(limits, startup), - } -} - -/// Serves requests from one sandbox under explicit bounds. -/// -/// Only compiled with the `reusable-sandbox` feature; [`serve_mode`] can never -/// return [`ServeMode::Reuse`] without it. -#[cfg(feature = "reusable-sandbox")] -fn serve_loop(limits: crate::sandbox::SandboxLimits, mut startup: StartupDiagnostics) { - // `serve_custom` owns the `Sandbox` and drops it when serving ends, so the - // retirement line reports snapshots taken while it was still borrowed. - // `RetirementCounters` reads the attempt count on the way out, so a build - // performed by the final callback is included. - let mut counters = RetirementCounters::default(); - - let summary = edgezero_adapter_fastly::lifecycle::serve_custom( - fastly::http::serve::Serve::new() - .with_max_requests(limits.max_requests) - .with_max_memory(limits.max_memory_mib) - .with_max_lifetime(limits.max_lifetime) - .with_timeout(limits.timeout), - |request, sandbox: &mut Sandbox| { - counters.observe(sandbox, |sandbox| { - handle_request(request, sandbox, &mut startup); - }); - }, - ); - - log::info!( - "sandbox retiring after {} attempted callback(s), {} observed, {} build attempt(s)", - summary.requests(), - counters.requests(), - counters.attempts() - ); -} - -#[cfg(not(feature = "reusable-sandbox"))] -fn serve_loop(_limits: crate::sandbox::SandboxLimits, _startup: StartupDiagnostics) { - unreachable!("serve_mode never selects reuse without the reusable-sandbox feature") -} - -/// Handles one request end to end, sending its own response. -/// -/// Returns `()` rather than a response so the streaming publisher path can -/// call [`fastly::Response::stream_to_client`] itself. The SDK's -/// `HandlerResult` impl for `()` treats that as already sent. -/// -/// Every non-panicking path through this function must send exactly once. In a -/// reused sandbox the SDK refuses to wait for the next request until the -/// current one is complete, so a missed send stalls the loop rather than -/// merely dropping one response. -fn handle_request(req: FastlyRequest, sandbox: &mut Sandbox, startup: &mut StartupDiagnostics) { - // The framework counts the callback before invoking it, including early - // returns, so this is already this request's 1-based ordinal. - let ordinal = sandbox.requests(); - - // Health probe bypasses logging, settings, and app construction as a cheap liveness signal. - if let Some(response) = health_response(&req) { - response.send_to_client(); - return; - } - - // Marked complete only once installation succeeds, so a failed install is - // retried on a later callback. `setup_once` rolls nothing back, so that is - // only correct because `init_logger` is harmless to repeat — see its docs. - match sandbox.setup_once(crate::logging::init_logger) { - Ok(()) => startup.flush(), - Err(error) => { - // Logger installation failed, so its own error cannot rely on log. - // Keep startup diagnostics pending until installation succeeds. - #[allow(clippy::print_stderr, reason = "logger installation failed")] - { - eprintln!("logger installation failed, retrying next callback: {error}"); - } - } - } - - // Correlation is request-local and never retained. `FASTLY_TRACE_ID` names - // the sandbox, not the request, so it is not used here. The id rides on the - // response only when metrics are enabled; the client request is left - // untouched so nothing new reaches origin in the default configuration. - let request_id = req - .get_client_request_id() - .map(str::to_owned) - .unwrap_or_else(|| format!("{}-{ordinal}", instance_id())); - - edgezero_main(req, sandbox, ordinal, &request_id); -} - -/// Builds the counters snapshot response. -/// -/// A snapshot only. It reports the sandbox that served *this* probe, which is -/// not necessarily the sandbox that served any preceding workload request, so -/// reuse is established from the counters attached to workload responses -/// rather than from polling this. -/// -/// `ordinal` is this probe's own 1-based position in the sandbox, which is -/// also the number of callbacks the sandbox has served including this one. -/// The lifetime count is therefore not reported separately. -#[cfg(feature = "reusable-sandbox")] -fn sandbox_metrics_response(sandbox: &Sandbox, ordinal: u64) -> FastlyResponse { - let body = serde_json::json!({ - "instance": instance_id(), - "ordinal": ordinal, - "builds": sandbox.initialization_attempts(), - }); - - FastlyResponse::from_status(fastly::http::StatusCode::OK) - .with_header("cache-control", "private, no-store") - .with_body_json(&body) - .unwrap_or_else(|e| { - log::error!("failed to serialize sandbox metrics: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - }) -} - -/// Attaches sandbox counters only to private, no-store workload responses. -/// -/// Called before headers are committed, which on the streaming path means -/// before `stream_to_client`. The counters therefore describe the request up -/// to commitment and cannot report its eventual outcome; a failure after -/// commitment is recorded in logs instead and reconciled during analysis. -fn attach_sandbox_counters(response: &mut HttpResponse, counters: &SandboxCounters) { - // Per-request measurements must never be replayed from a cache. Requiring - // no-store also excludes browser-cacheable and field-qualified privacy. - if !cache_control_headers_have_directive(response.headers(), "private") - || !cache_control_headers_have_directive(response.headers(), "no-store") - { - return; - } - - let headers = response.headers_mut(); - for (name, value) in [ - (sandbox::HEADER_SANDBOX_INSTANCE, instance_id()), - ( - sandbox::HEADER_SANDBOX_ORDINAL, - counters.ordinal.to_string(), - ), - (sandbox::HEADER_SANDBOX_BUILDS, counters.builds.to_string()), - ( - sandbox::HEADER_SANDBOX_REQUEST_ID, - counters.request_id.clone(), - ), - (sandbox::HEADER_SANDBOX_VCPU_MS, vcpu_ms()), - (sandbox::HEADER_SANDBOX_HEAP_MIB, heap_mib()), - ] { - match edgezero_core::http::HeaderValue::from_str(&value) { - Ok(value) => { - headers.insert(name, value); - } - Err(e) => log::warn!("sandbox counter `{name}` is not a valid header value: {e}"), - } - } -} - -/// Resolves how this sandbox will serve requests. -/// -/// Without the `reusable-sandbox` feature this is unconditionally -/// [`ServeMode::Single`] and reads nothing, so the default build does no -/// startup work the original entry point did not do. -/// Returns the mode alongside diagnostics that must wait for the logger. -/// -/// This runs before any logger exists, so the reasons reuse was declined are -/// carried out rather than logged here, where they would be discarded. -#[cfg(feature = "reusable-sandbox")] -fn serve_mode() -> (ServeMode, Vec) { - let (raw, diagnostics) = crate::sandbox::read_raw_limits(); - (crate::sandbox::resolve_mode(raw), diagnostics) -} - -#[cfg(not(feature = "reusable-sandbox"))] -fn serve_mode() -> (ServeMode, Vec) { - (ServeMode::Single, Vec::new()) -} - -/// Guest-instance identifier used to attribute requests to a sandbox. -/// -/// `FASTLY_TRACE_ID` describes the sandbox, which is exactly what is wanted -/// here and exactly why it must not be used as a request id. An absent value -/// is reported rather than synthesized, so a measurement run cannot silently -/// claim reuse it never observed. -fn instance_id() -> String { - // The SDK documents this as the per-sandbox identifier; on wasm32-wasip1 - // it resolves to `FASTLY_TRACE_ID`, which is why that value must never be - // used as a request id. - let id = fastly::compute_runtime::sandbox_id(); - if id.is_empty() { - return sandbox::INSTANCE_ID_UNAVAILABLE.to_owned(); - } - id.to_owned() -} - -/// Cumulative guest vCPU milliseconds, or a marker when unsupported. -fn vcpu_ms() -> String { - fastly::compute_runtime::elapsed_vcpu_ms().map_or_else( - |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), - |v| v.to_string(), - ) -} - -/// Guest heap snapshot in MiB, or a marker when unsupported. -fn heap_mib() -> String { - fastly::compute_runtime::heap_memory_snapshot_mib().map_or_else( - |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), - |v| v.to_string(), - ) -} - -/// Handles a request through the `EdgeZero` router path. -fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, request_id: &str) { - let runtime_env = runtime_env_config(TrustedServerApp::stores()); - let runtime_stores = RuntimeStoreConfig::from_env(&runtime_env); - - // Short-circuit the sandbox counters probe before app construction. It must - // not build the application: polling it would otherwise increment the very - // build counter it reports. - #[cfg(feature = "reusable-sandbox")] - if req.get_method() == FastlyMethod::GET && req.get_path() == sandbox::SANDBOX_METRICS_PATH { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) if sandbox::metrics_enabled(&settings) => { - sandbox_metrics_response(sandbox, ordinal).send_to_client(); - } - Ok(_) => { - FastlyResponse::from_status(fastly::http::StatusCode::NOT_FOUND).send_to_client(); - } - Err(e) => { - log::warn!("sandbox metrics endpoint: failed to load settings: {e:?}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - } - } - return; - } - - // Short-circuit the JA4 debug probe before app construction. Must run here - // because TLS/JA4 accessors are only available on FastlyRequest before - // conversion to edgezero types. - if req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) if settings.debug.ja4_endpoint_enabled => { - build_ja4_debug_response(&req).send_to_client(); - } - Ok(_) => { - FastlyResponse::from_status(fastly::http::StatusCode::NOT_FOUND).send_to_client(); - } - Err(e) => { - log::warn!("EdgeZero JA4 endpoint: failed to load settings: {e:?}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - } - } - return; - } - - let config_store = - match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { - Ok(cs) => cs, - Err(e) => { - log::error!("failed to open config store: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; - - // Build lazily, once per sandbox. Reached only past the health, JA4, and - // counters short-circuits, so none of those pays for construction. - // - // `initialize` builds only when the sandbox is empty, retains only - // success, and returns the error unchanged. A failed build hands back its - // error router as the error payload: that serves this request and is then - // dropped, so a transient config-store failure cannot pin the sandbox into - // permanent error mode, and the next callback retries construction. - let failed_build = sandbox - .initialize(|| { - let (app, state) = TrustedServerApp::build_app_with_state(&runtime_stores); - match state { - Some(state) => Ok(RetainedApp { app, state }), - None => Err(app), - } - }) - .err(); - - let (app, app_state): (&edgezero_core::app::App, Option>) = - match (failed_build.as_ref(), sandbox.state()) { - (Some(app), _) => (app, None), - (None, Some(retained)) => (&retained.app, Some(Arc::clone(&retained.state))), - (None, None) => { - log::error!("no application available after initialization"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; - - let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); - let counters = - SandboxCounters::capture(sandbox, ordinal, request_id, settings_snapshot.as_deref()); - let trusted_client_ip = settings_snapshot - .as_deref() - .and_then(|settings| settings.trusted_client_ip.as_ref()); - - // Resolve the trusted client IP, then strip client-spoofable forwarded - // headers before dispatch. One call keeps resolution ahead of the - // sanitization that removes the headers it reads. - let resolved_client_ip = compat::resolve_and_sanitize_client_ip(&mut req, trusted_client_ip); - - // Re-inject a trusted TLS scheme signal after sanitization has stripped any - // client-sent fastly-ssl header. Setting it from Fastly's native TLS - // metadata here is authoritative. detect_request_scheme in http_util checks - // this header so scheme-sensitive logic produces https URLs on HTTPS traffic. - if req.get_tls_protocol().ok().flatten().is_some() - || req.get_tls_cipher_openssl_name().ok().flatten().is_some() - { - req.set_header("fastly-ssl", "1"); - } - - // Strip any client-supplied x-ts-tls-* headers before injecting the trusted - // values from the Fastly SDK. Must run after sanitize_fastly_forwarded_headers. - req.remove_header("x-ts-tls-protocol"); - req.remove_header("x-ts-tls-cipher"); - if let Some(proto) = req.get_tls_protocol().ok().flatten().map(str::to_owned) { - req.set_header("x-ts-tls-protocol", proto); - } - if let Some(cipher) = req - .get_tls_cipher_openssl_name() - .ok() - .flatten() - .map(str::to_owned) - { - req.set_header("x-ts-tls-cipher", cipher); - } - - // Capture metadata from the original FastlyRequest before conversion. These - // accessors only return real values on the client request, so store them in - // request extensions for build_per_request_services and EC bot classification. - let client_info = client_info_from_request(&req, resolved_client_ip); - let client_ip = client_info.client_ip; - let device_signals = derive_device_signals(&req); - - // Dispatch directly through the EdgeZero router without an intermediate - // fastly::Response conversion. That preserves duplicate header values such - // as multiple Set-Cookie headers. - let mut response = match into_core_request(req) { - Ok(mut core_req) => { - core_req.extensions_mut().insert(config_store); - core_req.extensions_mut().insert(device_signals); - core_req.extensions_mut().insert(client_info); - match futures::executor::block_on(app.router().oneshot(core_req)) { - Ok(response) => response, - Err(error) => edge_error_response(error), - } - } - Err(e) => { - log::error!("EdgeZero request conversion failed: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; - - // Pop response extensions before the Fastly conversion, which drops them. - let ec_state = response.extensions_mut().remove::(); - let asset_cache_policy = response.extensions_mut().remove::(); - let request_filter_effects = response.extensions_mut().remove::(); - - if !take_finalize_sentinel(&mut response) { - if let Some(settings) = settings_snapshot.as_deref() { - apply_entry_point_finalize_headers(settings, &mut response, client_ip); - } else { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) => { - apply_entry_point_finalize_headers(&settings, &mut response, client_ip); - } - Err(e) => { - log::warn!("entry-point finalize skipped: failed to reload settings: {e:?}"); - // The compiled-in version needs no settings, so a deploy that cannot - // load its config still reports which build is serving. - apply_git_version_header(&mut response); - } - } - } - } - - if let Some(policy) = asset_cache_policy { - policy.apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); - } - - if let Some(mut ec_state) = ec_state { - if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { - Ok(partner_registry) => { - send_edgezero_response( - response, - request_filter_effects.as_ref(), - counters.as_ref(), - ); - run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); - return; - } - Err(e) => { - log::error!( - "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" - ); - } - } - } else { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { - Ok(partner_registry) => { - send_edgezero_response( - response, - request_filter_effects.as_ref(), - counters.as_ref(), - ); - run_edgezero_pull_sync_after_send( - &settings, - &partner_registry, - &ec_state, - ); - return; - } - Err(e) => { - log::error!( - "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" - ); - } - } - } - Err(e) => { - log::warn!("EdgeZero EC finalize skipped: failed to reload settings: {e:?}"); - } - } - } - } - - send_edgezero_response(response, request_filter_effects.as_ref(), counters.as_ref()); -} - -fn edge_error_response(error: EdgeError) -> HttpResponse { - log::error!("EdgeZero router returned error: {error:?}"); - match error.into_response() { - Ok(response) => response, - Err(error) => { - log::error!("failed to convert EdgeZero error into response: {error:?}"); - edgezero_core::http::response_builder() - .status(edgezero_core::http::StatusCode::INTERNAL_SERVER_ERROR) - .body(EdgeBody::from("Internal Server Error")) - .expect("should build EdgeZero error response") - } - } -} - -fn take_finalize_sentinel(response: &mut HttpResponse) -> bool { - response - .headers_mut() - .remove(HEADER_X_TS_FINALIZED) - .is_some() -} - -fn apply_entry_point_finalize_headers( - settings: &Settings, - response: &mut HttpResponse, - client_ip: Option, -) { - let geo_info = resolve_geo_for_response(response, client_ip, |client_ip| { - FastlyPlatformGeo.lookup(client_ip).unwrap_or_else(|e| { - log::warn!("entry-point geo lookup failed: {e}"); - None - }) - }); - apply_finalize_headers(settings, geo_info.as_ref(), response); -} - -fn apply_edgezero_ec_finalize( - settings: &Settings, - ec_state: &mut EcFinalizeState, - response: &mut HttpResponse, -) -> Result> { - let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; - let finalize_kv_graph = if ec_state.use_finalize_kv { - maybe_identity_graph(settings) - } else { - None - }; - ec_finalize_response( - settings, - &mut ec_state.ec_context, - finalize_kv_graph.as_ref(), - &partner_registry, - ec_state.eids_cookie.as_deref(), - ec_state.sharedid_cookie.as_deref(), - response, - ); - Ok(partner_registry) -} - -fn run_edgezero_pull_sync_after_send( - settings: &Settings, - partner_registry: &PartnerRegistry, - ec_state: &EcFinalizeState, -) { - if !ec_state.is_real_browser { - return; - } - - let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); - let Some((context, kv)) = - prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) - else { - return; - }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync( - settings, - &kv, - partner_registry, - &limiter, - &context, - &ec_state.services, - ); -} - -fn prepare_pull_sync_after_send( - context: Option, - graph_factory: F, -) -> Option<(PullSyncContext, KvIdentityGraph)> -where - F: FnOnce() -> Result>, -{ - let context = context?; - let kv = match graph_factory() { - Ok(kv) => kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return None; - } - }; - Some((context, kv)) -} - -/// Sends a finalized `EdgeZero` response to the client. -/// -/// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's -/// client stream so large asset and publisher-origin responses do not -/// materialize in the Wasm heap. -fn send_edgezero_response( - mut response: HttpResponse, - request_filter_effects: Option<&RequestFilterEffects>, - counters: Option<&SandboxCounters>, -) { - apply_terminal_response_effects(&mut response, request_filter_effects); - - // Captured before the body is consumed so post-commitment failures can be - // matched back to the response that carried these counters. - let counter_context = counters.map_or_else(String::new, |counters| { - format!( - " [instance={} ordinal={} request={}]", - instance_id(), - counters.ordinal, - counters.request_id - ) - }); - - // Before headers commit, including before `stream_to_client` below. - if let Some(counters) = counters.as_ref() { - attach_sandbox_counters(&mut response, counters); - } - - let (parts, body) = response.into_parts(); - - match body { - EdgeBody::Stream(_) => { - let skeleton = compat::to_fastly_response_skeleton(HttpResponse::from_parts( - parts, - EdgeBody::empty(), - )); - let mut streaming_body = skeleton.stream_to_client(); - match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { - Ok(()) => { - if let Err(e) = streaming_body.finish() { - // Also post-commitment: same attribution as the - // streaming failure below. - log::error!( - "failed to finish EdgeZero streaming body{counter_context}: {e}" - ); - } - } - Err(e) => { - // After commitment: log and stop. Returning an error here - // would let the SDK attempt a second response. Counters - // already went out with the headers, so the failure is - // tagged with the same identity for reconciliation. - log::error!("EdgeZero streaming failed{counter_context}: {e:?}"); - drop(streaming_body); - } - } - } - once => { - compat::to_fastly_response(HttpResponse::from_parts(parts, once)).send_to_client(); - } - } -} - -/// Apply every late response mutation, then restore privacy invariants before headers commit. -fn apply_terminal_response_effects( - response: &mut HttpResponse, - request_filter_effects: Option<&RequestFilterEffects>, -) { - let must_remain_private = response - .extensions() - .get::() - .is_some(); - if let Some(effects) = request_filter_effects { - effects.apply_to_response(response); - } - if must_remain_private { - trusted_server_core::response_privacy::enforce_private_no_store(response); - } - - // Final cache guards: EC finalization and request-filter effects may have - // added a per-user Set-Cookie or a private/no-store directive after - // `apply_finalize_headers` and normalized asset policy reapplication ran. - crate::middleware::enforce_set_cookie_cache_privacy(response); - crate::middleware::enforce_uncacheable_cache_privacy(response); -} - -const FALLBACK_UNAVAILABLE: &str = "unavailable"; -const FALLBACK_NOT_SENT: &str = "not sent"; -const FALLBACK_NONE: &str = "none"; - -// TODO: remove after JA4 evaluation completes - see #645 -fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { - let ja4 = req.get_tls_ja4().unwrap_or(FALLBACK_UNAVAILABLE); - let h2 = req - .get_client_h2_fingerprint() - .unwrap_or(FALLBACK_UNAVAILABLE); - let cipher = req - .get_tls_cipher_openssl_name() - .ok() - .flatten() - .unwrap_or(FALLBACK_UNAVAILABLE); - let tls_version = req - .get_tls_protocol() - .ok() - .flatten() - .unwrap_or(FALLBACK_UNAVAILABLE); - let ua = req.get_header_str("user-agent").unwrap_or(FALLBACK_NONE); - let ch_mobile = req - .get_header_str("sec-ch-ua-mobile") - .unwrap_or(FALLBACK_NOT_SENT); - let ch_platform = req - .get_header_str("sec-ch-ua-platform") - .unwrap_or(FALLBACK_NOT_SENT); - - let body = format!( - "ja4: {ja4}\n\ - h2_fp: {h2}\n\ - cipher: {cipher}\n\ - tls_version: {tls_version}\n\ - user-agent: {ua}\n\ - ch-mobile: {ch_mobile}\n\ - ch-platform: {ch_platform}\n" - ); - - FastlyResponse::from_status(fastly::http::StatusCode::OK) - .with_header(fastly::http::header::CACHE_CONTROL, "no-store, private") - .with_header( - fastly::http::header::VARY, - "User-Agent, Sec-CH-UA-Mobile, Sec-CH-UA-Platform", - ) - .with_content_type(fastly::mime::TEXT_PLAIN_UTF_8) - .with_body(body) -} - -pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option { - settings - .ec - .ec_store - .as_ref() - .map(|store_name| KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) -} - -/// Constructs a `KvIdentityGraph` from settings, or returns an error if the -/// `ec_store` config is not set. -pub(crate) fn require_identity_graph( - settings: &Settings, -) -> Result> { - let store_name = settings.ec.ec_store.as_deref().ok_or_else(|| { - Report::new(TrustedServerError::KvStore { - store_name: "ec.ec_store".to_owned(), - message: "ec.ec_store is not configured".to_owned(), - }) - })?; - Ok(KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) -} - -/// Extracts a named cookie value from the request's `Cookie` header. -pub(crate) fn extract_cookie_value(req: &HttpRequest, name: &str) -> Option { - let cookie_header = req.headers().get("cookie").and_then(|v| v.to_str().ok())?; - for pair in cookie_header.split(';') { - let pair = pair.trim(); - if let Some((key, value)) = pair.split_once('=') - && key.trim() == name - { - return Some(value.trim().to_owned()); - } - } - None -} - -/// Derives device signals from TLS, H2, and UA request data. -/// -/// All extraction is pure in-memory — no KV I/O. The Fastly SDK provides -/// `get_tls_ja4()` and `get_client_h2_fingerprint()` on client requests. -pub(crate) fn derive_device_signals(req: &FastlyRequest) -> DeviceSignals { - let ua = req.get_header_str("user-agent").unwrap_or(""); - let ja4 = req.get_tls_ja4(); - let h2_fp = req.get_client_h2_fingerprint(); - - DeviceSignals::derive(ua, ja4, h2_fp) -} - -#[cfg(test)] -mod tests { - use super::*; - use edgezero_core::body::Body as EdgeBody; - use edgezero_core::http::HeaderValue; - use edgezero_core::http::response_builder; - use fastly::mime; - use trusted_server_core::integrations::HeaderMutation; - - fn test_settings() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - "#, - ) - .expect("should parse test settings") - } - - #[test] - fn pull_sync_noop_states_skip_post_send_graph_factory() { - let calls = std::cell::Cell::new(0); - let result = prepare_pull_sync_after_send(None, || { - calls.set(calls.get() + 1); - Err(Report::new(TrustedServerError::KvStore { - store_name: "unexpected".to_owned(), - message: "graph factory should not run".to_owned(), - })) - }); - assert!( - result.is_none(), - "a skipped pull-sync plan should return none" - ); - assert_eq!(calls.get(), 0, "should not invoke the graph factory"); - } - - #[test] - fn health_response_short_circuits_get_health() { - let req = FastlyRequest::get("https://example.com/health"); - - let mut response = health_response(&req).expect("should build health response"); - - assert_eq!( - response.get_status(), - fastly::http::StatusCode::OK, - "should return 200 OK" - ); - assert_eq!( - response.take_body_str(), - "ok", - "should return the health body" - ); - } - - #[test] - fn health_response_reports_git_version_only() { - let req = FastlyRequest::get("https://example.com/health"); - - let response = health_response(&req).expect("should build health response"); - - assert_eq!( - response.get_header_str("x-ts-version"), - trusted_server_core::constants::TS_GIT_VERSION, - "should report the compiled-in git version on /health" - ); - assert!( - response.get_header("x-ts-fastly-version").is_none(), - "should keep x-ts-fastly-version off the /health fast path" - ); - } - - #[test] - fn health_response_ignores_non_health_paths() { - let req = FastlyRequest::get("https://example.com/auction"); - - assert!( - health_response(&req).is_none(), - "should only short-circuit /health" - ); - } - - #[test] - fn take_finalize_sentinel_strips_sentinel() { - let mut response = HttpResponse::new(EdgeBody::empty()); - response - .headers_mut() - .insert("x-ts-finalized", HeaderValue::from_static("1")); - - assert!( - take_finalize_sentinel(&mut response), - "should detect middleware-finalized responses" - ); - assert!( - response.headers().get("x-ts-finalized").is_none(), - "sentinel should not be sent to clients" - ); - } - - #[test] - fn sandbox_counters_never_appear_on_cacheable_responses() { - let counters = SandboxCounters { - ordinal: 2, - builds: 1, - request_id: "request-example".to_owned(), - }; - for policy in [ - None, - Some("public, s-maxage=3600"), - Some("private, max-age=60"), - Some("private=\"set-cookie\""), - Some("public, extension=\"private, no-store\""), - ] { - let mut response = HttpResponse::new(EdgeBody::empty()); - if let Some(policy) = policy { - response.headers_mut().insert( - "cache-control", - HeaderValue::from_str(policy).expect("should encode cache policy"), - ); - } - apply_terminal_response_effects(&mut response, None); - let original_headers = response.headers().clone(); - - attach_sandbox_counters(&mut response, &counters); - - assert_eq!( - response.headers(), - &original_headers, - "should preserve cache policy and omit all counters for {policy:?}" - ); - } - } - - #[test] - fn sandbox_counters_follow_final_private_no_store_policy() { - let counters = SandboxCounters { - ordinal: 2, - builds: 1, - request_id: "request-example".to_owned(), - }; - let mut response = response_builder() - .header("cache-control", "public, s-maxage=3600") - .body(EdgeBody::empty()) - .expect("should build response"); - response.extensions_mut().insert(TerminalPrivateResponse); - apply_terminal_response_effects(&mut response, None); - - attach_sandbox_counters(&mut response, &counters); - - assert_eq!( - response.headers()[sandbox::HEADER_SANDBOX_REQUEST_ID], - "request-example", - "should identify this uncached request" - ); - assert_eq!( - response.headers()[sandbox::HEADER_SANDBOX_ORDINAL], - "2", - "should report the current ordinal" - ); - assert_eq!( - response.headers()["cache-control"], - "no-store, private", - "should retain terminal privacy" - ); - } - - #[test] - fn late_filter_effects_cannot_make_an_assembled_response_public() { - let mut response = response_builder() - .header("cache-control", "private, no-store") - .header("etag", "\"reader-document\"") - .body(EdgeBody::empty()) - .expect("should build response"); - response.extensions_mut().insert(TerminalPrivateResponse); - let effects = RequestFilterEffects { - request_headers: Vec::new(), - response_headers: vec![ - HeaderMutation::set("cache-control", "public, s-maxage=3600"), - HeaderMutation::set("surrogate-control", "max-age=3600"), - HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), - ], - }; - - apply_terminal_response_effects(&mut response, Some(&effects)); - - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|value| value.to_str().ok()), - Some("no-store, private") - ); - assert!(response.headers().get("surrogate-control").is_none()); - assert!(response.headers().get("cdn-cache-control").is_none()); - assert!(response.headers().get("etag").is_none()); - } - - #[test] - fn late_filter_effects_cannot_make_a_page_bids_response_public() { - let mut response = trusted_server_core::publisher::page_bids_preflight_denied(); - let effects = RequestFilterEffects { - request_headers: Vec::new(), - response_headers: vec![ - HeaderMutation::set("cache-control", "public, s-maxage=3600"), - HeaderMutation::set("surrogate-control", "max-age=3600"), - HeaderMutation::set("cdn-cache-control", "public, max-age=3600"), - ], - }; - - apply_terminal_response_effects(&mut response, Some(&effects)); - - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|value| value.to_str().ok()), - Some("no-store, private") - ); - assert!(response.headers().get("surrogate-control").is_none()); - assert!(response.headers().get("cdn-cache-control").is_none()); - } - - fn diagnostics_settings() -> Settings { - Settings::from_toml( - r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - - [integrations.gpt_diagnostics] - enabled = true - "#, - ) - .expect("should parse diagnostics settings") - } - - #[test] - fn late_filter_effects_cannot_make_an_active_diagnostics_response_public() { - // The narrowest hole: an established diagnostics session sets no new cookie, so - // the `Set-Cookie` privacy net never fires, and before this the decision only - // stamped `Cache-Control` without leaving a marker for the terminal guard. - let mut request = edgezero_core::http::request_builder() - .method(fastly::http::Method::GET) - .uri("https://test-publisher.com/article") - .header("sec-fetch-dest", "document") - .header("cookie", "__Host-ts-console=1") - .body(EdgeBody::empty()) - .expect("should build request"); - let decision = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &diagnostics_settings(), - &mut request, - ) - .expect("should prepare the diagnostics decision"); - assert!( - decision.active(), - "the session cookie should activate diagnostics" - ); - - let mut response = response_builder() - .header("cache-control", "public, max-age=600") - .body(EdgeBody::empty()) - .expect("should build response"); - trusted_server_core::integrations::gpt_diagnostics::finalize_response( - &decision, - &mut response, - ); - - let effects = RequestFilterEffects { - request_headers: Vec::new(), - response_headers: vec![ - HeaderMutation::set("cache-control", "public, s-maxage=3600"), - HeaderMutation::set("surrogate-control", "max-age=3600"), - ], - }; - - apply_terminal_response_effects(&mut response, Some(&effects)); - - assert!( - response.headers().get("set-cookie").is_none(), - "the case under test is the one with no Set-Cookie to protect it" - ); - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "request-scoped diagnostics HTML must never become shared-cacheable" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "should strip CDN cache directives a late filter added" - ); - } - - #[test] - fn terminal_response_preserves_unmarked_origin_private_policy() { - let mut response = response_builder() - .header("cache-control", "private, max-age=600") - .header("etag", "\"origin\"") - .header("last-modified", "Wed, 12 Aug 2026 00:00:00 GMT") - .body(EdgeBody::empty()) - .expect("should build response"); - - apply_terminal_response_effects(&mut response, None); - - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|value| value.to_str().ok()), - Some("private, max-age=600"), - "should preserve the origin browser-cache policy" - ); - assert_eq!( - response - .headers() - .get("etag") - .and_then(|value| value.to_str().ok()), - Some("\"origin\""), - "should preserve the origin validator" - ); - assert_eq!( - response - .headers() - .get("last-modified") - .and_then(|value| value.to_str().ok()), - Some("Wed, 12 Aug 2026 00:00:00 GMT"), - "should preserve the origin modification date" - ); - } - - #[test] - #[allow(clippy::panic)] - fn entry_point_finalize_skips_geo_lookup_for_401() { - let settings = test_settings(); - let mut response = response_builder() - .status(edgezero_core::http::StatusCode::UNAUTHORIZED) - .body(EdgeBody::empty()) - .expect("should build response"); - - let geo_info = resolve_geo_for_response(&response, None, |_| { - panic!("should skip entry-point geo lookup for 401 responses"); - }); - apply_finalize_headers(&settings, geo_info.as_ref(), &mut response); - - assert_eq!( - response - .headers() - .get(trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE) - .and_then(|v| v.to_str().ok()), - Some("false"), - "401 responses should still carry geo-unavailable headers" - ); - } - - #[test] - fn ja4_debug_response_uses_plain_text_and_fallback_values() { - let req = FastlyRequest::get("https://example.com/_ts/debug/ja4"); - - let mut response = build_ja4_debug_response(&req); - - assert_eq!( - response.get_status(), - fastly::http::StatusCode::OK, - "should return 200 OK" - ); - assert_eq!( - response.get_content_type(), - Some(mime::TEXT_PLAIN_UTF_8), - "should return plain text content" - ); - assert_eq!( - response.get_header_str(fastly::http::header::CACHE_CONTROL), - Some("no-store, private"), - "should disable caching for the debug response" - ); - - let body = response.take_body_str(); - - assert!( - body.contains("ja4: unavailable"), - "should include JA4 fallback" - ); - assert!( - body.contains("h2_fp: unavailable"), - "should include H2 probabilistic identifier fallback" - ); - assert!( - body.contains("cipher: unavailable"), - "should include cipher fallback" - ); - assert!( - body.contains("tls_version: unavailable"), - "should include TLS version fallback" - ); - assert!( - body.contains("user-agent: none"), - "should include user-agent fallback" - ); - assert!( - body.contains("ch-mobile: not sent"), - "should include sec-ch-ua-mobile fallback" - ); - assert!( - body.contains("ch-platform: not sent"), - "should include sec-ch-ua-platform fallback" - ); - } + trusted_server_adapter_fastly::run(); } diff --git a/crates/trusted-server-adapter-fastly/src/management_api.rs b/crates/trusted-server-adapter-fastly/src/management_api.rs deleted file mode 100644 index a8469bcca7..0000000000 --- a/crates/trusted-server-adapter-fastly/src/management_api.rs +++ /dev/null @@ -1,416 +0,0 @@ -//! Fastly management API transport for store write operations. -//! -//! Provides [`FastlyManagementApiClient`], which wraps the Fastly REST -//! management API for write operations on config and secret stores. -//! Used by [`super::platform::FastlyPlatformConfigStore`] and -//! [`super::platform::FastlyPlatformSecretStore`] to back store write methods. -//! -//! # Credentials -//! -//! The Fastly API token is read from the `api-keys` secret store under the -//! `api_key` entry. The token must have config-store write and secret-store -//! write permissions only — no service-level admin or purge permissions. -//! -//! # Security -//! -//! Credential values are never logged. Log messages include store IDs and -//! operation names only. - -use std::io::Read; - -use base64::{Engine as _, engine::general_purpose}; -use error_stack::{Report, ResultExt}; -use fastly::http::{Method, StatusCode}; -use fastly::{Request, Response}; -use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreName}; - -use crate::backend::BackendConfig; -use crate::platform::FastlyPlatformSecretStore; - -const FASTLY_API_HOST: &str = "https://api.fastly.com"; -const API_KEYS_STORE: &str = "api-keys"; -const API_KEY_ENTRY: &str = "api_key"; -const ERROR_BODY_LIMIT: usize = 200; - -fn encode_path_segment(value: &str) -> String { - urlencoding::encode(value).into_owned() -} - -pub(crate) fn build_config_item_payload(value: &str) -> String { - format!("item_value={}", urlencoding::encode(value)) -} - -pub(crate) fn build_config_item_path(store_id: &str, key: &str) -> String { - format!( - "/resources/stores/config/{}/item/{}", - encode_path_segment(store_id), - encode_path_segment(key) - ) -} - -fn build_secret_collection_path(store_id: &str) -> String { - format!( - "/resources/stores/secret/{}/secrets", - encode_path_segment(store_id) - ) -} - -fn build_secret_path(store_id: &str, secret_name: &str) -> String { - format!( - "/resources/stores/secret/{}/secrets/{}", - encode_path_segment(store_id), - encode_path_segment(secret_name) - ) -} - -fn build_secret_payload(secret_name: &str, secret_value: &str) -> String { - serde_json::json!({ - "name": secret_name, - "secret": general_purpose::STANDARD.encode(secret_value.as_bytes()), - "method": "create_or_recreate", - }) - .to_string() -} - -fn truncate_error_body(body: &str) -> String { - body.trim().chars().take(ERROR_BODY_LIMIT).collect() -} - -fn check_response( - response: &mut Response, - error_kind: fn() -> PlatformError, - operation: &str, - entity_description: &str, - store_id: &str, -) -> Result<(), Report> { - if response.get_status().is_success() { - return Ok(()); - } - - let mut body = String::new(); - response - .get_body_mut() - .read_to_string(&mut body) - .change_context(error_kind())?; - - Err(Report::new(error_kind()).attach(format!( - "{} failed with HTTP {} - {} for {} in store '{}'", - operation, - response.get_status(), - truncate_error_body(&body), - entity_description, - store_id - ))) -} - -/// HTTP client for Fastly management API write operations. -/// -/// Backs the `put`/`delete` methods of -/// [`super::platform::FastlyPlatformConfigStore`] and the `create`/`delete` -/// methods of [`super::platform::FastlyPlatformSecretStore`]. -pub(crate) struct FastlyManagementApiClient { - api_key: String, - base_url: &'static str, - backend_name: String, -} - -impl FastlyManagementApiClient { - /// Initialize the client by reading the API token from the `api-keys` secret store. - /// - /// # Errors - /// - /// Returns [`PlatformError::Backend`] if the management API backend cannot - /// be registered, or [`PlatformError::SecretStore`] if the API key cannot - /// be read. - pub(crate) fn new() -> Result> { - let backend_name = BackendConfig::from_url(FASTLY_API_HOST, true) - .change_context(PlatformError::Backend) - .attach("failed to register Fastly management API backend")?; - - let api_key = FastlyPlatformSecretStore - .get_string(&StoreName::from(API_KEYS_STORE), API_KEY_ENTRY) - .change_context(PlatformError::SecretStore) - .attach("failed to read Fastly API key from secret store")?; - - log::debug!("FastlyManagementApiClient: initialized for management API operations"); - - Ok(Self { - api_key, - base_url: FASTLY_API_HOST, - backend_name, - }) - } - - /// Construct a client from explicit components for tests. - #[cfg(test)] - pub(crate) fn with_components( - api_key: String, - base_url: &'static str, - backend_name: String, - ) -> Self { - Self { - api_key, - base_url, - backend_name, - } - } - - fn make_request( - &self, - method: Method, - path: &str, - body: Option, - content_type: &str, - error_kind: fn() -> PlatformError, - ) -> Result> { - let url = format!("{}{}", self.base_url, path); - - let mut request = Request::new(method, &url); - - request = request - .with_header("Fastly-Key", &self.api_key) - .with_header("Accept", "application/json"); - - if let Some(body_content) = body { - request = request - .with_header("Content-Type", content_type) - .with_body(body_content); - } - - request.send(&self.backend_name).map_err(|e| { - Report::new(error_kind()).attach(format!("management API request failed: {}", e)) - }) - } - - /// Update or create a config store item. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns a non-OK status. - pub(crate) fn update_config_item( - &self, - store_id: &str, - key: &str, - value: &str, - ) -> Result<(), Report> { - let path = build_config_item_path(store_id, key); - let payload = build_config_item_payload(value); - - let mut response = self.make_request( - Method::PUT, - &path, - Some(payload), - "application/x-www-form-urlencoded", - || PlatformError::ConfigStore, - )?; - - let entity_description = format!("key '{}'", key); - check_response( - &mut response, - || PlatformError::ConfigStore, - "config item update", - &entity_description, - store_id, - )?; - - log::debug!( - "FastlyManagementApiClient: updated config key '{}' in store '{}'", - key, - store_id - ); - Ok(()) - } - - /// Delete a config store item. - /// - /// Returns `Ok(())` if the item does not exist (404), so retries after - /// partial failures converge without error. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns an unexpected non-2xx status. - pub(crate) fn delete_config_item( - &self, - store_id: &str, - key: &str, - ) -> Result<(), Report> { - let path = build_config_item_path(store_id, key); - - let mut response = - self.make_request(Method::DELETE, &path, None, "application/json", || { - PlatformError::ConfigStore - })?; - - if response.get_status() == StatusCode::NOT_FOUND { - return Ok(()); - } - - let entity_description = format!("key '{}'", key); - check_response( - &mut response, - || PlatformError::ConfigStore, - "config item delete", - &entity_description, - store_id, - )?; - - log::debug!( - "FastlyManagementApiClient: deleted config key '{}' from store '{}'", - key, - store_id - ); - Ok(()) - } - - /// Create or overwrite a secret store entry. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns a non-OK status. - pub(crate) fn create_secret( - &self, - store_id: &str, - secret_name: &str, - secret_value: &str, - ) -> Result<(), Report> { - let path = build_secret_collection_path(store_id); - let payload = build_secret_payload(secret_name, secret_value); - - let mut response = self.make_request( - Method::POST, - &path, - Some(payload), - "application/json", - || PlatformError::SecretStore, - )?; - - let entity_description = format!("name '{}'", secret_name); - check_response( - &mut response, - || PlatformError::SecretStore, - "secret upsert", - &entity_description, - store_id, - )?; - - log::debug!( - "FastlyManagementApiClient: upserted secret '{}' in store '{}'", - secret_name, - store_id - ); - Ok(()) - } - - /// Delete a secret store entry. - /// - /// Returns `Ok(())` if the secret does not exist (404), so retries after - /// partial failures converge without error. - /// - /// # Errors - /// - /// Returns an error if the API request fails or returns an unexpected non-2xx status. - pub(crate) fn delete_secret( - &self, - store_id: &str, - secret_name: &str, - ) -> Result<(), Report> { - let path = build_secret_path(store_id, secret_name); - - let mut response = - self.make_request(Method::DELETE, &path, None, "application/json", || { - PlatformError::SecretStore - })?; - - if response.get_status() == StatusCode::NOT_FOUND { - return Ok(()); - } - - let entity_description = format!("name '{}'", secret_name); - check_response( - &mut response, - || PlatformError::SecretStore, - "secret delete", - &entity_description, - store_id, - )?; - - log::debug!( - "FastlyManagementApiClient: deleted secret '{}' from store '{}'", - secret_name, - store_id - ); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn build_config_item_payload_url_encodes_reserved_characters() { - let payload = build_config_item_payload(r#"value with spaces + symbols &= {"kid":"a+b"}"#); - - assert_eq!( - payload, - "item_value=value%20with%20spaces%20%2B%20symbols%20%26%3D%20%7B%22kid%22%3A%22a%2Bb%22%7D", - "should URL-encode config item values in form payloads" - ); - } - - #[test] - fn build_config_item_path_url_encodes_store_id_and_key() { - let path = build_config_item_path("store/id", "current?kid+#1"); - - assert_eq!( - path, "/resources/stores/config/store%2Fid/item/current%3Fkid%2B%231", - "should percent-encode reserved path characters" - ); - } - - #[test] - fn build_secret_payload_base64_encodes_raw_secret_value() { - let payload = build_secret_payload("signing-key", "raw-secret-value"); - let json: serde_json::Value = - serde_json::from_str(&payload).expect("should serialize secret payload as JSON"); - - assert_eq!(json["name"], "signing-key"); - assert_eq!( - json["secret"], - base64::engine::general_purpose::STANDARD.encode("raw-secret-value"), - "should base64-encode the secret payload for the Fastly API" - ); - assert_eq!( - json["method"], "create_or_recreate", - "should request upsert semantics so re-rotation of the same kid succeeds" - ); - } - - #[test] - fn truncate_error_body_limits_length_after_trimming() { - let body = format!(" {} ", "a".repeat(250)); - - let truncated = truncate_error_body(&body); - - assert_eq!(truncated.len(), 200, "should cap error bodies at 200 chars"); - assert_eq!(truncated, "a".repeat(200), "should trim before truncating"); - } - - #[test] - fn create_secret_uses_secret_store_error_for_transport_failures() { - let client = FastlyManagementApiClient::with_components( - "test-api-key".to_string(), - FASTLY_API_HOST, - "missing-management-backend".to_string(), - ); - - let err = client - .create_secret("store-id", "secret-name", "secret-value") - .expect_err("should fail when the management API backend is unavailable"); - - assert!( - matches!(err.current_context(), &PlatformError::SecretStore), - "should classify secret transport failures as secret-store errors" - ); - } -} diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 9bbc25d7ba..10a0511ced 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -1,13 +1,8 @@ //! Middleware implementations for the `EdgeZero` entry point. //! -//! Provides two middleware types used by the `EdgeZero` entry point: -//! -//! - [`FinalizeResponseMiddleware`] — geo lookup and standard TS header injection -//! - [`AuthMiddleware`] — basic-auth enforcement via [`enforce_basic_auth`] -//! -//! Registration order in [`crate::app`]: `FinalizeResponseMiddleware` outermost, -//! then `AuthMiddleware`. This ensures auth-rejected responses also receive the -//! standard TS headers before being returned to the client. +//! Provides the middleware the `EdgeZero` entry point registers, +//! [`FinalizeResponseMiddleware`], which looks the reader's location up and +//! injects the standard TS headers. use std::sync::Arc; @@ -18,16 +13,16 @@ use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; -use std::net::IpAddr; -use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::{ - ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, HEADER_X_GEO_INFO_AVAILABLE, - HEADER_X_TS_ENV, HEADER_X_TS_FASTLY_VERSION, + ENV_FASTLY_IS_STAGING, HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, }; use trusted_server_core::geo::GeoInfo; -use trusted_server_core::platform::{ClientInfo, PlatformGeo}; +use trusted_server_core::http_util::{ROUTE_NAMESPACE, strip_deployment_headers}; +use trusted_server_core::integrations::IntegrationRegistry; +use trusted_server_core::module_context::ModuleRequest; +use trusted_server_core::platform::{ClientInfo, PlatformGeo, RuntimeServices}; +use trusted_server_core::prefix::Prefixes; use trusted_server_core::settings::Settings; -use trusted_server_core::version_header::apply_git_version_header; pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; @@ -37,33 +32,54 @@ pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; /// Outermost middleware: performs geo lookup and injects all standard TS response headers. /// -/// Registered first in the middleware chain so that it wraps all inner middleware -/// (including [`AuthMiddleware`]) and the handler. This guarantees every registered-route -/// response — including auth-rejected ones — carries a consistent set of headers. +/// Registered first in the middleware chain so that it wraps the handler. This +/// guarantees every registered-route response carries a consistent set of headers. /// /// Router-level 405/404 responses for unregistered HTTP methods (e.g. TRACE) bypass the /// middleware chain. Those are covered by a second call to [`apply_finalize_headers`] at -/// the `main.rs` entry point. Middleware-finalized responses carry +/// the `lib.rs` entry point. Middleware-finalized responses carry /// [`HEADER_X_TS_FINALIZED`] so the entry point can skip duplicate finalization. /// /// # Header precedence /// /// Headers are written in this order (last write wins): /// 1. Geo headers (or `X-Geo-Info-Available: false` when geo is unavailable) -/// 2. `X-TS-Version` from the compiled-in git version -/// (`TRUSTED_SERVER__GIT_VERSION`), and `X-TS-Fastly-Version` from the -/// `FASTLY_SERVICE_VERSION` env var -/// 3. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` -/// 4. Operator-configured `settings.response_headers` (can override any managed header) +/// 2. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` +/// 3. Operator-configured `settings.response_headers` (can override any managed header) pub struct FinalizeResponseMiddleware { settings: Arc, geo: Arc, + /// The services a geo module is given for the response-side lookup. + /// Built once with the application and cloned per request with that + /// request's client metadata applied. + services: RuntimeServices, + /// The registry whose selected prefix module names the publisher's + /// prefixes, or `None` where no application state was built. + registry: Option>, } impl FinalizeResponseMiddleware { - /// Creates a new [`FinalizeResponseMiddleware`] with the given settings and geo lookup service. - pub fn new(settings: Arc, geo: Arc) -> Self { - Self { settings, geo } + /// Creates a new [`FinalizeResponseMiddleware`] with the given settings, + /// geo lookup service, and the services graph handed to the geo module. + pub fn new( + settings: Arc, + geo: Arc, + services: RuntimeServices, + ) -> Self { + Self { + settings, + geo, + services, + registry: None, + } + } + + /// The registry whose selected prefix module names each request's + /// publisher prefixes. + #[must_use] + pub fn with_registry(mut self, registry: Arc) -> Self { + self.registry = Some(registry); + self } } @@ -74,6 +90,25 @@ impl Middleware for FinalizeResponseMiddleware { || FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), |info| info.client_ip, ); + let path = ctx.request().uri().path().to_owned(); + // The publisher this request resolved to names the keys the geo + // module's answers are held under, so the services carry its prefixes, + // which are core's own where no registry names a prefix module. + let client = ctx + .request() + .extensions() + .get::() + .cloned() + .unwrap_or_default(); + let services = match &self.registry { + Some(registry) => self.services.clone().with_prefixes( + registry.prefixes(&self.settings, ModuleRequest::of(ctx.request(), &client)), + ), + None => self + .services + .clone() + .with_prefixes(Prefixes::trusted_server(&self.settings)), + }; let mut response = match next.run(ctx).await { Ok(r) => r, @@ -83,13 +118,24 @@ impl Middleware for FinalizeResponseMiddleware { } }; - let geo_info = resolve_geo_for_response(&response, client_ip, |ip| { - self.geo.lookup(ip).unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }) - }); + let geo_info = if geo_allowed_for_response(&response) { + let services = services.clone().with_client_info(ClientInfo { + client_ip, + ..ClientInfo::default() + }); + self.geo + .lookup(client_ip, &services) + .await + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }) + } else { + None + }; + // Before finalizing, so a header finalizing writes is kept. + strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); response .headers_mut() @@ -99,63 +145,22 @@ impl Middleware for FinalizeResponseMiddleware { } } -// --------------------------------------------------------------------------- -// AuthMiddleware -// --------------------------------------------------------------------------- - -/// Inner middleware: enforces basic-auth before the handler runs. -/// -/// - `Ok(Some(response))` from [`enforce_basic_auth`] → auth failed; return the -/// challenge response (bubbles through [`FinalizeResponseMiddleware`] for header injection). -/// - `Ok(None)` → no auth required or credentials accepted; continue the chain. -/// - `Err(report)` → internal error; log and convert to an HTTP response via -/// [`crate::app::http_error`] using the error's documented status code. -/// -/// # Errors -/// -/// When [`enforce_basic_auth`] returns an error report, converts it to an HTTP -/// response via [`crate::app::http_error`] (preserving the error's status code) -/// so that [`FinalizeResponseMiddleware`] can still inject standard TS headers -/// before the response reaches the client. -pub struct AuthMiddleware { - settings: Arc, -} - -impl AuthMiddleware { - /// Creates a new [`AuthMiddleware`] with the given settings. - pub fn new(settings: Arc) -> Self { - Self { settings } - } -} - -#[async_trait(?Send)] -impl Middleware for AuthMiddleware { - async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { - Ok(Some(response)) => return Ok(response), - Ok(None) => {} - Err(report) => { - log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); - } - } - - next.run(ctx).await - } -} - // --------------------------------------------------------------------------- // Shared geo resolution helper // --------------------------------------------------------------------------- /// Resolves geo for a response, skipping the lookup for 401 responses. /// -/// Returns `None` for authentication rejections (401) without calling `lookup_geo` -/// to avoid unnecessary work and exposing geo data to unauthenticated callers. -/// All other responses call `lookup_geo` and return its result. +/// Returns `false` for authentication rejections (401), so the caller skips the +/// lookup entirely rather than doing unnecessary work and exposing geo data to +/// unauthenticated callers. Every other response is allowed a lookup. +/// +/// This is a predicate rather than a wrapper that runs the lookup itself +/// because [`PlatformGeo::lookup`] is asynchronous, and the decision is the +/// only part worth sharing between the two callers. /// /// Used by both [`FinalizeResponseMiddleware`] and the entry-point finalization -/// in `main.rs` so the 401-skip rule is defined in one place. +/// in `lib.rs` so the 401-skip rule is defined in one place. /// /// # Parity note /// @@ -165,19 +170,8 @@ impl Middleware for AuthMiddleware { /// is intentionally more conservative: geo data is not sent to any /// unauthenticated caller regardless of whether the 401 originated from this /// server or the upstream origin. -pub(crate) fn resolve_geo_for_response( - response: &Response, - client_ip: Option, - lookup_geo: F, -) -> Option -where - F: FnOnce(Option) -> Option, -{ - if response.status() == StatusCode::UNAUTHORIZED { - None - } else { - lookup_geo(client_ip) - } +pub(crate) fn geo_allowed_for_response(response: &Response) -> bool { + response.status() != StatusCode::UNAUTHORIZED } // --------------------------------------------------------------------------- @@ -190,15 +184,14 @@ where /// /// Header write order (last write wins): /// 1. Geo headers (`x-geo-*`) — or `X-Geo-Info-Available: false` when absent -/// 2. `X-TS-Version` from the compiled-in git version -/// (`TRUSTED_SERVER__GIT_VERSION`), and `X-TS-Fastly-Version` from the -/// `FASTLY_SERVICE_VERSION` env var -/// 3. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` -/// 4. Set-Cookie cache privacy — strip surrogate cache headers and downgrade +/// 2. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` +/// 3. Set-Cookie cache privacy — strip surrogate cache headers and downgrade /// `Cache-Control` to `private, max-age=0` on cookie-bearing responses -/// 5. `settings.response_headers` — operator-configured overrides, except the +/// 4. `settings.response_headers` — operator-configured overrides, except the /// cache-controlling headers are skipped on uncacheable (`private`/`no-store`) /// responses so operators cannot re-enable shared caching for per-user payloads +/// 5. `X-Robots-Tag: noindex, nofollow` when `[robots-txt] modules` selects +/// `refuse_all` pub(crate) fn apply_finalize_headers( settings: &Settings, geo_info: Option<&GeoInfo>, @@ -213,13 +206,6 @@ pub(crate) fn apply_finalize_headers( ); } - apply_git_version_header(response); - - apply_fastly_version_header( - std::env::var(ENV_FASTLY_SERVICE_VERSION).ok().as_deref(), - response, - ); - if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() == Ok("1") { response .headers_mut() @@ -234,12 +220,16 @@ pub(crate) fn apply_finalize_headers( trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( settings, response, ); + + // Last, so that nothing an origin or an operator's header set can loosen + // a site taken out of search. + trusted_server_core::robots_txt::apply_response_robots_tag(settings, response); } /// Forces cookie-bearing responses to stay private to shared caches. /// /// Re-exported from [`trusted_server_core::response_privacy`] so the `EdgeZero` -/// entry point (`main.rs`) can re-apply it after +/// entry point (`lib.rs`) can re-apply it after /// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) /// writes the EC identity `Set-Cookie`, using the single shared implementation. pub(crate) use trusted_server_core::response_privacy::{ @@ -250,25 +240,6 @@ pub(crate) use trusted_server_core::response_privacy::{ // Tests // --------------------------------------------------------------------------- -/// Sets `X-TS-Fastly-Version` to `service_version`, or removes an inherited -/// value when the version is unset or invalid, so it is never reported as ours. -fn apply_fastly_version_header(service_version: Option<&str>, response: &mut Response) { - match service_version.map(HeaderValue::from_str) { - Some(Ok(value)) => { - response - .headers_mut() - .insert(HEADER_X_TS_FASTLY_VERSION, value); - } - Some(Err(_)) => { - log::warn!("Skipping invalid FASTLY_SERVICE_VERSION response header value"); - response.headers_mut().remove(HEADER_X_TS_FASTLY_VERSION); - } - None => { - response.headers_mut().remove(HEADER_X_TS_FASTLY_VERSION); - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -305,8 +276,13 @@ mod tests { struct FixedGeo(Option); + #[async_trait::async_trait(?Send)] impl PlatformGeo for FixedGeo { - fn lookup(&self, _: Option) -> Result, Report> { + async fn lookup( + &self, + _: Option, + _services: &trusted_server_core::platform::RuntimeServices, + ) -> Result, Report> { Ok(self.0.clone()) } } @@ -315,10 +291,12 @@ mod tests { lookups: Arc>>>, } + #[async_trait::async_trait(?Send)] impl PlatformGeo for RecordingGeo { - fn lookup( + async fn lookup( &self, client_ip: Option, + _services: &trusted_server_core::platform::RuntimeServices, ) -> Result, Report> { self.lookups .lock() @@ -331,24 +309,23 @@ mod tests { fn test_settings() -> Settings { Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" origin_url = "https://origin.test-publisher.com" proxy_secret = "unit-test-proxy-secret" + [geo] + assume_single_jurisdiction = true + [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [request_signing] enabled = false - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" "#, ) .expect("should parse test settings") @@ -410,6 +387,45 @@ mod tests { response } + #[test] + fn the_data_page_keeps_its_sandbox_under_an_operator_content_security_policy() { + let settings = settings_with_response_headers(vec![ + ("Content-Security-Policy", "default-src 'self'"), + ("X-Frame-Options", "SAMEORIGIN"), + ]); + let request = request_builder() + .method(Method::GET) + .uri("https://publisher.example/_ts/data") + .body(Body::empty()) + .expect("should build request"); + let mut response = trusted_server_core::inspect::data::handle_data( + None, + None, + &trusted_server_core::prefix::PublicPrefix::default(), + &request, + ) + .expect("should answer"); + + apply_finalize_headers(&settings, None, &mut response); + + let policy = response + .headers() + .get("content-security-policy") + .and_then(|v| v.to_str().ok()) + .expect("should carry a policy"); + assert!( + policy.starts_with("sandbox;"), + "an operator policy must not remove the sandbox: {policy}" + ); + assert_eq!( + response + .headers() + .get("x-frame-options") + .and_then(|v| v.to_str().ok()), + Some("DENY") + ); + } + #[test] fn apply_finalize_headers_downgrades_public_set_cookie_response() { // A per-user cookie response that arrives shared-cacheable (origin-public @@ -574,6 +590,17 @@ mod tests { ); } + /// The services graph the finalize middleware hands its geo module, + /// built the same way the router builds it so the tests exercise the + /// production shape rather than a stand-in. + fn test_finalize_services() -> RuntimeServices { + crate::app::build_finalize_services( + &test_settings(), + Arc::new(trusted_server_core::platform::UnavailableKvStore), + Arc::default(), + ) + } + // --------------------------------------------------------------------------- // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- @@ -589,6 +616,7 @@ mod tests { Arc::new(RecordingGeo { lookups: Arc::clone(&lookups), }), + test_finalize_services(), ); let mut ctx = empty_ctx(); ctx.request_mut().extensions_mut().insert(ClientInfo { @@ -625,6 +653,7 @@ mod tests { Arc::new(RecordingGeo { lookups: Arc::clone(&lookups), }), + test_finalize_services(), ); let mut ctx = empty_ctx(); ctx.request_mut() @@ -660,6 +689,7 @@ mod tests { Arc::new(RecordingGeo { lookups: Arc::clone(&lookups), }), + test_finalize_services(), ); let mut ctx = empty_ctx(); FastlyRequestContext::insert( @@ -685,8 +715,11 @@ mod tests { #[test] fn finalize_handle_injects_geo_unavailable_on_ok_response() { let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(None)), + test_finalize_services(), + ); let handler = Arc::new( |_ctx: RequestContext| async move { Ok::(empty_response()) }, @@ -708,8 +741,11 @@ mod tests { #[test] fn finalize_handle_marks_response_as_finalized() { let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(None)), + test_finalize_services(), + ); let handler = Arc::new( |_ctx: RequestContext| async move { Ok::(empty_response()) }, @@ -731,8 +767,11 @@ mod tests { #[test] fn finalize_handle_absorbs_handler_error_and_injects_headers() { let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(None)), + test_finalize_services(), + ); let handler = Arc::new(|_ctx: RequestContext| async move { Err::(EdgeError::service_unavailable("test error")) }); @@ -754,14 +793,23 @@ mod tests { #[allow(clippy::panic)] fn finalize_handle_skips_geo_lookup_for_401() { struct PanicGeo; + #[async_trait::async_trait(?Send)] impl PlatformGeo for PanicGeo { - fn lookup(&self, _: Option) -> Result, Report> { + async fn lookup( + &self, + _: Option, + _services: &trusted_server_core::platform::RuntimeServices, + ) -> Result, Report> { panic!("should not call geo for 401 responses") } } let settings = settings_with_response_headers(vec![]); - let middleware = FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(PanicGeo)); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(PanicGeo), + test_finalize_services(), + ); let handler = Arc::new(|_ctx: RequestContext| async move { let mut resp = empty_response(); *resp.status_mut() = StatusCode::UNAUTHORIZED; @@ -786,10 +834,6 @@ mod tests { ); } - // --------------------------------------------------------------------------- - // AuthMiddleware::handle tests - // --------------------------------------------------------------------------- - #[test] fn finalize_handle_preserves_duplicate_set_cookie_headers() { // Regression guard: FinalizeResponseMiddleware must not drop duplicate @@ -797,8 +841,11 @@ mod tests { // collapsed them because fastly::Response uses set_header (last-wins). // This test verifies the EdgeZero middleware chain is header-transparent. let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(None)), + test_finalize_services(), + ); let handler = Arc::new(|_ctx: RequestContext| async move { let resp = response_builder() .header("set-cookie", "session=abc; Path=/; HttpOnly") @@ -818,64 +865,62 @@ mod tests { ); } - #[test] - fn auth_handle_passes_through_when_auth_not_configured() { - let settings = test_settings(); - let middleware = AuthMiddleware::new(Arc::new(settings)); - let handler = - Arc::new( - |_ctx: RequestContext| async move { Ok::(empty_response()) }, - ); - - let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) - .expect("should pass through when auth is not configured"); + fn ctx_for(path: &str) -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri(path) + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } - assert_eq!( - response.status(), - StatusCode::OK, - "should reach the handler when auth is not required" - ); + /// A response as an origin behind a cache sends it, naming the node that + /// answered and how the cache treated the request. + fn response_naming_its_node() -> Response { + response_builder() + .header("x-served-by", "cache-node-1") + .header("x-cache", "HIT") + .header("x-cache-hits", "3") + .body(Body::empty()) + .expect("should build test response") } - #[test] - fn version_headers_split_git_and_fastly_versions() { - let settings = settings_with_response_headers(vec![]); - let mut response = empty_response(); - response.headers_mut().insert( - HEADER_X_TS_FASTLY_VERSION, - HeaderValue::from_static("origin-value"), + /// What the finalize middleware makes of that response for a request to + /// `path`. + fn finalized(path: &str, settings: Settings) -> Response { + let middleware = FinalizeResponseMiddleware::new( + Arc::new(settings), + Arc::new(FixedGeo(None)), + test_finalize_services(), ); + let handler = Arc::new(|_ctx: RequestContext| async move { + Ok::(response_naming_its_node()) + }); - apply_finalize_headers(&settings, None, &mut response); - - let header = |name: &str| response.headers().get(name).and_then(|v| v.to_str().ok()); - assert_eq!( - header("x-ts-version"), - trusted_server_core::constants::TS_GIT_VERSION, - "should report the compiled-in git version as x-ts-version" - ); - assert_eq!( - header("x-ts-fastly-version"), - std::env::var(ENV_FASTLY_SERVICE_VERSION).ok().as_deref(), - "should report FASTLY_SERVICE_VERSION as x-ts-fastly-version" - ); + block_on(middleware.handle(ctx_for(path), Next::new(&[], &*handler))) + .expect("should run middleware") } #[test] - fn fastly_version_header_removes_inherited_value_when_unknown() { - for service_version in [None, Some("bad\nvalue")] { - let mut response = empty_response(); - response.headers_mut().insert( - HEADER_X_TS_FASTLY_VERSION, - HeaderValue::from_static("origin-value"), - ); - - apply_fastly_version_header(service_version, &mut response); + fn finalize_middleware_strips_deployment_headers_before_it_writes_the_operators() { + let response = finalized( + "/articles/an-article", + settings_with_response_headers(vec![("X-Cache", "configured")]), + ); + for name in ["x-served-by", "x-cache-hits"] { assert!( - response.headers().get(HEADER_X_TS_FASTLY_VERSION).is_none(), - "should remove an inherited x-ts-fastly-version for {service_version:?}" + !response.headers().contains_key(name), + "should strip {name} from a publisher's page" ); } + assert_eq!( + response + .headers() + .get("x-cache") + .and_then(|v| v.to_str().ok()), + Some("configured"), + "should carry the operator's header and not the origin's" + ); } } diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index d0f4ee9d0e..66a5c3b797 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -3,7 +3,6 @@ use bytes::Bytes; use error_stack::{Report, ResultExt}; -use fastly::geo::{Geo, geo_lookup}; use fastly::{ConfigStore, Request, SecretStore}; use std::io::Read as _; use std::net::IpAddr; @@ -11,12 +10,11 @@ use std::net::IpAddr; use crate::backend::BackendConfig; pub(crate) use trusted_server_core::platform::UnavailableKvStore; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, - PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, - PlatformHttpRequest, PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, - PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformImageOptimizerRegion, - PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, - StoreName, + BackendNamingPolicy, ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, + PlatformConfigStore, PlatformError, PlatformHttpClient, PlatformHttpRequest, + PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, + PlatformImageOptimizerParams, PlatformImageOptimizerRegion, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, }; use trusted_server_core::settings::TrustedClientIpConfig; @@ -26,17 +24,9 @@ use trusted_server_core::settings::TrustedClientIpConfig; /// Fastly [`ConfigStore`]-backed implementation of [`PlatformConfigStore`]. /// -/// Stateless — the store name is supplied per call, matching the trait -/// signature. This replaces the store-name-at-construction pattern of -/// the legacy `FastlyConfigStore` (removed). -/// -/// # Write cost -/// -/// `put` and `delete` each perform a synchronous outbound HTTPS request to the -/// Fastly management API (`api.fastly.com`). Callers that issue many writes in -/// one request pay one round-trip per call. The `"api-keys"` secret store is -/// opened per call to read the management token; the Fastly Compute SDK caches -/// the open handle so that cost is negligible. +/// Reads take the store name on each call. The service never writes to a +/// config store, so `put` and `delete` are refused. Signing keys are written +/// by `ts keys`. pub struct FastlyPlatformConfigStore; impl PlatformConfigStore for FastlyPlatformConfigStore { @@ -59,14 +49,14 @@ impl PlatformConfigStore for FastlyPlatformConfigStore { }) } - fn put(&self, store_id: &StoreId, key: &str, value: &str) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.update_config_item(store_id.as_ref(), key, value) + fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::ConfigStore) + .attach("the service does not write to config stores")) } - fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.delete_config_item(store_id.as_ref(), key) + fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::ConfigStore) + .attach("the service does not write to config stores")) } } @@ -76,15 +66,9 @@ impl PlatformConfigStore for FastlyPlatformConfigStore { /// Fastly [`SecretStore`]-backed implementation of [`PlatformSecretStore`]. /// -/// Stateless — the store name is supplied per call. This replaces the -/// store-name-at-construction pattern of the legacy `FastlySecretStore` -/// (removed). -/// -/// # Write cost -/// -/// `create` and `delete` have the same per-call -/// [`crate::management_api::FastlyManagementApiClient`] cost described on -/// [`FastlyPlatformConfigStore`]. +/// Reads take the store name on each call. The service never writes to a +/// secret store, so `create` and `delete` are refused. Signing keys are +/// written by `ts keys`. pub struct FastlyPlatformSecretStore; impl PlatformSecretStore for FastlyPlatformSecretStore { @@ -120,19 +104,14 @@ impl PlatformSecretStore for FastlyPlatformSecretStore { }) } - fn create( - &self, - store_id: &StoreId, - name: &str, - value: &str, - ) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.create_secret(store_id.as_ref(), name, value) + fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::SecretStore) + .attach("the service does not write to secret stores")) } - fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.delete_secret(store_id.as_ref(), name) + fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::SecretStore) + .attach("the service does not write to secret stores")) } } @@ -654,33 +633,12 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { // FastlyPlatformGeo // --------------------------------------------------------------------------- -/// Convert a Fastly [`Geo`] value into a platform-neutral [`GeoInfo`]. -/// -/// Shared by `FastlyPlatformGeo::lookup` in `trusted-server-adapter-fastly` so -/// that field mapping is never duplicated. -fn geo_from_fastly(geo: &Geo) -> GeoInfo { - GeoInfo { - city: geo.city().to_string(), - country: geo.country_code().to_string(), - continent: format!("{:?}", geo.continent()), - latitude: geo.latitude(), - longitude: geo.longitude(), - metro_code: geo.metro_code(), - region: geo.region().map(str::to_string), - asn: None, - } -} - -/// Fastly geo-lookup implementation of [`PlatformGeo`]. -pub struct FastlyPlatformGeo; - -impl PlatformGeo for FastlyPlatformGeo { - fn lookup(&self, client_ip: Option) -> Result, Report> { - Ok(client_ip - .and_then(geo_lookup) - .map(|geo| geo_from_fastly(&geo))) - } -} +/// The Fastly host geo module lives in its own crate, +/// `trusted-server-geo-fastly`, as every module implementation sits under +/// `crates//`. It is re-exported here so this module's +/// [`build_runtime_services`] and the adapter's call sites refer to it +/// through `crate::platform`. +pub(crate) use trusted_server_geo_fastly::FastlyPlatformGeo; fn single_utf8_header<'a>(req: &'a Request, name: &str) -> Option<&'a str> { let mut values = req.get_header_all(name); @@ -1633,7 +1591,7 @@ mod tests { "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ multiple nor a ladder rung" ); - // The mediator hold bound relies on canonicalization never + // The ad server's hold bound relies on canonicalization never // extending a transport cap past the wall-clock budget. assert!( value <= remaining, @@ -1650,7 +1608,7 @@ mod tests { #[test] fn canonicalize_budget_derived_names_stay_bounded_for_large_ceiling() { - // A large configured ceiling (e.g. a 60s mediator budget) must not let + // A large configured ceiling (e.g. a 60s ad server budget) must not let // the budget-derived buckets grow with the ceiling. Without the coarse // ladder a 60,000ms ceiling would mint ~240 distinct 250ms buckets and // blow past Fastly's documented per-service dynamic backend limit (200). @@ -1669,7 +1627,7 @@ mod tests { "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ multiple nor a ladder rung" ); - // The mediator hold bound relies on canonicalization never + // The ad server's hold bound relies on canonicalization never // extending a transport cap past the wall-clock budget. assert!( value <= remaining, diff --git a/crates/trusted-server-adapter-fastly/src/sandbox.rs b/crates/trusted-server-adapter-fastly/src/sandbox.rs index 228106047a..7154c3ec84 100644 --- a/crates/trusted-server-adapter-fastly/src/sandbox.rs +++ b/crates/trusted-server-adapter-fastly/src/sandbox.rs @@ -6,8 +6,11 @@ //! per-sandbox bookkeeping the entry point carries across requests. //! //! [`Sandbox`] owns the state the entry point carries across requests: the -//! logger guard, the measurement counters, and the retained application. +//! logger guard, the measurement counters, and the retained applications. +use std::cell::{Cell, RefCell}; +use std::convert::Infallible; +use std::rc::Rc; use std::sync::Arc; use std::time::Duration; @@ -55,21 +58,103 @@ pub(crate) const INSTANCE_ID_UNAVAILABLE: &str = "unavailable"; /// Per-sandbox state, owned by `edgezero_adapter_fastly::lifecycle::Sandbox`. /// -/// The framework owns lazy successful-only retention, the callback count, the -/// initialization-attempt count, and the one-time setup guard. This alias -/// names the application-specific payload it retains. -pub(crate) type Sandbox = edgezero_adapter_fastly::lifecycle::Sandbox; +/// The framework owns the callback count and the one-time setup guard, and +/// holds the payload this alias names. The payload owns retention. +pub(crate) type Sandbox = edgezero_adapter_fastly::lifecycle::Sandbox; -/// The application state a retained sandbox carries. +/// How many applications a sandbox keeps. +/// +/// Each holds a publisher's settings, registry and routes, and all of them +/// share the sandbox's one memory bound, so the least recently served is +/// dropped past this many. +pub(crate) const MAX_RETAINED_APPS: usize = 8; + +/// The application state a retained sandbox carries for one app-config blob. /// /// Only ever reachable after a successful build: a failed build returns the -/// error router as the `initialize` error instead, so it is served for the +/// error router as the build's error instead, so it is served for the /// current request and dropped rather than retained. pub(crate) struct RetainedApp { pub(crate) app: App, pub(crate) state: Arc, } +/// The applications a sandbox has built, each under the key of the app-config +/// blob it was built from. +/// +/// The framework hands its payload out by shared reference, so the list sits +/// behind a [`RefCell`]. +#[derive(Default)] +pub(crate) struct RetainedApps { + /// Most recently served first. + apps: RefCell)>>, + build_attempts: Cell, +} + +impl RetainedApps { + /// The application built from the blob at `config_key`, building it when + /// the sandbox holds none. + /// + /// Retains only success. A failed build hands its error back unchanged, + /// which serves this request and is dropped, so a transient config-store + /// failure cannot pin the sandbox into error mode and the next request + /// for the key builds again. + /// + /// # Errors + /// + /// Returns the error `build` returned. + pub(crate) fn get_or_build( + &self, + config_key: &str, + build: impl FnOnce() -> Result, + ) -> Result, E> { + if let Some(app) = self.served(config_key) { + return Ok(app); + } + + self.build_attempts + .set(self.build_attempts.get().saturating_add(1)); + let app = Rc::new(build()?); + + let mut apps = self.apps.borrow_mut(); + apps.insert(0, (config_key.to_owned(), Rc::clone(&app))); + apps.truncate(MAX_RETAINED_APPS); + Ok(app) + } + + /// The retained application for `config_key`, marked as the most recently + /// served. + fn served(&self, config_key: &str) -> Option> { + let mut apps = self.apps.borrow_mut(); + let position = apps.iter().position(|(key, _)| key == config_key)?; + let entry = apps.remove(position); + let app = Rc::clone(&entry.1); + apps.insert(0, entry); + Some(app) + } + + /// Builds attempted, failed ones included. + pub(crate) fn build_attempts(&self) -> u64 { + self.build_attempts.get() + } + + /// How many applications are retained. + #[cfg(test)] + pub(crate) fn retained(&self) -> usize { + self.apps.borrow().len() + } +} + +/// Makes the holder of `sandbox`'s applications, once. +pub(crate) fn hold_applications(sandbox: &mut Sandbox) { + let Ok(()) = sandbox.initialize(|| Ok::<_, Infallible>(RetainedApps::default())); +} + +/// Application builds `sandbox` has attempted, failed ones included. +pub(crate) fn build_attempts(sandbox: &Sandbox) -> u64 { + sandbox.state().map_or(0, RetainedApps::build_attempts) +} + /// Startup diagnostics held until a logger exists. /// /// Limit resolution runs in `main`, before any logger is installed, so its @@ -99,16 +184,17 @@ impl StartupDiagnostics { } } -/// Snapshots of the framework's counters, taken around one callback. +/// Snapshots of the sandbox's counters, taken around one callback. /// /// `serve_custom` owns the [`Sandbox`] and drops it when serving ends, so the -/// retirement line cannot read it afterwards. These are snapshots of -/// `EdgeZero`'s counters taken while the sandbox is still borrowed; nothing here -/// increments anything. +/// retirement line cannot read it afterwards. These are snapshots of the +/// callback count `EdgeZero` keeps and the build count [`RetainedApps`] keeps, +/// taken while the sandbox is still borrowed. Nothing here increments +/// anything. /// /// The two are read at different points on purpose. The framework increments /// its callback count *before* invoking the callback, so `requests` is correct -/// on entry. Initialization happens *during* the callback, so `attempts` must +/// on entry. An application is built *during* the callback, so `attempts` must /// be read on the way out. The count itself is never lost — it lives in the /// sandbox until serving ends — but a snapshot taken on entry is stale by the /// time the final callback finishes, so a build it performed goes unreported. @@ -129,7 +215,7 @@ impl RetirementCounters { ) -> R { self.requests = sandbox.requests(); let outcome = callback(sandbox); - self.attempts = sandbox.initialization_attempts(); + self.attempts = build_attempts(sandbox); outcome } @@ -138,7 +224,7 @@ impl RetirementCounters { self.requests } - /// Initialization attempts observed on exit from the last callback. + /// Build attempts observed on exit from the last callback. pub(crate) fn attempts(&self) -> u64 { self.attempts } @@ -170,7 +256,7 @@ impl SandboxCounters { .filter(|settings| metrics_enabled(settings)) .map(|_| Self { ordinal, - builds: sandbox.initialization_attempts(), + builds: build_attempts(sandbox), request_id: request_id.to_owned(), }) } @@ -527,17 +613,15 @@ mod tests { fn test_settings() -> Settings { Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example" cookie_domain = ".test-publisher.example" origin_url = "https://origin.test-publisher.example" proxy_secret = "unit-test-proxy-secret" + [geo] + assume_single_jurisdiction = true + [ec] passphrase = "test-secret-key-32-bytes-minimum" "#, @@ -736,6 +820,27 @@ mod tests { ); } + /// A sandbox holding its applications, as the entry point makes it. + fn holding_sandbox() -> Sandbox { + let mut sandbox = Sandbox::default(); + hold_applications(&mut sandbox); + sandbox + } + + /// Hands back the application for `key`, which must already be retained. + fn retained_for(apps: &RetainedApps, key: &str) -> Rc { + apps.get_or_build(key, || -> Result { + panic!("`{key}` is retained and must not be rebuilt") + }) + .expect("a retained application should be reused") + } + + /// Builds and retains an application for `key`. + fn build_for(apps: &RetainedApps, key: &str) -> Rc { + apps.get_or_build(key, || Ok::<_, &str>(test_retained())) + .expect("the build should succeed") + } + #[test] fn a_sandbox_starts_with_no_retained_application() { let sandbox = Sandbox::default(); @@ -745,19 +850,54 @@ mod tests { "construction must be lazy so the health probe never pays for it" ); assert_eq!( - sandbox.initialization_attempts(), + build_attempts(&sandbox), 0, "a sandbox that has served nothing should report no build attempts" ); } + #[test] + fn holding_applications_builds_none() { + let mut sandbox = holding_sandbox(); + + assert_eq!( + sandbox + .state() + .expect("should hold its applications") + .retained(), + 0, + "should hold no application before one is asked for" + ); + assert_eq!( + build_attempts(&sandbox), + 0, + "making the holder should not count as a build" + ); + + // Asking again must keep what is held, or every request would start + // from an empty holder and nothing would ever be reused. + build_for( + sandbox.state().expect("should hold its applications"), + "trusted_server_config", + ); + hold_applications(&mut sandbox); + assert_eq!( + sandbox + .state() + .expect("should hold its applications") + .retained(), + 1, + "holding again should keep the applications already built" + ); + } + #[test] fn a_retained_application_is_reused_without_rebuilding() { - let mut sandbox = Sandbox::default(); + let apps = RetainedApps::default(); let builds = Cell::new(0_u32); - sandbox - .initialize(|| { + let first = apps + .get_or_build("trusted_server_config", || { builds.set(builds.get() + 1); Ok::<_, &str>(test_retained()) }) @@ -765,12 +905,16 @@ mod tests { assert_eq!(builds.get(), 1, "the first request should build"); for attempt in 2..=4_u32 { - sandbox - .initialize(|| -> Result { + let again = apps + .get_or_build("trusted_server_config", || -> Result { builds.set(builds.get() + 1); panic!("request {attempt} must not rebuild a retained application") }) .expect("a retained application should be reused"); + assert!( + Rc::ptr_eq(&first, &again), + "request {attempt} should be handed the application built first" + ); } assert_eq!( @@ -778,21 +922,17 @@ mod tests { 1, "four requests should cost exactly one build; that is the whole point" ); - assert_eq!( - sandbox.initialization_attempts(), - 1, - "the framework counter should agree" - ); + assert_eq!(apps.build_attempts(), 1, "the build counter should agree"); } #[test] fn a_failed_build_is_retried_and_a_later_success_is_retained() { - let mut sandbox = Sandbox::default(); + let apps = RetainedApps::default(); let builds = Cell::new(0_u32); // Request 1: the build fails. The error payload stands in for the // error router `build_app_with_state` returns when state is `None`. - let failed = sandbox.initialize(|| { + let failed = apps.get_or_build("trusted_server_config", || { builds.set(builds.get() + 1); Err::("error router") }); @@ -801,31 +941,23 @@ mod tests { Some("error router"), "a failed build must be handed back for this request only" ); - assert!( - sandbox.state().is_none(), - "a failed build must not be retained" - ); + assert_eq!(apps.retained(), 0, "a failed build must not be retained"); // Request 2: the retry succeeds. Only reachable because the failure // was not retained. - sandbox - .initialize(|| { - builds.set(builds.get() + 1); - Ok::<_, &str>(test_retained()) - }) - .expect("the retry should succeed"); - assert!( - sandbox.state().is_some(), + apps.get_or_build("trusted_server_config", || { + builds.set(builds.get() + 1); + Ok::<_, &str>(test_retained()) + }) + .expect("the retry should succeed"); + assert_eq!( + apps.retained(), + 1, "the recovered application should be retained" ); - // Request 3: recovery is durable — no further build. - sandbox - .initialize(|| -> Result { - builds.set(builds.get() + 1); - panic!("a recovered application must not be rebuilt") - }) - .expect("request 3 should reuse the recovered app"); + // Request 3: recovery is durable, so no further build. + retained_for(&apps, "trusted_server_config"); assert_eq!( builds.get(), @@ -833,12 +965,129 @@ mod tests { "one failed build plus one successful build, then reuse" ); assert_eq!( - sandbox.initialization_attempts(), + apps.build_attempts(), 2, "both attempts should be counted so a retry loop stays visible" ); } + #[test] + fn an_application_answers_only_for_the_key_it_was_built_from() { + let apps = RetainedApps::default(); + + let first = build_for(&apps, "a.example"); + let second = build_for(&apps, "b.example"); + + assert!( + !Rc::ptr_eq(&first, &second), + "a second publisher must be built an application of its own" + ); + assert_eq!(apps.build_attempts(), 2, "two blobs should cost two builds"); + + for (key, built) in [ + ("b.example", &second), + ("a.example", &first), + ("a.example", &first), + ("b.example", &second), + ] { + assert!( + Rc::ptr_eq(&retained_for(&apps, key), built), + "`{key}` must be answered by the application built from its own blob" + ); + } + assert_eq!( + apps.build_attempts(), + 2, + "serving both in any order should build nothing more" + ); + } + + #[test] + fn a_failed_build_for_one_key_leaves_another_s_application_retained() { + let apps = RetainedApps::default(); + let first = build_for(&apps, "a.example"); + + let failed = apps.get_or_build("b.example", || Err::("error router")); + assert_eq!( + failed.err(), + Some("error router"), + "the second publisher's failed build should be handed back" + ); + + assert!( + Rc::ptr_eq(&retained_for(&apps, "a.example"), &first), + "the first publisher's application should still be retained" + ); + assert_eq!( + apps.retained(), + 1, + "the failed build should not have been retained under either key" + ); + } + + #[test] + fn the_least_recently_served_application_is_dropped_past_the_bound() { + let apps = RetainedApps::default(); + let key = |index: usize| format!("{index}.example"); + + for index in 0..MAX_RETAINED_APPS { + build_for(&apps, &key(index)); + } + assert_eq!( + apps.retained(), + MAX_RETAINED_APPS, + "should keep every application up to the bound" + ); + + // Serving the oldest makes the second oldest the one to drop. + retained_for(&apps, &key(0)); + build_for(&apps, "one-more.example"); + + assert_eq!( + apps.retained(), + MAX_RETAINED_APPS, + "should never keep more than the bound" + ); + retained_for(&apps, &key(0)); + retained_for(&apps, "one-more.example"); + for index in 2..MAX_RETAINED_APPS { + retained_for(&apps, &key(index)); + } + + let rebuilt = Cell::new(false); + apps.get_or_build(&key(1), || { + rebuilt.set(true); + Ok::<_, &str>(test_retained()) + }) + .expect("the dropped application should build again"); + assert!( + rebuilt.get(), + "the least recently served application should have been dropped" + ); + } + + #[test] + fn captured_counters_report_the_builds_the_sandbox_attempted() { + let sandbox = holding_sandbox(); + let mut settings = Settings::default(); + settings.debug.sandbox_metrics_enabled = true; + + let apps = sandbox.state().expect("should hold its applications"); + build_for(apps, "a.example"); + build_for(apps, "b.example"); + retained_for(apps, "a.example"); + + assert_eq!( + SandboxCounters::capture(&sandbox, 3, "req-3", Some(&settings)), + Some(SandboxCounters { + ordinal: 3, + builds: 2, + request_id: "req-3".to_owned(), + }), + "should report a build for each blob and none for a reuse" + ); + } + /// Drives the real reporting path: the same `observe` that `serve_loop` /// wraps every callback in. A build performed by the FINAL callback must /// appear in the retirement snapshot; reading the attempt count on entry @@ -853,15 +1102,17 @@ mod tests { assert_eq!( counters.attempts(), 0, - "a callback that never initializes should report no attempts" + "a callback that never builds should report no attempts" ); // Callback 2 succeeds. The build happens DURING this callback, so it // only shows up if the count is read on the way out. counters.observe(&mut sandbox, |sandbox| { - sandbox - .initialize(|| Ok::<_, &str>(test_retained())) - .expect("the build should succeed"); + hold_applications(sandbox); + build_for( + sandbox.state().expect("should hold its applications"), + "trusted_server_config", + ); }); assert_eq!( counters.attempts(), @@ -875,11 +1126,17 @@ mod tests { let mut sandbox = Sandbox::default(); let mut counters = RetirementCounters::default(); - // The application state is not retained, but the attempt itself stays - // in `initialization_attempts()` until the sandbox is dropped. What a - // stale snapshot loses is the report, not the count. + // The application is not retained, but the attempt itself stays in + // the holder's count until the sandbox is dropped. What a stale + // snapshot loses is the report, not the count. counters.observe(&mut sandbox, |sandbox| { - let failed = sandbox.initialize(|| Err::("error router")); + hold_applications(sandbox); + let failed = sandbox + .state() + .expect("should hold its applications") + .get_or_build("trusted_server_config", || { + Err::("error router") + }); assert_eq!( failed.err(), Some("error router"), @@ -892,8 +1149,12 @@ mod tests { 1, "a failed build on the final callback must still be reported" ); - assert!( - sandbox.state().is_none(), + assert_eq!( + sandbox + .state() + .expect("should hold its applications") + .retained(), + 0, "a failed build must not be retained" ); } @@ -954,7 +1215,7 @@ mod tests { ); } - // Request ordinals and build attempts are counted by + // Request ordinals are counted by // `edgezero_adapter_fastly::lifecycle::Sandbox` itself, incremented in its // private per-callback hook. They are exercised through `serve_custom` / // `run_custom` at runtime and covered by the framework's own tests, so diff --git a/crates/trusted-server-adapter-fastly/src/template_cache.rs b/crates/trusted-server-adapter-fastly/src/template_cache.rs index e6f3ba2c08..58fbc2714d 100644 --- a/crates/trusted-server-adapter-fastly/src/template_cache.rs +++ b/crates/trusted-server-adapter-fastly/src/template_cache.rs @@ -18,8 +18,7 @@ use fastly::cache::core::{CacheKey, Found, Transaction}; use std::io::Write as _; use std::time::Duration; use trusted_server_core::platform::{ - PlatformTemplateCache, PlatformTemplateCacheReservation, - TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, TemplateCacheError, TemplateCacheKey, + PlatformTemplateCache, PlatformTemplateCacheReservation, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; @@ -279,16 +278,6 @@ impl PlatformTemplateCache for FastlyTemplateCache { fastly::http::purge::purge_surrogate_key(&key.url_surrogate_key()) .map_err(|e| backend_error(format!("purging invalid template failed: {e:?}"))) } - - async fn purge_url_surrogate_key(&self, key: &str) -> Result<(), TemplateCacheError> { - fastly::http::purge::purge_surrogate_key(key) - .map_err(|e| backend_error(format!("purging surrogate key {key} failed: {e:?}"))) - } - - async fn purge_all(&self) -> Result<(), TemplateCacheError> { - fastly::http::purge::purge_surrogate_key(TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY) - .map_err(|e| backend_error(format!("purging templates failed: {e:?}"))) - } } #[cfg(test)] @@ -296,7 +285,9 @@ mod tests { use super::*; use std::io; use trusted_server_core::creative_opportunities::AssemblyMode; - use trusted_server_core::platform::TEMPLATE_SCHEMA_VERSION; + use trusted_server_core::platform::{ + TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, TEMPLATE_SCHEMA_VERSION, + }; struct FailingReader { returned_prefix: bool, @@ -552,9 +543,9 @@ mod tests { } #[test] - fn purge_all_clears_stored_templates() { - // The rollback lever. Without this, backing out a bad template means waiting - // for the TTL. + fn purging_the_key_every_template_carries_clears_stored_templates() { + // The rollback lever, which `ts cache purge --all` asks the platform to pull. + // Without it, backing out a bad template means waiting for the TTL. let cache = cache(); let key = key("https://example.com/purge"); let body = b"template".to_vec(); @@ -562,7 +553,8 @@ mod tests { .expect("should store"); run(cache.get(&key)).expect("should be present before purge"); - run(cache.purge_all()).expect("should purge"); + fastly::http::purge::purge_surrogate_key(TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY) + .expect("should purge"); assert!( run(cache.get(&key)).is_err(), @@ -570,6 +562,36 @@ mod tests { ); } + #[test] + fn purging_a_pages_reader_key_clears_that_pages_template() { + // The key `ts cache purge --page` asks the platform to purge. The + // helper gives every key one reader page, so each gets its own here. + let cache = cache(); + let mut purged = key("https://origin.example.com/purge-one"); + purged.request_path = "/purge-one".to_string(); + let mut kept = key("https://origin.example.com/keep-one"); + kept.request_path = "/keep-one".to_string(); + assert_ne!( + purged.reader_url_surrogate_key(), + kept.reader_url_surrogate_key(), + "the two pages should have reader keys of their own" + ); + for stored in [&purged, &kept] { + let body = b"template".to_vec(); + run(cache.put(stored, &metadata_for(&body), body, Duration::from_secs(60))) + .expect("should store"); + } + + fastly::http::purge::purge_surrogate_key(&purged.reader_url_surrogate_key()) + .expect("should purge"); + + assert!( + run(cache.get(&purged)).is_err(), + "should clear the page whose key was purged" + ); + run(cache.get(&kept)).expect("should leave another page's template in place"); + } + #[test] fn a_second_put_on_a_fresh_entry_is_a_no_op() { // Exercises the `must_insert_or_update` early return: a concurrent writer diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index 504dbd9fba..31363181cb 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -20,21 +20,18 @@ const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; -/// Build the configured auction telemetry sink. +/// Build the auction telemetry sink, which sends only when `[analytics]` +/// selects the analytics module. #[must_use] pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - if settings.tinybird.enabled { - Arc::new(FastlyTinybirdAuctionTelemetrySink::new( - settings.tinybird.clone(), - )) - } else { - Arc::new(NoopAuctionTelemetrySink) + match settings.analytics_module() { + Some(telemetry) => Arc::new(FastlyTinybirdAuctionTelemetrySink::new(telemetry.clone())), + None => Arc::new(NoopAuctionTelemetrySink), } } #[derive(Debug, Clone)] struct FastlyTinybirdAuctionTelemetrySink { - enabled: bool, target: TinybirdEventsTarget, } @@ -57,7 +54,7 @@ impl TinybirdEventsTarget { dataset: config.auction_dataset, append_token: config .auction_token_secret - .expect("should contain a resolved Tinybird auction token when enabled"), + .expect("should contain a resolved Tinybird auction token when selected"), uri, backend_spec, max_body_bytes: config.max_body_bytes, @@ -67,9 +64,7 @@ impl TinybirdEventsTarget { impl FastlyTinybirdAuctionTelemetrySink { fn new(config: TinybirdSettings) -> Self { - let enabled = config.enabled; Self { - enabled, target: TinybirdEventsTarget::from_config(config), } } @@ -149,16 +144,12 @@ impl FastlyTinybirdAuctionTelemetrySink { #[async_trait::async_trait(?Send)] impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { - fn is_enabled(&self) -> bool { - self.enabled - } - async fn emit_auction_events( &self, services: &RuntimeServices, batch: AuctionEventBatch, ) -> Result<(), Report> { - if !self.enabled || batch.is_empty() { + if batch.is_empty() { return Ok(()); } @@ -364,10 +355,12 @@ mod tests { struct NoopGeo; + #[async_trait::async_trait(?Send)] impl PlatformGeo for NoopGeo { - fn lookup( + async fn lookup( &self, _client_ip: Option, + _services: &trusted_server_core::platform::RuntimeServices, ) -> Result, Report> { Ok(None) } @@ -428,9 +421,8 @@ mod tests { .build() } - fn enabled_config() -> TinybirdSettings { + fn tinybird_settings() -> TinybirdSettings { TinybirdSettings { - enabled: true, api_host: "api.us-east.aws.tinybird.co".to_owned(), secret_store: None, auction_dataset: "auction_events_raw".to_owned(), @@ -477,7 +469,7 @@ mod tests { Arc::clone(&http_client), HashMap::new(), ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); futures::executor::block_on( sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), @@ -525,7 +517,7 @@ mod tests { } #[test] - fn disabled_sink_does_not_dispatch() { + fn settings_selecting_no_analytics_module_build_a_sink_that_sends_nothing() { let backend = Arc::new(RecordingBackend::default()); let http_client = Arc::new(RecordingHttpClient::default()); let services = services( @@ -533,18 +525,17 @@ mod tests { Arc::clone(&http_client), HashMap::new(), ); - let mut config = enabled_config(); - config.enabled = false; - let sink = FastlyTinybirdAuctionTelemetrySink::new(config); + let sink = auction_sink_from_settings(&Settings::default()); + assert!(!sink.is_enabled(), "should build the no-op sink"); futures::executor::block_on( sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), ) - .expect("should ignore disabled sink"); + .expect("the no-op sink should accept the batch"); assert!( backend.specs.lock().expect("should lock specs").is_empty(), - "should not ensure backend when disabled" + "should not ensure a backend when nothing is selected" ); assert!( http_client @@ -552,10 +543,23 @@ mod tests { .lock() .expect("should lock recorded requests") .is_empty(), - "should not send when disabled" + "should not send when nothing is selected" ); } + #[test] + fn settings_selecting_the_analytics_module_build_a_sink_that_sends() { + let mut settings = Settings::default(); + settings.analytics = Some(trusted_server_core::settings::AnalyticsConfig { + module: Some("tinybird".to_owned()), + tinybird: Some(tinybird_settings()), + }); + + let sink = auction_sink_from_settings(&settings); + + assert!(sink.is_enabled(), "should build the sink that sends"); + } + #[test] fn empty_batch_does_not_dispatch() { let backend = Arc::new(RecordingBackend::default()); @@ -565,7 +569,7 @@ mod tests { Arc::clone(&http_client), HashMap::new(), ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); futures::executor::block_on( sink.emit_auction_events(&services, AuctionEventBatch::new(Vec::new())), @@ -598,7 +602,7 @@ mod tests { b"append-token".to_vec(), )]), ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + let sink = FastlyTinybirdAuctionTelemetrySink::new(tinybird_settings()); let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; let result = futures::executor::block_on( diff --git a/crates/trusted-server-adapter-fastly/src/value_cache.rs b/crates/trusted-server-adapter-fastly/src/value_cache.rs new file mode 100644 index 0000000000..df3376754c --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/value_cache.rs @@ -0,0 +1,280 @@ +//! Fastly's cache as the platform value cache. +//! +//! An entry is held at the point of presence that wrote it and shared by every +//! instance there, which is where a visitor's next request, and a second lookup +//! in the same request, arrive. Nothing needs creating or linking on the +//! service. +//! +//! A transaction lookup that finds nothing obliges this instance to fill the +//! key and makes every other instance that looks the key up wait for it, so a +//! loading cache over this store loads a missing key once per location rather +//! than once per instance. + +use std::io::{Read as _, Write as _}; +use std::time::Duration; + +use bytes::Bytes; +use fastly::cache::core::{CacheKey, Found, Transaction}; +use trusted_server_core::platform::{PlatformValueCache, ValueLookup, ValueReservation}; + +/// Values held in Fastly's cache. +#[derive(Debug, Default)] +pub struct FastlyValueCache; + +/// The surrogate key an entry is written with, so [`PlatformValueCache::remove`] +/// can purge it. A 64-bit FNV-1a hash of the key, in hex, because a cache key +/// may hold characters a surrogate key may not. +fn surrogate_key(key: &str) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in key.bytes() { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("vc-{hash:016x}") +} + +fn cache_key(key: &str) -> CacheKey { + CacheKey::from(key.as_bytes().to_vec()) +} + +/// The bytes of a fresh entry, or `None` for a stale or unreadable one. +fn read(found: &Found) -> Option { + if found.is_stale() { + return None; + } + let mut body = Vec::new(); + found + .to_stream() + .inspect_err(|error| log::debug!("value cache body could not be opened: {error:?}")) + .ok()? + .read_to_end(&mut body) + .inspect_err(|error| log::debug!("value cache body could not be read: {error}")) + .ok()?; + Some(Bytes::from(body)) +} + +#[async_trait::async_trait(?Send)] +impl PlatformValueCache for FastlyValueCache { + async fn get(&self, key: &str) -> Option { + // A plain lookup rather than a transaction, because a caller that only + // reads must take no obligation it will not fill. + let found = fastly::cache::core::lookup(cache_key(key)) + .execute() + .inspect_err(|error| log::debug!("value cache lookup failed: {error:?}")) + .ok()??; + read(&found) + } + + async fn put(&self, key: &str, value: Bytes, lifetime: Duration) { + let mut writer = match fastly::cache::core::insert(cache_key(key), lifetime) + .surrogate_keys([surrogate_key(key).as_str()]) + .known_length(value.len() as u64) + .execute() + { + Ok(writer) => writer, + Err(error) => { + log::debug!("value cache insert could not begin: {error:?}"); + return; + } + }; + if let Err(error) = writer.write_all(&value) { + log::debug!("value cache insert could not be written: {error}"); + return; + } + if let Err(error) = writer.finish() { + log::debug!("value cache insert could not be finished: {error:?}"); + } + } + + async fn lookup_or_reserve(&self, key: &str) -> ValueLookup { + let transaction = match Transaction::lookup(cache_key(key)).execute() { + Ok(transaction) => transaction, + Err(error) => { + log::debug!("value cache transaction lookup failed: {error:?}"); + return ValueLookup::Miss; + } + }; + // Asked before `found`, because a stale entry comes with both, and + // reading it would leave the obligation unfilled and every other + // instance waiting until it times out. + if transaction.must_insert_or_update() { + return ValueLookup::Reserved(Box::new(FastlyReservation { + transaction: Some(transaction), + surrogate_key: surrogate_key(key), + })); + } + match transaction.found().as_ref().and_then(read) { + Some(value) => ValueLookup::Hit(value), + None => ValueLookup::Miss, + } + } + + async fn remove(&self, key: &str) { + if let Err(error) = fastly::http::purge::purge_surrogate_key(&surrogate_key(key)) { + log::debug!("value cache entry could not be purged: {error:?}"); + } + } +} + +/// This instance's obligation to fill a key every other instance waits for. +struct FastlyReservation { + transaction: Option, + surrogate_key: String, +} + +impl ValueReservation for FastlyReservation { + fn fill(mut self: Box, value: Bytes, lifetime: Duration) { + let Some(transaction) = self.transaction.take() else { + return; + }; + // `insert` takes the transaction, so a write that fails part way can no + // longer be canceled. The entry's own format checks its length when it + // is read, so a short entry is a miss rather than a wrong value. + let mut writer = match transaction + .insert(lifetime) + .surrogate_keys([self.surrogate_key.as_str()]) + .known_length(value.len() as u64) + .execute() + { + Ok(writer) => writer, + Err(error) => { + log::debug!("value cache reserved insert could not begin: {error:?}"); + return; + } + }; + if let Err(error) = writer.write_all(&value) { + log::debug!("value cache reserved insert could not be written: {error}"); + return; + } + if let Err(error) = writer.finish() { + log::debug!("value cache reserved insert could not be finished: {error:?}"); + } + } +} + +impl Drop for FastlyReservation { + fn drop(&mut self) { + // Unfilled, so the obligation passes to an instance waiting on it. + if let Some(transaction) = self.transaction.take() + && let Err(error) = transaction.cancel_insert_or_update() + { + log::debug!("value cache reservation could not be released: {error:?}"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(future: F) -> F::Output { + futures::executor::block_on(future) + } + + #[test] + fn a_value_put_is_read_back() { + let cache = FastlyValueCache; + run(cache.put( + "test/value-cache/read-back", + Bytes::from_static(b"answer"), + Duration::from_secs(60), + )); + + assert_eq!( + run(cache.get("test/value-cache/read-back")).as_deref(), + Some(&b"answer"[..]), + "a value put should be read back" + ); + } + + #[test] + fn a_key_never_put_is_a_miss() { + assert_eq!( + run(FastlyValueCache.get("test/value-cache/never-put")), + None, + "a key never put should be a miss" + ); + } + + #[test] + fn a_missing_key_is_reserved_and_its_fill_is_then_a_hit() { + let key = "test/value-cache/reserve-then-fill"; + let ValueLookup::Reserved(reservation) = run(FastlyValueCache.lookup_or_reserve(key)) + else { + panic!("a missing key should oblige this caller to fill it"); + }; + reservation.fill(Bytes::from_static(b"filled"), Duration::from_secs(60)); + + match run(FastlyValueCache.lookup_or_reserve(key)) { + ValueLookup::Hit(value) => assert_eq!(value, Bytes::from_static(b"filled")), + other => panic!("the filled key should be a hit, got {other:?}"), + } + } + + #[test] + fn a_dropped_reservation_passes_the_obligation_on() { + let key = "test/value-cache/dropped-reservation"; + let ValueLookup::Reserved(first) = run(FastlyValueCache.lookup_or_reserve(key)) else { + panic!("a missing key should be reserved"); + }; + drop(first); + + let ValueLookup::Reserved(second) = run(FastlyValueCache.lookup_or_reserve(key)) else { + panic!("an unfilled reservation should be given to the next caller"); + }; + second.fill(Bytes::from_static(b"second"), Duration::from_secs(60)); + assert_eq!( + run(FastlyValueCache.get(key)).as_deref(), + Some(&b"second"[..]) + ); + } + + #[test] + fn a_stale_entry_is_reserved_rather_than_served() { + // Present and stale, which a revalidation window keeps readable after + // its lifetime has ended. + let key = "test/value-cache/stale"; + let mut writer = fastly::cache::core::insert(cache_key(key), Duration::from_secs(0)) + .stale_while_revalidate(Duration::from_secs(60)) + .execute() + .expect("should begin the insert"); + writer.write_all(b"stale").expect("should write the body"); + writer.finish().expect("should finish the insert"); + + assert_eq!( + run(FastlyValueCache.get(key)), + None, + "a plain read should not serve it" + ); + match run(FastlyValueCache.lookup_or_reserve(key)) { + ValueLookup::Reserved(reservation) => { + reservation.fill(Bytes::from_static(b"fresh"), Duration::from_secs(60)); + } + other => panic!("a stale entry should be reserved for a fresh load, got {other:?}"), + } + assert_eq!( + run(FastlyValueCache.get(key)).as_deref(), + Some(&b"fresh"[..]) + ); + } + + #[test] + fn remove_purges_the_entry() { + let key = "test/value-cache/remove"; + run(FastlyValueCache.put(key, Bytes::from_static(b"gone"), Duration::from_secs(60))); + assert!(run(FastlyValueCache.get(key)).is_some()); + + run(FastlyValueCache.remove(key)); + + assert_eq!(run(FastlyValueCache.get(key)), None); + } + + #[test] + fn the_surrogate_key_is_short_and_safe() { + let key = surrogate_key("ts.example.com/page/ts template; with ^ odd | chars"); + assert!(key.starts_with("vc-")); + assert_eq!(key.len(), 3 + 16); + assert!(key[3..].bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_ne!(surrogate_key("a"), surrogate_key("b")); + } +} diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 81eeb94307..2bdab4f9d5 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -32,6 +32,12 @@ futures = { workspace = true } http-body-util = { workspace = true } log = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-modules = { workspace = true } +trusted-server-permission-signal-gpc = { workspace = true } +trusted-server-permission-signal-gpp = { workspace = true } +trusted-server-permission-signal-mtm = { workspace = true } +trusted-server-permission-signal-tcf = { workspace = true } +trusted-server-permission-signal-us-privacy = { workspace = true } trusted-server-js = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -41,4 +47,5 @@ spin-sdk = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/trusted-server-adapter-spin/spin.toml b/crates/trusted-server-adapter-spin/spin.toml index 7055f9563d..d1f3f26593 100644 --- a/crates/trusted-server-adapter-spin/spin.toml +++ b/crates/trusted-server-adapter-spin/spin.toml @@ -19,15 +19,13 @@ version = "0.1.0" # each private signing key. Public signing metadata remains in the KV store. [variables] # These declared variables match the example config's secret key names. Regenerate -# or extend them for deployment-specific keys, including handler key names such as -# `admin_password` or `api_handler_password`. Replace the empty defaults with values +# or extend them for deployment-specific keys. Replace the empty defaults with values # supplied by the deployment's secret provider; never commit values here. v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = { default = "", secret = true } v_trusted_x5fserver_x5fsecrets_v_trusted_x5fclient_x5fip_x5fshared_x5fsecret = { default = "", secret = true } v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = { default = "", secret = true } v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken = { default = "", secret = true } v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken = { default = "", secret = true } -v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword = { default = "", secret = true } [[trigger.http]] route = "/..." @@ -49,7 +47,6 @@ v_trusted_x5fserver_x5fsecrets_v_trusted_x5fclient_x5fip_x5fshared_x5fsecret = " v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = "{{ v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase }}" v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken = "{{ v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken }}" v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken = "{{ v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken }}" -v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword = "{{ v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword }}" [component.trusted-server.build] command = "cargo build --target wasm32-wasip1 --release -p trusted-server-adapter-spin --features spin" diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 804a23b620..448aaf3c3e 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -12,22 +12,27 @@ use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; use edgezero_core::router::RouterService; use error_stack::Report; +use trusted_server_core::attestation::{PlatformIdentity, handle_prefix, prefix_routes}; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ - AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan_with, }; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::closed_paths::closed_path_response; #[cfg(all(feature = "spin", target_arch = "wasm32"))] -use trusted_server_core::config_payload::settings_from_config_blob; +use trusted_server_core::config_payload::settings_from_config_blob_with; use trusted_server_core::ec::EcContext; -use trusted_server_core::ec::admin::{ - admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, - deny_admin_diagnostic_fallback, handle_admin_eids_lookup, -}; -use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::ec::module::{EdgeCookieModule, build_reusable_module}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; -use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; +use trusted_server_core::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH, handle_config}; +use trusted_server_core::inspect::data::{DATA_PAGE_PATH, handle_data}; +use trusted_server_core::inspect::permissions::{ + PERMISSIONS_JSON_PATH, PERMISSIONS_PAGE_PATH, handle_permissions, +}; +use trusted_server_core::integrations::{ + IntegrationBuilder, IntegrationRegistry, ProxyDispatchInput, +}; use trusted_server_core::platform::RuntimeServices; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::platform::{PlatformConfigStore, StoreName}; @@ -36,7 +41,7 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, + AppContext, AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; @@ -48,7 +53,7 @@ use trusted_server_core::settings::Settings; use trusted_server_core::settings_data::{default_config_key, default_secret_store_name}; use crate::middleware::{ - AuthMiddleware, FinalizeResponseMiddleware, NormalizeMiddleware, SanitizeRequestMiddleware, + FinalizeResponseMiddleware, NormalizeMiddleware, SanitizeRequestMiddleware, }; use crate::platform::build_runtime_services; #[cfg(all(feature = "spin", target_arch = "wasm32"))] @@ -67,15 +72,41 @@ pub struct AppState { settings: Arc, orchestrator: Arc, registry: Arc, + /// The Edge Cookie module `[ec] module` selects, resolved once here. + /// + /// This adapter runs a fresh instance per request and resolving reads no + /// request data, so the selection is resolved when the state is built + /// and handed to every request through + /// [`RuntimeServices::resolved_ec_module`](trusted_server_core::platform::RuntimeServices::resolved_ec_module), + /// rather than resolved again on the request path. + /// `None` for a deployment that selects no module. + ec_module: Option>, + /// The permission signal modules `[permission-signal] modules` selects + /// from the scheme crates this adapter links, in the order they run. + /// Selected once here so a name no crate answers to fails startup rather + /// than the first request, and handed to every request's services. + permission_signal_modules: + Arc<[Arc]>, + /// Services a caller supplied for every request, rather than services built + /// from the request context. `None` in a deployment. services: Option, + /// The loading caches the `[cache.]` tables select from the linked + /// builders, taken when this state is built and handed to every request's + /// services. + loading_caches: Arc, } /// Build the application state, loading settings and constructing all per-application components. /// +/// Settings are read from the platform config store at run time, the same way +/// the Fastly and Axum adapters read them, so an operator publishes one with +/// `ts config push` and the deployed component picks it up. +/// /// # Errors /// -/// Returns an error when settings, the auction orchestrator, or the integration -/// registry fail to initialise. +/// Returns an error when the config store holds no readable app config, or when +/// settings, the auction orchestrator, or the integration registry fail to +/// initialize. fn build_state() -> Result, Report> { let settings = load_startup_settings()?; build_state_with_settings(settings) @@ -104,7 +135,14 @@ fn load_startup_settings() -> Result> { .attach(error.to_string()) })?; let secret_store = SpinSecretStoreAdapter; - settings_from_config_blob(&raw_envelope, &secret_store, &default_secret_store_name()) + // The settings are validated as they load, against the same builders the + // state is built with. + settings_from_config_blob_with( + &raw_envelope, + &secret_store, + &default_secret_store_name(), + &trusted_server_modules::builders(), + ) } #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] @@ -119,39 +157,174 @@ fn load_startup_settings() -> Result> { /// /// # Errors /// -/// Returns an error when the auction orchestrator or the integration -/// registry fail to initialise. +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, or when the auction orchestrator or the integration registry +/// fail to initialize. fn build_state_with_settings( settings: Settings, ) -> Result, Report> { - build_state_with_services(settings, None) + build_state_with_registrations(settings, &[]) } +/// Build the application state from explicit settings, composing the modules +/// a stock build ships with the externally supplied builders in +/// `integrations`. +/// +/// A deployment that ships a vendor crate calls this to add that crate's +/// integration builder without the adapter naming the vendor. Auction +/// providers come from the compiled auction plan, as they do without any +/// external builders. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this adapter, when the auction plan does not compile or cannot run on this +/// adapter, or when the auction orchestrator or the integration registry fail +/// to initialize, which includes two builders claiming the same integration +/// id. +pub fn build_state_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], +) -> Result, Report> { + build_state_with_registrations_and_services(settings, integrations, None) +} + +/// Build the application state with the services every request will use, +/// rather than services built per request from the request context. fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { - let plan = Arc::new(compile_auction_plan(&settings)?); + build_state_with_registrations_and_services(settings, &[], services) +} + +fn build_state_with_registrations_and_services( + settings: Settings, + integrations: &[IntegrationBuilder], + services: Option, +) -> Result, Report> { + // The modules a stock build ships come first and the deployment's own + // follow, which is the order their hooks run in. + let integrations = trusted_server_modules::builders_with(integrations); + let integrations = integrations.as_slice(); + + // The plan is compiled with the integrations this adapter was given, so an + // `[ad-server]` or `[demand]` name one of their builders supplies resolves + // here. Compiling without them would drop the implementation and report the + // name as one no builder registers. + let plan = Arc::new(compile_auction_plan_with(&settings, integrations)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; - let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; - let registry = IntegrationRegistry::with_plan(&settings, plan)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan))?; + let registry = IntegrationRegistry::with_plan_and_registrations(&settings, plan, integrations)? + .with_platform(PlatformIdentity::named("spin")); + + // Composition root: resolve the module selection once, before any request + // is served, so a selection this adapter can never supply fails here rather + // than on the first request. Keeping what the resolution produced is what + // stops the request path resolving the same settings again. The registry is + // built first because a module can supply the vendor Edge Cookie module + // the selector names, and resolving without it would reject a selection + // this deployment can in fact satisfy. A caller supplying its own + // `RuntimeServices` may have resolved one already, and that one comes first + // because it is what the request path will see. This adapter supplies no + // host signals, so that argument stays `None` until it does. + let ec_module = build_reusable_module( + &settings.ec, + None, + services + .as_ref() + .and_then(RuntimeServices::resolved_ec_module) + .or_else(|| registry.ec_module()), + )?; + let permission_signal_modules = + trusted_server_core::permission_signal::build_permission_signal_modules( + &settings, + &shipped_signal_modules(), + )?; + let loading_caches = + trusted_server_core::integrations::loading_caches_from(&settings, integrations)?; Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), + ec_module, + permission_signal_modules, services, + loading_caches, })) } +/// The permission signal modules this adapter links, in the order they run +/// when configuration names none. Global Privacy Control is first because it +/// is a browser setting with no interface of its own, and the three that +/// carry a choice someone made through an interface follow, so an answer +/// given at a prompt amends the header the visitor arrived with. +/// +/// Core supplies no module of its own, so this is where a deployment's +/// schemes are decided. A scheme is added by linking its crate here, and a +/// scheme core has never heard of plugs in the same way. +fn shipped_signal_modules() +-> Vec> { + vec![ + Arc::new(trusted_server_permission_signal_gpc::GpcModule::new()), + Arc::new(trusted_server_permission_signal_gpp::GppSaleOptOutModule::new()), + Arc::new(trusted_server_permission_signal_us_privacy::UsPrivacyModule::new()), + Arc::new(trusted_server_permission_signal_tcf::TcfModule::new()), + Arc::new(trusted_server_permission_signal_mtm::MtmModule::new()), + ] +} + impl AppState { + /// Build per-request [`RuntimeServices`], carrying the Edge Cookie module + /// the composition root already resolved and applying the module-supplied + /// geo, Edge Cookie and device modules selected by `[geo]`, `[ec]` and + /// `[device] module`. + /// + /// No Edge Cookie module is carried when the composition root found + /// nothing safe to keep, so the request path resolves the selection for + /// itself. + /// + /// For geo, unset and `"none"` both resolve nothing, so no client IP + /// reaches a host geo service. `"platform"` opts in to this adapter's own + /// lookup, and any other key names an integration module that declares a + /// geo module. Identity and device are applied the same way when a module + /// supplies them. fn services_for_request(&self, ctx: &RequestContext) -> RuntimeServices { - self.services + let mut services = self + .services .clone() - .unwrap_or_else(|| build_runtime_services(ctx)) + .unwrap_or_else(|| { + build_runtime_services(ctx, &self.settings, &self.permission_signal_modules) + .with_loading_caches(Arc::clone(&self.loading_caches)) + }) + .with_resolved_ec_module(self.ec_module.clone()); + if let Some(module) = self.registry.geo_module() { + services = services.with_geo(module); + } + if let Some(module) = self.registry.ec_module() { + services = services.with_ec_module(module); + } + if let Some(module) = self.registry.device_module() { + services = services.with_device_module(module); + } + // The publisher this request resolved to names every key in a shared + // store and every name the browser holds. + let prefixes = self.registry.prefixes( + &self.settings, + trusted_server_core::module_context::ModuleRequest::of( + ctx.request(), + services.client_info(), + ), + ); + services.with_prefixes(prefixes) } } +// --------------------------------------------------------------------------- +// Per-request RuntimeServices +// --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- // Publisher response helper // --------------------------------------------------------------------------- @@ -205,34 +378,22 @@ fn publisher_fallback_methods() -> [Method; 7] { // reaches the publisher origin rather than a router-level 405. // // The EC API routes that the Fastly entry point registers — POST -// `/_ts/api/v1/batch-sync`, GET/OPTIONS `/_ts/api/v1/identify` — are +// `/_ts/api/v1/batch-sync`, GET/OPTIONS `/_ts/api/v1/identify`, POST +// `/_ts/api/v1/ec/resolve`, are // intentionally absent here, matching the Axum and Cloudflare adapters: those // handlers require a platform KV `ec_store` (and, for batch-sync, a partner // registry and rate limiter) that the portability adapters do not yet wire. // On Spin these paths fall through to the publisher/integration fallback, // identical to the other non-Fastly adapters. -const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ - Method::GET, - Method::POST, - Method::HEAD, - Method::OPTIONS, - Method::PUT, - Method::PATCH, - Method::DELETE, -]; - -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 17] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 14] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), - ("/_ts/admin/keys/rotate", &[Method::POST]), - ("/_ts/admin/keys/deactivate", &[Method::POST]), - ("/_ts/admin/ec", &[Method::GET]), - ("/_ts/admin/ec/{id}", &[Method::GET]), - ("/_ts/admin/eids", &[Method::GET]), - ("/_ts/admin/cache/purge", LEGACY_ADMIN_DENY_METHODS), - ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), - ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), + (PERMISSIONS_PAGE_PATH, &[Method::GET]), + (PERMISSIONS_JSON_PATH, &[Method::GET]), + (CONFIG_PAGE_PATH, &[Method::GET]), + (CONFIG_JSON_PATH, &[Method::GET]), + (DATA_PAGE_PATH, &[Method::GET]), ("/auction", &[Method::POST]), (PAGE_BIDS_PATH, &[Method::GET, Method::OPTIONS]), (PAGE_BIDS_LEGACY_PATH, &[Method::GET, Method::OPTIONS]), @@ -409,57 +570,31 @@ fn health_response() -> Response { /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, /// `/_ts/page-bids`, and the publisher fallback). /// -/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction -/// Unknown, which fails the auction consent gate closed even for consented -/// users. Spin's platform geo is a no-op, so jurisdiction stays Unknown unless -/// the request carries TCF consent. A malformed consent string is logged and -/// falls back to the default (fail-closed) context rather than being silently -/// swallowed. -fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { - let geo_info = services - .geo() - .lookup(services.client_info().client_ip) - .unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }); - EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) - .unwrap_or_else(|e| { - log::warn!("EC context read failed: {e:?}"); - EcContext::default() - }) -} - -fn cache_purge_not_supported() -> Response { - let body = edgezero_core::body::Body::from( - "Template cache purge is not supported on Spin.\n\ - Use the Fastly adapter (via Viceroy or deployed) to purge.\n", - ); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::NOT_IMPLEMENTED; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - -fn admin_key_management_not_supported() -> Response { - let body = edgezero_core::body::Body::from( - "Admin key management is not supported on Fermyon Spin.\n\ - Use the Fastly adapter (via Viceroy or deployed) to rotate or deactivate keys.\n", - ); - let mut response = Response::new(body); - *response.status_mut() = StatusCode::NOT_IMPLEMENTED; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - -fn admin_ec_lookup_not_supported() -> Response { - core_admin_ec_lookup_not_supported() +/// The geo lookup runs inside +/// [`EcContext::read_from_request_resolving_geo`], so every adapter reports the +/// same distinction: no location falls back to the top of the +/// `permissions.yaml` rules tree, while a failed lookup resolves every +/// permission at the requires-signal floor and is logged at error level. +/// Spin's platform geo is a no-op, so a request resolves at that top node +/// unless it carries a signal. +/// +/// Mirrors the Fastly entry point, which keeps the report and answers with an +/// error response: when the Edge Cookie context cannot be read the request +/// fails rather than continuing with `EcContext::default()`, which would serve +/// every request with no identity. A malformed cookie value, a bad consent +/// string and a failed geo lookup do not reach this error path at all, so +/// failing here does not fail requests for ordinary parse problems. +/// +/// # Errors +/// +/// Returns an error when the selected Edge Cookie module cannot be built for +/// this request, or when the request's `Cookie` header is not valid UTF-8. +async fn build_ec_context( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> Result> { + EcContext::read_from_request_resolving_geo(settings, req, services).await } // --------------------------------------------------------------------------- @@ -481,23 +616,6 @@ pub(crate) fn http_error(report: &Report) -> Response { response } -/// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` -/// aliases on the Spin adapter. -/// -/// These non-`/_ts` aliases are not matched by the `^/_ts/admin` basic-auth -/// handler, so they fail closed locally rather than fall through to the -/// publisher fallback, which would forward the caller's `Authorization` header -/// and key-management payload to the origin. -fn legacy_admin_alias_denied() -> Response { - let mut response = Response::new(edgezero_core::body::Body::from("Not found\n")); - *response.status_mut() = StatusCode::NOT_FOUND; - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - // --------------------------------------------------------------------------- // Startup error fallback // --------------------------------------------------------------------------- @@ -581,6 +699,28 @@ impl TrustedServerApp { Ok(build_router(&state)) } + /// Build the full application router from explicit settings, composing the + /// built-in integrations with the externally supplied builders in + /// `integrations`. + /// + /// The route table is the one [`TrustedServerApp::routes_with_settings`] + /// builds, so a composed deployment routes exactly as the plain one does. + /// + /// # Errors + /// + /// Returns an error when the selected Edge Cookie module cannot be built + /// for this adapter, when the auction plan does not compile or cannot run + /// on this adapter, or when the auction orchestrator or the integration + /// registry fail to initialize, which includes two builders claiming the + /// same integration id. + pub fn routes_with_registrations( + settings: Settings, + integrations: &[IntegrationBuilder], + ) -> Result> { + let state = build_state_with_registrations(settings, integrations)?; + Ok(build_router(&state)) + } + /// Build the full router with explicit settings and runtime services. /// /// Each request receives a clone of the supplied services, allowing callers @@ -622,39 +762,54 @@ fn build_router(state: &Arc) -> RouterService { } }; - // /verify-signature + // /_ts/permissions and /_ts/permissions.json, what the deployment + // decided for the asking request, shown to anyone. Read only, so + // nothing is written for the reader. let s = Arc::clone(&state); - let verify_handler = move |ctx: RequestContext| { + let permissions_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { let services = s.services_for_request(&ctx); let req = ctx.into_request(); - Ok(handle_verify_signature(&s.settings, &services, req) - .unwrap_or_else(|e| http_error(&e))) + Ok::( + handle_permissions(&s.settings, &services, &req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) } }; - let admin_not_supported_handler = |_ctx: RequestContext| async { - Ok::(admin_key_management_not_supported()) + // /_ts/config and /_ts/config.json, the settings the deployment is + // running, masked, shown to anyone. + let s = Arc::clone(&state); + let config_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { Ok::(handle_config(&s.settings, &ctx.into_request())) } }; - let cache_purge_unsupported_handler = - |_ctx: RequestContext| async { Ok::(cache_purge_not_supported()) }; - - let admin_ec_not_supported_handler = |_ctx: RequestContext| async { - Ok::(admin_ec_lookup_not_supported()) + // /_ts/data, what is held against the request's own Edge Cookie. This + // adapter keeps no identity graph, so the page says nothing is held. + let s = Arc::clone(&state); + let data_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let services = s.services_for_request(&ctx); + Ok::( + handle_data(None, None, services.public_prefix(), &ctx.into_request()) + .unwrap_or_else(|e| http_error(&e)), + ) + } }; - // Admin EIDs echo: pure request inspection (no KV), so this adapter - // serves the real handler. + // /verify-signature let s = Arc::clone(&state); - let admin_eids_handler = move |ctx: RequestContext| { + let verify_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { + let services = s.services_for_request(&ctx); let req = ctx.into_request(); - let result = PartnerRegistry::from_config(&s.settings.ec.partners) - .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); - Ok::(result.unwrap_or_else(|e| http_error(&e))) + Ok(handle_verify_signature(&s.settings, &services, req) + .unwrap_or_else(|e| http_error(&e))) } }; @@ -670,18 +825,18 @@ fn build_router(state: &Arc) -> RouterService { // OpenRTB metadata that auction signing derives from // `RequestInfo::from_request` uses the trusted runtime authority. let mut req = ctx.into_request(); - if let Err(error) = - trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &s.settings, - &mut req, - ) - { + if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { return Ok(http_error(&error)); } // Build the geo-aware EC context so the auction consent gate sees // the caller's jurisdiction — `EcContext::default()` fails it - // closed for consented users. - let mut ec_context = build_ec_context(&s.settings, &services, &req); + // closed for consented users. When identity cannot be + // established at all, answer with an error rather than running + // the auction with no identity. + let mut ec_context = match build_ec_context(&s.settings, &services, &req).await { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; Ok(handle_auction( &s.settings, &s.orchestrator, @@ -703,15 +858,16 @@ fn build_router(state: &Arc) -> RouterService { async move { let services = s.services_for_request(&ctx); let mut req = ctx.into_request(); - if let Err(error) = - trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &s.settings, - &mut req, - ) - { + if let Err(error) = s.registry.prepare_request(&s.settings, &mut req) { return Ok(http_error(&error)); } - let mut ec_context = build_ec_context(&s.settings, &services, &req); + // Identity could not be established (for example the selected + // Edge Cookie module is unavailable). Answer with an error + // rather than re-running the auction with no identity. + let mut ec_context = match build_ec_context(&s.settings, &services, &req).await { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; let auction = AuctionDispatch { orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), @@ -797,13 +953,10 @@ fn build_router(state: &Arc) -> RouterService { ) -> Result { let services = state.services_for_request(&ctx); let mut req = ctx.into_request(); - if let Some(response) = deny_admin_diagnostic_fallback(&req) { + if let Some(response) = closed_path_response(&req) { return Ok(response); } - if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( - &state.settings, - &mut req, - ) { + if let Err(error) = state.registry.prepare_request(&state.settings, &mut req) { return Ok(http_error(&error)); } @@ -815,7 +968,13 @@ fn build_router(state: &Arc) -> RouterService { let result = if method == Method::GET && path.starts_with("/static/tsjs=") { handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); + // A module route is handed what was resolved for its request, + // the permissions among them, as the publisher path is. + let mut ec_context = match build_ec_context(&state.settings, &services, &req).await + { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; state .registry .handle_proxy(ProxyDispatchInput { @@ -834,14 +993,24 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - let mut ec_context = build_ec_context(&state.settings, &services, &req); + // Identity could not be established (for example the selected + // Edge Cookie module is unavailable). Answer with an error + // rather than serving the page with no identity. + let mut ec_context = match build_ec_context(&state.settings, &services, &req).await + { + Ok(context) => context, + Err(report) => return Ok(http_error(&report)), + }; let auction = AuctionDispatch { orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, }; match handle_publisher_request( - &state.settings, + AppContext { + settings: &state.settings, + integration_registry: &state.registry, + }, &services, None, &mut ec_context, @@ -877,8 +1046,6 @@ fn build_router(state: &Arc) -> RouterService { let s = Arc::clone(&s); dispatch(s, ctx) }; - let legacy_admin_deny = - |_ctx: RequestContext| async { Ok::(legacy_admin_alias_denied()) }; let mut builder = RouterService::builder() // Outermost middleware: strips the configured trusted-client-IP @@ -887,12 +1054,10 @@ fn build_router(state: &Arc) -> RouterService { // shared-secret authentication header. .middleware(SanitizeRequestMiddleware::new(Arc::clone(&state.settings))) .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) - .middleware(AuthMiddleware::new(Arc::clone(&state.settings))) // Innermost middleware: normalize every routed request (strip // spoofable forwarded headers, derive the trusted Host/scheme/client-IP // from Spin's synthetic runtime headers) so no handler can opt out of - // the de-spoofing invariant. Runs after auth so the basic-auth gate - // continues to see the original request, matching prior behaviour. + // the de-spoofing invariant. .middleware(NormalizeMiddleware::new()) // Cheap liveness probe, matching the Fastly/Axum adapters. Registered // explicitly so it is not absorbed by the publisher `/{*rest}` fallback. @@ -901,25 +1066,11 @@ fn build_router(state: &Arc) -> RouterService { }) .get("/.well-known/trusted-server.json", discovery_handler) .post("/verify-signature", verify_handler) - // Canonical admin key routes. These match `Settings::ADMIN_ENDPOINTS` - // and the production basic-auth handler regex (`^/_ts/admin`), so they - // are auth-gated under a production-shaped config. - // - // The legacy non-`/_ts` aliases (`/admin/keys/*`) are registered below - // to a local 404 for every publisher-fallback method: the production - // handler regex `^/_ts/admin` does not match them, and letting them - // fall through to the publisher fallback would forward admin - // credentials and key-management payloads to the origin. - .post("/_ts/admin/keys/rotate", admin_not_supported_handler) - .post("/_ts/admin/keys/deactivate", admin_not_supported_handler) - // Admin EC lookup routes. Registered explicitly (like the key - // routes above) so they never fall through to the publisher - // fallback, and they match `Settings::ADMIN_ENDPOINTS` for auth - // coverage. The EC identity graph is Fastly KV backed, so this - // adapter has no store to read. - .get("/_ts/admin/ec", admin_ec_not_supported_handler) - .get("/_ts/admin/ec/{id}", admin_ec_not_supported_handler) - .get("/_ts/admin/eids", admin_eids_handler) + .get(PERMISSIONS_PAGE_PATH, permissions_handler.clone()) + .get(PERMISSIONS_JSON_PATH, permissions_handler) + .get(CONFIG_PAGE_PATH, config_handler.clone()) + .get(CONFIG_JSON_PATH, config_handler) + .get(DATA_PAGE_PATH, data_handler) .post("/auction", auction_handler) .get(PAGE_BIDS_PATH, page_bids_handler.clone()) .route(PAGE_BIDS_PATH, Method::OPTIONS, page_bids_options_handler) @@ -940,19 +1091,6 @@ fn build_router(state: &Arc) -> RouterService { .get("/first-party/proxy-rebuild", fp_rebuild_handler) .post("/first-party/proxy-rebuild", fp_rebuild_post_handler); - for method in LEGACY_ADMIN_DENY_METHODS { - builder = builder.route( - "/_ts/admin/cache/purge", - method.clone(), - cache_purge_unsupported_handler, - ); - } - - for method in LEGACY_ADMIN_DENY_METHODS { - builder = builder.route("/admin/keys/rotate", method.clone(), legacy_admin_deny); - builder = builder.route("/admin/keys/deactivate", method.clone(), legacy_admin_deny); - } - // Mirror the Fastly/Axum publisher fallback: every supported method that is // not a named route's primary method falls through to the publisher origin // (e.g. HEAD /, OPTIONS /page preflight, HEAD /first-party/proxy) instead of @@ -964,6 +1102,31 @@ fn build_router(state: &Arc) -> RouterService { } } } + // Routes chosen by the settings, so not in the fixed table. + let routes = prefix_routes(&state.settings); + if !routes.is_empty() { + let s = Arc::clone(&state); + let attestation_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + Ok::(handle_prefix( + &s.settings, + s.registry.platform(), + &ctx.into_request(), + )) + } + }; + for route in &routes { + for method in publisher_fallback_methods() { + if matches!(method, Method::GET | Method::HEAD) || route.every_method { + builder = builder.route(&route.path, method, attestation_handler.clone()); + } else { + builder = builder.route(&route.path, method, fallback.clone()); + } + } + } + } + for method in publisher_fallback_methods() { builder = builder.route("/", method.clone(), fallback.clone()); builder = builder.route("/{*rest}", method, fallback.clone()); @@ -975,16 +1138,14 @@ fn build_router(state: &Arc) -> RouterService { #[cfg(test)] mod tests { + use edgezero_core::http::request_builder; + use edgezero_core::params::PathParams; + use super::*; fn multi_provider_settings() -> Settings { let mut settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-password" - [publisher] domain = "publisher.example" cookie_domain = ".publisher.example" @@ -993,50 +1154,62 @@ mod tests { [ec] passphrase = "fictional-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse startup test settings"); settings.auction.enabled = true; - settings.auction.providers = - std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) - .map(|id| { + let names = ["provider_a", "provider_b"]; + settings.demand = trusted_server_core::provider_table::ProviderList::new( + names.iter().map(|name| (*name).to_string()).collect(), + names + .iter() + .map(|name| { ( - id.parse().expect("should parse provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: format!("https://{id}.example/openrtb"), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: - trusted_server_core::auction::NotificationConfig::default(), - profile_config: "{}" - .parse() - .expect("should parse empty profile config object"), - }, + (*name).to_string(), + serde_json::Map::from_iter([ + ( + "implementation".to_string(), + serde_json::json!("auction-protocol.openrtb"), + ), + ( + "endpoint".to_string(), + serde_json::json!(format!("https://{name}.example/openrtb")), + ), + ("routing".to_string(), serde_json::json!("all_eligible")), + ]), ) }) - .collect(); + .collect(), + ); settings } #[test] fn startup_registers_aps_renderer_route() { let mut settings = multi_provider_settings(); - settings.auction.providers.clear(); - settings.auction.providers.insert( - "aps-main".parse().expect("should parse APS provider ID"), - trusted_server_core::auction::ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: None, - routing: trusted_server_core::auction::RoutingMode::AllEligible, - notifications: trusted_server_core::auction::NotificationConfig::default(), - profile_config: "{\"account_id\":\"example-account\"}" - .parse() - .expect("should parse APS profile config"), - }, + settings.demand = trusted_server_core::provider_table::ProviderList::new( + vec!["aps_main".to_string()], + std::collections::BTreeMap::from([( + "aps_main".to_string(), + serde_json::Map::from_iter([ + ( + "implementation".to_string(), + serde_json::json!("auction.aps"), + ), + ( + "endpoint".to_string(), + serde_json::json!("https://aps.example/e/pb/bid"), + ), + ("routing".to_string(), serde_json::json!("all_eligible")), + ( + "account_id".to_string(), + serde_json::json!("example-account"), + ), + ]), + )]), ); let state = @@ -1050,6 +1223,41 @@ mod tests { ); } + /// Every request's services carry the prefixes of the publisher its + /// settings describe, which start every shared store key and every name + /// the browser holds. + #[test] + fn a_requests_services_carry_the_publishers_prefixes() { + let settings = Settings::from_toml( + r#" + [publisher] + domain = "www.publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret-32-bytes-ok" + + [geo] + assume_single_jurisdiction = true + "#, + ) + .expect("should parse the settings"); + let state = build_state_with_settings(settings).expect("should build the state"); + let request = request_builder() + .method("GET") + .uri("https://www.publisher.example/") + .body(edgezero_core::body::Body::empty()) + .expect("should build the request"); + let ctx = RequestContext::new(request, PathParams::default()); + + let services = state.services_for_request(&ctx); + + assert_eq!( + services.server_key_prefix().as_str(), + "ts.publisher.example" + ); + assert_eq!(services.public_prefix().as_str(), "ts"); + } + #[test] fn disabled_startup_accepts_dormant_multi_provider_auction_plan() { let mut settings = multi_provider_settings(); @@ -1071,6 +1279,86 @@ mod tests { ); } + #[test] + fn build_state_takes_its_settings_from_the_platform_config_store() { + // Every other test enters through the `routes_with_settings` parity + // seam, so this is the one that calls `build_state` itself. There is + // no Spin runtime under `cargo test`, so there are no component + // variables to read and this cannot return `Ok` here. Its settings + // come from the config store alone, so the failure has to be the + // absence of a config store, and never a configuration compiled into + // the binary, such as a template's placeholder password. + let Err(error) = build_state() else { + return; + }; + let message = format!("{error:?}"); + assert!( + message.contains("config store"), + "build_state should fail only for want of a config store, got: {message}" + ); + assert!( + !message.to_lowercase().contains("password"), + "build_state must not fail on a configuration compiled into the binary, got: {message}" + ); + } + + /// Settings selecting a vendor Edge Cookie module this adapter does not + /// inject, with the `[ec.acme]` block that module's settings live in. + /// `acme` is a fictional vendor key. + const UNINJECTED_MODULE_TOML: &str = r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + module = "acme" + + [ec.acme] + endpoint = "https://ec.acme.example.com" + + # An Edge Cookie module is configured, so single-jurisdiction + # operation is acknowledged because no geo module is selected. + [geo] + assume_single_jurisdiction = true + "#; + + /// The per-request Edge Cookie read must return its error rather than a + /// default context. + /// + /// Continuing with `EcContext::default()` would serve every request with + /// no identity when the selected module cannot be built. The call sites + /// propagate the error to `http_error`, matching the Fastly adapter. The + /// settings are parsed directly, bypassing the composition root's startup + /// check, so the per-request behavior can be exercised with a selection + /// the adapter cannot supply. + #[tokio::test] + async fn build_ec_context_fails_when_the_selected_module_is_unavailable() { + let settings = Settings::from_toml(UNINJECTED_MODULE_TOML) + .expect("should parse settings selecting an uninjected module"); + let req = request_builder() + .method("POST") + .uri("https://test-publisher.example.com/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build test request"); + let ctx = RequestContext::new(req, PathParams::default()); + // No resolved module is threaded here, so the request path resolves + // the selection itself, which is what an embedder driving core + // directly does and where the loud failure has to stay. + let services = build_runtime_services(&ctx, &settings, &Arc::default()); + let req = ctx.into_request(); + + let error = build_ec_context(&settings, &services, &req) + .await + .expect_err("an unavailable Edge Cookie module must fail the request"); + + assert!( + error.to_string().contains("acme"), + "the error should name the selected module, got: {error}" + ); + } + #[test] fn scheme_host_from_spin_url_extracts_localhost_with_port() { assert_eq!( @@ -1393,12 +1681,9 @@ mod tests { 200, "GET /health must return 200 from the startup fallback" ); - assert_eq!( - resp.headers() - .get("x-ts-version") - .and_then(|v| v.to_str().ok()), - trusted_server_core::constants::TS_GIT_VERSION, - "startup-fallback /health should report the compiled-in git version" + assert!( + resp.headers().get("x-ts-version").is_none(), + "startup-fallback /health should carry no version" ); let body = resp.into_body().into_bytes().unwrap_or_default(); assert_eq!( diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index fc1b939bd5..ecb104b7fb 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -5,9 +5,10 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; -use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; -use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; +use trusted_server_core::http_util::{ + ROUTE_NAMESPACE, sanitize_trusted_client_ip_headers, strip_deployment_headers, +}; use trusted_server_core::settings::Settings; // --------------------------------------------------------------------------- @@ -55,9 +56,8 @@ impl Middleware for SanitizeRequestMiddleware { /// Spin does not expose geo headers to the application, so /// `X-Geo-Info-Available: false` is emitted for every response. /// -/// Registered directly inside [`SanitizeRequestMiddleware`] and ahead of -/// [`AuthMiddleware`] so that every outgoing response — including auth-rejected -/// ones — carries a consistent set of headers. +/// Registered directly inside [`SanitizeRequestMiddleware`], so that every +/// outgoing response carries a consistent set of headers. pub struct FinalizeResponseMiddleware { settings: Arc, } @@ -75,50 +75,15 @@ impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let geo_available = false; + let path = ctx.request().uri().path().to_owned(); let mut response = next.run(ctx).await?; + // Before finalizing, so a header finalizing writes is kept. + strip_deployment_headers(&path, ROUTE_NAMESPACE, &mut response); apply_finalize_headers(&self.settings, geo_available, &mut response); Ok(response) } } -// --------------------------------------------------------------------------- -// AuthMiddleware -// --------------------------------------------------------------------------- - -/// Inner middleware: enforces basic-auth before the handler runs. -/// -/// - `Ok(Some(response))` from [`enforce_basic_auth`] → auth failed; return the -/// challenge response (bubbles through [`FinalizeResponseMiddleware`] for header injection). -/// - `Ok(None)` → no auth required or credentials accepted; continue the chain. -/// - `Err(report)` → internal error; log and convert to a 500 HTTP response. -pub struct AuthMiddleware { - settings: Arc, -} - -impl AuthMiddleware { - /// Creates a new [`AuthMiddleware`] with the given settings. - #[must_use] - pub fn new(settings: Arc) -> Self { - Self { settings } - } -} - -#[async_trait(?Send)] -impl Middleware for AuthMiddleware { - async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { - Ok(Some(response)) => return Ok(response), - Ok(None) => {} - Err(report) => { - log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); - } - } - - next.run(ctx).await - } -} - // --------------------------------------------------------------------------- // NormalizeMiddleware // --------------------------------------------------------------------------- @@ -132,9 +97,6 @@ impl Middleware for AuthMiddleware { /// structurally rather than by per-handler convention. A future route, or a /// signing handler that begins deriving an issuer/audience from `RequestInfo`, /// cannot silently trust spoofable input by forgetting to opt in. -/// -/// Registered after [`AuthMiddleware`] (innermost) so the basic-auth gate still -/// evaluates the original request, preserving prior behaviour. #[derive(Default)] pub struct NormalizeMiddleware; @@ -174,13 +136,15 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - trusted_server_core::version_header::apply_git_version_header(response); - // Cookie-bearing responses stay private to shared caches and operator // headers cannot re-enable caching for uncacheable per-user payloads. trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( settings, response, ); + + // Last, so that nothing an origin or an operator's header set can loosen + // a site taken out of search. + trusted_server_core::robots_txt::apply_response_robots_tag(settings, response); } // --------------------------------------------------------------------------- @@ -226,11 +190,6 @@ mod tests { // by design. let mut s = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -238,7 +197,13 @@ mod tests { proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should load test settings"); @@ -350,21 +315,58 @@ mod tests { ); } + fn ctx_for(path: &str) -> RequestContext { + let req = request_builder() + .method(Method::GET) + .uri(path) + .body(Body::empty()) + .expect("should build test request"); + RequestContext::new(req, PathParams::new(HashMap::new())) + } + + /// A response as an origin behind a cache sends it, naming the node that + /// answered and how the cache treated the request. + fn response_naming_its_node() -> Response { + response_builder() + .header("x-served-by", "cache-node-1") + .header("x-cache", "HIT") + .header("x-cache-hits", "3") + .body(Body::empty()) + .expect("should build test response") + } + + /// What the finalize middleware makes of that response for a request to + /// `path`. + fn finalized(path: &str, settings: Settings) -> Response { + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let handler = Arc::new(|_ctx: RequestContext| async move { + Ok::(response_naming_its_node()) + }); + + block_on(middleware.handle(ctx_for(path), Next::new(&[], &*handler))) + .expect("should run middleware") + } + #[test] - fn emits_git_version_header() { - let mut response = empty_response(); - apply_finalize_headers( - &settings_with_response_headers(vec![]), - false, - &mut response, + fn finalize_middleware_strips_deployment_headers_before_it_writes_the_operators() { + let response = finalized( + "/articles/an-article", + settings_with_response_headers(vec![("X-Cache", "configured")]), ); + + for name in ["x-served-by", "x-cache-hits"] { + assert!( + !response.headers().contains_key(name), + "should strip {name} from a publisher's page" + ); + } assert_eq!( response .headers() - .get("x-ts-version") + .get("x-cache") .and_then(|v| v.to_str().ok()), - trusted_server_core::constants::TS_GIT_VERSION, - "should report the compiled-in git version as x-ts-version" + Some("configured"), + "should carry the operator's header and not the origin's" ); } } diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index b1ced0f70a..b4914202af 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -704,7 +704,13 @@ impl PlatformSecretStore for SpinSecretStoreAdapter { /// before routing. Secrets are read synchronously from Spin component /// variables because Trusted Server's platform secret trait is sync. #[must_use] -pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { +pub fn build_runtime_services( + ctx: &edgezero_core::context::RequestContext, + settings: &trusted_server_core::settings::Settings, + permission_signal_modules: &Arc< + [Arc], + >, +) -> RuntimeServices { let client_ip = extract_client_ip(ctx); #[cfg(all(feature = "spin", target_arch = "wasm32"))] @@ -733,7 +739,17 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .kv_store(kv_store) .backend(Arc::new(NoopBackend)) .http_client(http_client) - .geo(Arc::new(NullGeo)) + // Routed through the [geo] module selector like the Fastly adapter, + // so the selector behaves the same on every adapter. Spin has no host + // geo service, so the host default resolves nothing either way. + .geo(trusted_server_core::platform::build_geo_module( + settings, + Arc::new(NullGeo), + )) + // The signal modules were selected once at startup from the scheme + // crates this adapter links, so every request asks exactly the ones + // configuration named, in that order. + .permission_signal_modules(Arc::clone(permission_signal_modules)) .client_info(ClientInfo { client_ip, tls_protocol: None, @@ -754,8 +770,13 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R struct NullGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for NullGeo { - fn lookup(&self, _client_ip: Option) -> Result, Report> { + async fn lookup( + &self, + _client_ip: Option, + _services: &trusted_server_core::platform::RuntimeServices, + ) -> Result, Report> { Ok(None) } } @@ -802,6 +823,16 @@ mod tests { use flate2::write::GzEncoder; use std::io::Write as _; + /// The services graph a module is handed, built the same way the request + /// path builds it so the tests exercise the production shape. + fn test_services() -> RuntimeServices { + build_runtime_services( + &make_ctx_without_spin_context(), + &trusted_server_core::settings::Settings::default(), + &Arc::default(), + ) + } + struct InMemoryConfigStore(std::collections::BTreeMap); #[async_trait::async_trait(?Send)] @@ -911,18 +942,25 @@ mod tests { ); } - #[test] - fn null_geo_always_returns_none() { + #[tokio::test] + async fn null_geo_always_returns_none() { let geo = NullGeo; assert!( - geo.lookup(None).expect("should not fail").is_none(), + geo.lookup(None, &test_services()) + .await + .expect("should not fail") + .is_none(), "should return None without a client IP" ); assert!( - geo.lookup(Some("127.0.0.1".parse().expect("should parse IP"))) - .expect("should not fail") - .is_none(), + geo.lookup( + Some("127.0.0.1".parse().expect("should parse IP")), + &test_services() + ) + .await + .expect("should not fail") + .is_none(), "should return None for any client IP" ); } @@ -1040,7 +1078,11 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn build_runtime_services_uses_noop_native_stores_without_handles() { let ctx = make_ctx_without_spin_context(); - let services = build_runtime_services(&ctx); + let services = build_runtime_services( + &ctx, + &trusted_server_core::settings::Settings::default(), + &Arc::default(), + ); assert!( services.client_info().client_ip.is_none(), @@ -1049,7 +1091,8 @@ mod tests { assert!( services .geo() - .lookup(None) + .lookup(None, &test_services()) + .await .expect("should not fail") .is_none(), "should use null geo" diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 613832b059..7bae070ca4 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -17,17 +17,9 @@ use trusted_server_core::settings::Settings; /// The settings baked into the binary contain placeholder secrets that /// `get_settings()` rejects by design, which would turn every route into a /// startup error page (and its route table into the fallback-only set). -/// The handler regex is the production-shaped `^/_ts/admin`, matching -/// `Settings::ADMIN_ENDPOINTS` and the default config, so the canonical -/// `/_ts/admin/keys/*` routes are auth-gated exactly as in production. fn test_router() -> RouterService { let settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -35,7 +27,13 @@ fn test_router() -> RouterService { proxy_secret = "route-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse route test settings"); @@ -83,147 +81,17 @@ fn edgezero_manifest_loads_and_resolves_spin_stores() { } // --------------------------------------------------------------------------- -// Middleware regression tests — verify FinalizeResponseMiddleware and -// AuthMiddleware are wired so they cannot be removed silently. +// Middleware regression tests, which verify FinalizeResponseMiddleware is +// wired so it cannot be removed silently. // --------------------------------------------------------------------------- #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_routes_return_501() { - for (path, body) in [ - ("/_ts/admin/keys/rotate", "{}"), - ( - "/_ts/admin/keys/deactivate", - r#"{"kid":"test-key","delete":false}"#, - ), - ] { - let req = request_builder() - .method("POST") - .uri(path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(body)) - .expect("should build request"); - let resp = route(test_router(), req).await; - - assert_eq!( - resp.status().as_u16(), - 501, - "{path} should report that Spin key management is unsupported" - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_ec_routes_return_501() { - // The EC identity graph is Fastly KV backed, so Spin answers the admin - // EC lookup routes locally with 501 instead of letting them fall through - // to the publisher fallback. - let sample_ec_id = format!("{}.abc123", "a".repeat(64)); - for path in [ - "/_ts/admin/ec".to_owned(), - format!("/_ts/admin/ec/{sample_ec_id}"), - ] { - let req = request_builder() - .method("GET") - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = route(test_router(), req).await; - - assert_eq!( - resp.status().as_u16(), - 501, - "{path} should report that Spin EC lookup is unsupported" - ); - assert_eq!( - resp.headers() - .get("content-type") - .and_then(|v| v.to_str().ok()), - Some("application/json") - ); - assert_eq!( - resp.headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_ec_route_without_credentials_returns_401() { - let req = request_builder() - .method("GET") - .uri("/_ts/admin/ec") - .body(edgezero_core::body::Body::empty()) - .expect("should build unauthenticated admin EC request"); - let resp = route(test_router(), req).await; - - assert_eq!(resp.status().as_u16(), 401); - assert!( - resp.headers().contains_key("www-authenticate"), - "admin EC 401 should include the Basic authentication challenge" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_eids_route_returns_200() { - // The EIDs echo is pure request inspection (no KV), so this adapter - // serves the real handler. - let req = request_builder() - .method("GET") - .uri("/_ts/admin/eids") - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = route(test_router(), req).await; - - assert_eq!( - resp.status().as_u16(), - 200, - "/_ts/admin/eids should serve the real EIDs echo handler" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { +async fn an_authenticated_request_to_a_closed_path_is_answered_here() { let ec_id = format!("{}.abc123", "a".repeat(64)); - let valid_paths = [ + for path in [ "/_ts/admin/ec".to_owned(), format!("/_ts/admin/ec/{ec_id}"), "/_ts/admin/eids".to_owned(), - ]; - - for path in valid_paths { - for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - let request = request_builder() - .method(method) - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(edgezero_core::body::Body::from("sensitive-admin-body")) - .expect("should build authenticated admin request"); - let response = route(test_router(), request).await; - - assert_eq!(response.status().as_u16(), 405); - assert_eq!( - response - .headers() - .get("allow") - .and_then(|v| v.to_str().ok()), - Some("GET") - ); - assert_eq!( - response - .headers() - .get("cache-control") - .and_then(|v| v.to_str().ok()), - Some("no-store") - ); - } - } - - for path in [ "/_ts/admin/ec/".to_owned(), format!("/_ts/admin/ec/{ec_id}/extra"), "/_ts/admin/eids/".to_owned(), @@ -231,14 +99,13 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts/admin/eids.json".to_owned(), "/_ts/admin/ec;foo".to_owned(), format!("/_ts/admin/ec%2F{ec_id}"), - // Percent-encoded separators match the `^/_ts/admin` basic-auth - // handler but not a literal-slash namespace check, so they must be - // reserved before publisher fallback forwards credentials upstream. + // A check for a literal slash would miss a percent-encoded separator, + // so these are closed before the publisher fallback forwards + // credentials upstream. "/_ts/admin%2Fec".to_owned(), "/_ts/admin%2fec".to_owned(), - // Retired non-`/_ts` alias namespace: only the two exact paths are - // routed to a local deny, so descendants and encoded separators must - // be reserved at the shared fallback boundary. + // The alias outside `/_ts`, with its descendants and encoded + // separators. "/admin/keys".to_owned(), "/admin/keys/rotate/extra".to_owned(), "/admin/keys%2Frotate".to_owned(), @@ -314,14 +181,9 @@ async fn finalize_middleware_injects_geo_header() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auth_middleware_runs_in_chain_for_protected_routes() { - // Verifies that AuthMiddleware is wired into the middleware chain for auction - // requests. Without it, FinalizeResponseMiddleware would still run but auth - // challenges would be skipped silently. - // - // CI settings may not have basic_auth configured, so this test does not - // assert 401 — it asserts that both middleware layers ran (X-Geo-Info-Available - // present) and that the route is actually reached (status != 404). +async fn the_middleware_chain_runs_for_a_routed_request() { + // Asserts that the finalizing middleware ran (X-Geo-Info-Available present) + // and that the route is actually reached (status != 404). let router = test_router(); let req = request_builder() @@ -335,7 +197,7 @@ async fn auth_middleware_runs_in_chain_for_protected_routes() { assert!( resp.headers().contains_key("x-geo-info-available"), - "middleware chain must inject X-Geo-Info-Available even on auth-rejected responses" + "middleware chain must inject X-Geo-Info-Available" ); assert_ne!( resp.status().as_u16(), @@ -368,7 +230,9 @@ async fn tsjs_route_is_routed_not_5xx() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { let router = test_router(); - let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); + let src = trusted_server_core::tsjs::tsjs_script_src( + &trusted_server_core::tsjs_bundle::compile_time_parts(&["creative"]), + ); let req = request_builder() .method("GET") .uri(src) @@ -438,46 +302,18 @@ async fn verify_signature_put_falls_through_to_publisher_fallback() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_rotate_key_is_routed() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_ne!( - resp.status().as_u16(), - 404, - "/admin/keys/rotate must be routed" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_deactivate_key_is_routed() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/deactivate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_ne!( - resp.status().as_u16(), - 404, - "/admin/keys/deactivate must be routed" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { - // The production basic-auth regex `^/_ts/admin` does not match - // `/admin/keys/*`, so those aliases are not auth-gated. Any - // publisher-fallback method carrying an `Authorization` header must be - // denied locally with 404, never proxied to the publisher origin. - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { +async fn key_administration_paths_are_answered_here_and_never_proxied() { + // A request to a key administration path carries an operator's + // credentials and a key payload. Every publisher-fallback method must be + // answered here with 404 and never proxied to the publisher origin, which + // would be handed both. A publisher-fallback proxy without a backend would + // surface as a 5xx, so a 404 proves the request was answered here. + for path in [ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/admin/keys/rotate", + "/admin/keys/deactivate", + ] { for method in ["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { let router = test_router(); let req = request_builder() @@ -486,18 +322,18 @@ async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") .header("content-type", "application/json") .body(edgezero_core::body::Body::from("{\"key_id\":\"leak-me\"}")) - .expect("should build authorized legacy-alias request"); + .expect("should build authorized key administration request"); let resp = route(router, req).await; assert_eq!( resp.status().as_u16(), 404, - "legacy {method} {path} with Authorization must be denied locally (404), not proxied to publisher" + "{method} {path} with Authorization must be answered here (404), not proxied to publisher" ); assert!( !resp.headers().contains_key("www-authenticate"), - "legacy {method} {path} must not issue an admin auth challenge" + "{method} {path} must not issue an admin auth challenge" ); } } @@ -832,145 +668,154 @@ async fn first_party_proxy_round_trip_through_spin_router() { } // --------------------------------------------------------------------------- -// Basic-auth parity tests +// Closed paths and the inspection endpoints // --------------------------------------------------------------------------- #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_without_credentials_returns_401() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin route must return 401 without credentials" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_without_credentials_includes_www_authenticate_header() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "should be 401 before checking header" - ); - assert!( - resp.headers().contains_key("www-authenticate"), - "401 response must include WWW-Authenticate header" - ); - let www_auth = resp - .headers() - .get("www-authenticate") - .expect("should have www-authenticate header") - .to_str() - .expect("should be valid UTF-8"); - assert!( - www_auth.starts_with("Basic realm="), - "WWW-Authenticate must be Basic scheme, got: {www_auth}" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_route_with_wrong_credentials_returns_401() { - use base64::Engine as _; - let creds = base64::engine::general_purpose::STANDARD.encode("admin:wrong-password"); - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .header("authorization", format!("Basic {creds}")) - .body(edgezero_core::body::Body::from("{}")) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin route must reject wrong credentials with 401" - ); +async fn permissions_endpoint_answers_anyone_as_data_and_as_a_page() { + for (path, content_type) in [ + ("/_ts/permissions.json", "application/json"), + ("/_ts/permissions", "text/html; charset=utf-8"), + ] { + let req = request_builder() + .method("GET") + .uri(path) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + assert_eq!( + resp.status().as_u16(), + 200, + "{path} should answer with no credential" + ); + assert_eq!( + resp.headers()["content-type"], + content_type, + "{path} should answer in its own form" + ); + assert_eq!( + resp.headers()["cache-control"], + "no-store", + "{path} is one request's own answer" + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn discovery_endpoint_does_not_require_auth() { - let router = test_router(); - let req = request_builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = route(router, req).await; - assert_ne!( - resp.status().as_u16(), - 401, - "/.well-known/trusted-server.json must not require auth" - ); +async fn config_endpoint_answers_anyone_and_masks_secrets() { + for (path, content_type) in [ + ("/_ts/config.json", "application/json"), + ("/_ts/config", "text/html; charset=utf-8"), + ] { + let req = request_builder() + .method("GET") + .uri(path) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + assert_eq!( + resp.status().as_u16(), + 200, + "{path} should answer with no credential" + ); + assert_eq!( + resp.headers()["content-type"], + content_type, + "{path} should answer in its own form" + ); + let body = resp.into_body().into_bytes().unwrap_or_default(); + let text = String::from_utf8_lossy(&body); + assert!( + text.contains("XXXX"), + "{path} should mask what it does not show" + ); + assert!( + !text.contains("route-test-proxy-secret"), + "{path} should not carry the proxy secret" + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auction_endpoint_does_not_require_auth() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/auction") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(r#"{"adUnits":[]}"#)) - .expect("should build request"); - let resp = route(router, req).await; - assert_ne!( - resp.status().as_u16(), - 401, - "/auction must not apply admin basic-auth gate" - ); +async fn data_endpoint_answers_a_browser_opening_the_page_and_nothing_else() { + for (opened_as_a_page, expected) in [(false, 403), (true, 200)] { + let mut builder = request_builder().method("GET").uri("/_ts/data"); + if opened_as_a_page { + builder = builder + .header("sec-fetch-mode", "navigate") + .header("sec-fetch-dest", "document"); + } + let req = builder + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + assert_eq!( + resp.status().as_u16(), + expected, + "opened as a page: {opened_as_a_page}" + ); + assert_eq!( + resp.headers()["cache-control"], + "no-store, private", + "should never be stored" + ); + let body = resp.into_body().into_bytes().unwrap_or_default(); + assert_eq!( + String::from_utf8_lossy(&body).contains("keeps no record"), + opened_as_a_page, + "should say nothing is held to a browser opening the page, and to nothing else" + ); + } } // --------------------------------------------------------------------------- -// Admin key route full path coverage +// Edge Cookie module availability // --------------------------------------------------------------------------- -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_rotate_key_auth_fail_returns_401() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin/keys/rotate without credentials must return 401" - ); -} +/// Test settings selecting a vendor Edge Cookie module this adapter does not +/// inject, with the `[ec.acme]` block that module's settings live in. +/// `acme` is a fictional vendor key. +const UNINJECTED_MODULE_TOML: &str = r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "route-test-proxy-secret" + + [ec] + module = "acme" + + [ec.acme] + endpoint = "https://ec.acme.example.com" + + # An Edge Cookie module is configured, so single-jurisdiction operation + # is acknowledged because no geo module is selected. + [geo] + assume_single_jurisdiction = true +"#; + +/// A module selection this adapter can never supply must fail while the +/// application state is built, before any request is served. +/// +/// Configuration validation accepts this selection, because only the adapter +/// that injects a module knows what that module needs, and this adapter +/// injects no vendor Edge Cookie module, so only the composition root can +/// catch it. Without the startup check the deployment would come up and answer +/// every request. +#[test] +fn selecting_a_module_this_adapter_cannot_supply_fails_at_startup() { + let settings = Settings::from_toml(UNINJECTED_MODULE_TOML) + .expect("should parse settings selecting an uninjected module"); -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_deactivate_key_auth_fail_returns_401() { - let router = test_router(); - let req = request_builder() - .method("POST") - .uri("/_ts/admin/keys/deactivate") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = route(router, req).await; - assert_eq!( - resp.status().as_u16(), - 401, - "admin/keys/deactivate without credentials must return 401" + // `RouterService` is not `Debug`, so take the error side directly rather + // than through `expect_err`. + let error = TrustedServerApp::routes_with_settings(settings) + .err() + .expect("building state with an uninjected module should fail"); + + assert!( + error.to_string().contains("acme"), + "the startup error should name the selected module, got: {error}" ); } @@ -1044,3 +889,115 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { "should not leak generated placeholders: {html}" ); } + +/// Every route registered beneath an underscore prefix is one of the +/// addresses an attestation endpoint may not take, so a new fixed route +/// cannot leave an endpoint free to register its address twice. +#[test] +fn every_fixed_route_beneath_an_underscore_prefix_is_reserved() { + use trusted_server_core::attestation::RESERVED_PATHS; + + for route in test_router().routes() { + let path = route.path(); + if path.starts_with("/_") { + assert!( + RESERVED_PATHS.contains(&path), + "{path} should be listed in attestation::RESERVED_PATHS" + ); + } + } +} + +// --------------------------------------------------------------------------- +// robots.txt +// --------------------------------------------------------------------------- + +/// The route test settings with a `[robots-txt]` section that refuses every +/// crawler and keeps one path open. +fn router_refusing_every_crawler() -> RouterService { + let settings = Settings::from_toml( + r#" + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "route-test-proxy-secret" + + [ec] + module = "hmac" + + [ec.hmac] + passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true + + [robots-txt] + modules = "refuse_all" + always_allow = ["/ads.txt"] + + [[fetch]] + media_type = "text/plain" + path = "/robots.txt" + middleware = ["robots_txt"] + "#, + ) + .expect("should parse route test settings"); + + TrustedServerApp::routes_with_settings(settings) + .expect("should build router from test settings") +} + +fn robots_request(method: &str, path: &str) -> Request { + request_builder() + .method(method) + .uri(path) + .body(edgezero_core::body::Body::empty()) + .expect("should build request") +} + +fn robots_tag(response: &Response) -> Option { + response + .headers() + .get("x-robots-tag") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) +} + +/// The file itself is the publisher path's, made by the `robots_txt` +/// middleware the entry names, so no route of this server names the path. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn refusing_every_crawler_tags_every_response_and_registers_no_route() { + let registered: Vec = router_refusing_every_crawler() + .routes() + .into_iter() + .map(|route| route.path().to_string()) + .collect(); + assert!( + !registered.iter().any(|path| path == "/robots.txt"), + "the file is the publisher path's: {registered:?}" + ); + + let other = route( + router_refusing_every_crawler(), + robots_request("GET", "/.well-known/trusted-server.json"), + ) + .await; + + assert_eq!( + robots_tag(&other).as_deref(), + Some("noindex, nofollow"), + "no route may be the one page a crawler is allowed to index" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_response_carries_no_blanket_rule_without_the_section() { + let response = route( + test_router(), + robots_request("GET", "/.well-known/trusted-server.json"), + ) + .await; + + assert_eq!(robots_tag(&response), None); +} diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index ce97919c81..2da02463d1 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -56,7 +56,9 @@ tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } tracing = { workspace = true } +trusted-server-auction-prebid-server = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-modules = { workspace = true } url = { workspace = true } uuid = { workspace = true } which = { workspace = true } @@ -117,3 +119,4 @@ predicates = { workspace = true } temp-env = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["test-util"] } +trusted-server-auction-prebid = { workspace = true } diff --git a/crates/trusted-server-cli/README.md b/crates/trusted-server-cli/README.md index e73c646283..bee706925a 100644 --- a/crates/trusted-server-cli/README.md +++ b/crates/trusted-server-cli/README.md @@ -58,7 +58,7 @@ Not implemented: container deployment, rollback, runtime secret injection, calle ### Discovering requirements -`inspect` reads exactly the chosen file and never rewrites or publishes it. Omitting `--config` reads `trusted-server.toml` in the working directory. It discovers server demand from `[auction.providers.*]` entries using the `prebid-server` profile and bidders routed through `[auction.bidders.*]`. The JSON report groups routed bidders under each provider. Browser settings still come from `[integrations.prebid]`, including the runtime's array, indexed-map, and string encodings for `client_side_bidders`. Bundle selections come from `[integrations.prebid.bundle.modules]`: `bidder`, `user_id`, and `analytics` appear as `bundle_adapters`, `identity_modules`, and `analytics_modules` in JSON and as separate lists in human output. The command reports explicitly supplied values only; it does not expand defaults, environment overrides, remote configuration, or request-time inputs. Confirm which source and environment are authoritative before relying on the report. +`inspect` reads exactly the chosen file and never rewrites or publishes it. Omitting `--config` reads `trusted-server.toml` in the working directory. It discovers server demand from the `[demand.]` tables whose `implementation` line is `auction.prebid-server` and the bidders routed to them through `[auction.bidders.*]`. The JSON report groups routed bidders under each demand source. Browser settings still come from `[auction.prebid]`, including the runtime's array, indexed-map, and string encodings for `client_side_bidders`. Bundle selections come from `[auction.prebid.bundle.modules]`: `bidder`, `user_id`, and `analytics` appear as `bundle_adapters`, `identity_modules`, and `analytics_modules` in JSON and as separate lists in human output. The command reports explicitly supplied values only; it does not expand defaults, environment overrides, remote configuration, or request-time inputs. Confirm which source and environment are authoritative before relying on the report. Account identifiers, endpoint values, and bid-parameter values are withheld. Parser errors also withhold source snippets. Server-side bidders, client-side bidders, and browser bundle adapters remain separate; listing a bidder does not establish partner authorization or a host-secret requirement. Disabled auctions and integrations remain disabled. diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs index bee536146b..179aa592e7 100644 --- a/crates/trusted-server-cli/src/app_config.rs +++ b/crates/trusted-server-cli/src/app_config.rs @@ -27,6 +27,15 @@ pub struct AppConfigArgs { pub no_env: bool, } +/// Registers the modules a stock build ships with deploy validation. +/// +/// [`TrustedServerAppConfig`] validates through a trait that takes no +/// arguments, so every path that validates settings calls this first. The +/// first call wins and a later one changes nothing. +pub(crate) fn register_stock_modules() { + trusted_server_core::config::register_deploy_integrations(trusted_server_modules::builders()); +} + /// Effective settings plus the resolved app-config path they were loaded from. #[derive(Debug)] pub struct LoadedSettings { @@ -108,6 +117,7 @@ fn load_settings_with_env_overlay( let mut opts = AppConfigLoadOptions::default(); opts.env_overlay = env_overlay; + register_stock_modules(); let app_config = app_config::deserialize_app_config_with_options::( &app_config_path, &app_name, diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs index 010522f8fb..bb81eef875 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -19,6 +19,7 @@ use serde::Serialize; use trusted_server_core::creative_opportunities::{ CreativeOpportunitiesConfig, validate_page_pattern, }; +use trusted_server_core::middleware::MiddlewarePhase; use url::Url; use crate::commands::audit::ad_templates::{origin_changed, without_fragment}; @@ -158,9 +159,16 @@ struct DraftConfig { js_asset_proxy_candidate_count: usize, } +/// Id of the integration the audit writes its discovered assets for. +const JS_ASSET_PROXY_ID: &str = "js_asset_proxy"; +const GOOGLE_TAG_MANAGER_ID: &str = "google_tag_manager"; + #[derive(Debug, Clone)] struct JsAssetProxySection { + /// The block to write when the audit found at least one asset. toml: String, + /// Comments explaining why no block was written, used when it found none. + notes: String, candidate_count: usize, } @@ -460,38 +468,55 @@ fn build_draft_config_with_generator( .map(|integration| integration.id.as_str()) .collect::>(); - if detected.contains("gpt") { - draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; + // A module runs when the section of its type selects it, so the audit + // writes that section for each module it can configure on its own. The + // rest go to manual review below. + let gtm_container_id = if detected.contains("google_tag_manager") { + extract_gtm_container_id(artifact) + } else { + None + }; + let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; + if asset_proxy_section.candidate_count > 0 { + draft = select_js_asset_proxy(&draft)?; } - if detected.contains("didomi") { - draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; + + // The template documents every module as a commented example, so the + // sections the audit fills in are appended after it. The integrations + // whose modules it selects are kept, so their page changes can be placed. + let mut selected: Vec<&str> = Vec::new(); + for id in ["datadome", "didomi", "gpt"] { + if detected.contains(id) + && let Some((section, written)) = module_section(id) + { + append_section( + &mut draft, + &format!( + "# Selected by `ts audit`, which detected {id} on the audited page.\n\ + [{section}]\n\ + modules = [\"{written}\"]\n" + ), + ); + selected.push(id); + } } - if detected.contains("datadome") { - draft = - replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; + if let Some(container_id) = >m_container_id { + append_section(&mut draft, &build_google_tag_manager_section(container_id)); + selected.push(GOOGLE_TAG_MANAGER_ID); + } + if asset_proxy_section.candidate_count > 0 { + append_section(&mut draft, &asset_proxy_section.toml); + selected.push(JS_ASSET_PROXY_ID); + } else { + append_section(&mut draft, &asset_proxy_section.notes); + } + if let Some(entries) = build_page_entries(&selected) { + append_section(&mut draft, &entries); } - - let asset_proxy_section = build_js_asset_proxy_section(artifact, path_generator)?; - draft = replace_js_asset_proxy_section(&draft, &asset_proxy_section.toml)?; let mut manual_review = Vec::new(); - if detected.contains("google_tag_manager") { - if let Some(gtm_id) = extract_gtm_container_id(artifact) { - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "enabled", - "enabled = true", - )?; - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "container_id", - &format!("container_id = \"{gtm_id}\""), - )?; - } else { - manual_review.push("google_tag_manager"); - } + if detected.contains("google_tag_manager") && gtm_container_id.is_none() { + manual_review.push("google_tag_manager"); } for integration in detected { @@ -509,9 +534,17 @@ fn build_draft_config_with_generator( } draft.push_str("\n# Audit findings requiring manual review\n"); for integration in manual_review { - draft.push_str(&format!( - "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" - )); + draft.push_str(&match module_section(integration) { + Some((section, written)) => format!( + "# - Detected {integration}; review the `[{section}.{written}]` \ + settings in this file, then select \"{written}\" in [{section}] \ + to run it.\n" + ), + None => format!( + "# - Detected {integration}, which no module in this build \ + configures; review it by hand.\n" + ), + }); } } @@ -540,9 +573,14 @@ fn build_js_asset_proxy_section( let (candidates, skipped) = select_js_asset_proxy_candidates(artifact); let mut used_paths = BTreeSet::new(); let mut toml = String::new(); - - toml.push_str("[integrations.js_asset_proxy]\n"); - toml.push_str("enabled = false\n"); + let mut notes = String::new(); + + toml.push_str("# Generated by `ts audit`. Every asset below starts `disabled`, so nothing\n"); + toml.push_str("# is served or rewritten until you review it and set `proxy = \"enabled\"`.\n"); + toml.push_str("# Audit note: some discovered scripts may be runtime-injected and may not\n"); + toml.push_str("# appear in origin HTML. JS Asset Proxy rewrites only matching script src\n"); + toml.push_str("# URLs present in HTML processed by Trusted Server.\n"); + toml.push_str(&format!("[proxy.{JS_ASSET_PROXY_ID}]\n")); toml.push_str("# Uncomment to override upstream cache headers for every asset below.\n"); toml.push_str("# This replaces upstream directives, including private and no-store.\n"); toml.push_str("# Use only when each asset's bytes are identical for every visitor.\n"); @@ -551,35 +589,34 @@ fn build_js_asset_proxy_section( "# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n", ); toml.push_str( - "# that vary by browser User-Agent or use integrity hashes for UA-specific bytes.\n\n", - ); - toml.push_str("# Generated by `ts audit`; review before enabling.\n"); - toml.push_str( - "# Audit note: some discovered scripts may be runtime-injected and may not appear\n", - ); - toml.push_str( - "# in origin HTML. JS Asset Proxy rewrites only matching script src URLs present in\n", + "# that vary by browser User-Agent or use integrity hashes for UA-specific bytes.\n", ); - toml.push_str("# HTML processed by Trusted Server.\n"); if candidates.is_empty() { - toml.push_str( - "# No eligible third-party HTTPS script assets were detected by `ts audit`.\n", + notes.push_str( + "# No eligible third-party HTTPS script assets were detected by `ts audit`, so\n", ); + notes.push_str(&format!( + "# no [proxy.{JS_ASSET_PROXY_ID}] table is written.\n" + )); + append_js_asset_proxy_skip_comments(&mut notes, &skipped); } for candidate in &candidates { let generated_path = generate_unique_asset_path(path_generator, &mut used_paths)?; toml.push('\n'); - toml.push_str("# Generated by `ts audit`; review before enabling.\n"); if let Some(integration) = candidate.integration { let integration = sanitized_comment_value(integration); toml.push_str(&format!("# Detected integration: {integration}\n")); toml.push_str(&format!( - "# Native integration may be preferable: [integrations.{integration}]\n" + "# Native integration may be preferable: [{}]\n", + module_section(&integration).map_or_else( + || integration.clone(), + |(section, written)| format!("{section}.{written}") + ) )); } - toml.push_str("[[integrations.js_asset_proxy.assets]]\n"); + toml.push_str("[[proxy.js_asset_proxy.assets]]\n"); toml.push_str(&format!("path = {}\n", toml_string(&generated_path))); toml.push_str(&format!( "origin_url = {}\n", @@ -594,14 +631,90 @@ fn build_js_asset_proxy_section( } append_js_asset_proxy_skip_comments(&mut toml, &skipped); - toml.push('\n'); Ok(JsAssetProxySection { toml, + notes, candidate_count: candidates.len(), }) } +/// Appends a generated section, separated from what is above it by one blank +/// line. +fn append_section(draft: &mut String, section: &str) { + if !draft.ends_with('\n') { + draft.push('\n'); + } + draft.push('\n'); + draft.push_str(section); +} + +/// The Google Tag Manager block, written from the container the audit found. +fn build_google_tag_manager_section(container_id: &str) -> String { + format!( + "# Generated by `ts audit` from the container found on the audited page.\n\ + [tag]\n\ + modules = [\"google-tag-manager\"]\n\ + \n\ + [tag.google-tag-manager]\n\ + container_id = {}\n", + toml_string(container_id) + ) +} + +/// The entries that run the middleware of the selected modules on every +/// HTML page, or `None` when none of them changes a page. +/// +/// A module changes a page only where an entry names its middleware, so a +/// draft that selected a module and wrote no entry would leave the module +/// changing nothing. The names and their order come from the stock list. +fn build_page_entries(integration_ids: &[&str]) -> Option { + let mut entries = String::new(); + for phase in MiddlewarePhase::ALL { + let names = trusted_server_modules::middleware_for(integration_ids, phase); + if names.is_empty() { + continue; + } + if entries.is_empty() { + entries.push_str( + "# Written by `ts audit`. A module changes a page only where an entry names\n\ + # its middleware, so these run what the modules selected above supply on\n\ + # every HTML page, in the order written.\n", + ); + } else { + entries.push('\n'); + } + let listed = names + .iter() + .map(|name| toml_string(name)) + .collect::>() + .join(", "); + entries.push_str(&format!( + "[[{phase}]]\nmedia_type = \"text/html\"\nmiddleware = [{listed}]\n" + )); + } + (!entries.is_empty()).then_some(entries) +} + +/// Selects the JavaScript asset proxy in the template's `[proxy]` section by +/// uncommenting the selection the template documents there. +fn select_js_asset_proxy(draft: &str) -> CliResult { + const COMMENTED: &str = "# modules = [\"js_asset_proxy\"]"; + if draft.matches(COMMENTED).count() != 1 { + return cli_error(format!( + "failed to update starter config because `{COMMENTED}` was not found once in [proxy]" + )); + } + Ok(draft.replacen(COMMENTED, "modules = [\"js_asset_proxy\"]", 1)) +} + +/// The section a detected integration's module is selected in, and the name it +/// is written under there, or `None` for one no module in this build +/// configures. +fn module_section(id: &str) -> Option<(&'static str, &'static str)> { + trusted_server_modules::selection_of(id) +} + fn select_js_asset_proxy_candidates( artifact: &AuditArtifact, ) -> (Vec>, JsAssetProxySkipCounts) { @@ -680,55 +793,6 @@ fn is_valid_generated_asset_path(path: &str) -> bool { .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase()) } -fn replace_js_asset_proxy_section(document: &str, replacement: &str) -> CliResult { - let lines = document.lines().collect::>(); - let start = lines - .iter() - .position(|line| line.trim() == "[integrations.js_asset_proxy]") - .ok_or_else(|| { - report_error( - "failed to update starter config because section `[integrations.js_asset_proxy]` was not found", - ) - })?; - let mut end = start + 1; - - while end < lines.len() { - let trimmed = lines[end].trim(); - if trimmed.starts_with('[') - && trimmed.ends_with(']') - && trimmed != "[[integrations.js_asset_proxy.assets]]" - { - break; - } - end += 1; - } - - // Blank lines and comments directly above the next section header document - // that section, not this one, so leave them in the draft. - while end > start + 1 { - let trimmed = lines[end - 1].trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - end -= 1; - } else { - break; - } - } - - let mut output_lines = Vec::new(); - output_lines.extend_from_slice(&lines[..start]); - output_lines.extend(replacement.trim_end_matches('\n').lines()); - if end < lines.len() && !lines[end].trim().is_empty() { - output_lines.push(""); - } - output_lines.extend_from_slice(&lines[end..]); - - let mut output = output_lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); - } - Ok(output) -} - fn append_js_asset_proxy_skip_comments(toml: &mut String, skipped: &JsAssetProxySkipCounts) { if skipped.first_party == 0 && skipped.malformed_url == 0 @@ -2414,11 +2478,7 @@ mod tests { draft.js_asset_proxy_candidate_count, 2, "should report generated disabled entries" ); - assert!( - draft - .toml - .contains("[integrations.js_asset_proxy]\nenabled = false") - ); + assert!(draft.toml.contains("[proxy.js_asset_proxy]\n")); assert!(draft.toml.contains("/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js")); assert!(draft.toml.contains("/assets/bbbbbbbbbbbbbbbbbbbbbbbb.js")); assert!( @@ -2431,22 +2491,24 @@ mod tests { assert!( draft .toml - .contains("Native integration may be preferable: [integrations.gpt]") + .contains("Native integration may be preferable: [ad-tag.google]") ); assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" + draft + .toml + .contains("# path = \"/assets/example-vendor-loader.js\""), + "should leave the template's commented example as a comment" ); assert!( draft.toml.contains( "# Proxy behavior and first-party asset routing. Kept active with defaults.\n[proxy]" ), - "should preserve documentation for the section following the replaced block" + "should leave the documentation of the sections it appends below" ); let parsed = toml::from_str::(&draft.toml).expect("draft should parse as TOML"); assert!( - parsed["integrations"]["js_asset_proxy"] + parsed["proxy"]["js_asset_proxy"] .get("cache_ttl_seconds") .is_none(), "generated config should inherit upstream cache headers by default" @@ -2538,9 +2600,14 @@ mod tests { assert_eq!(draft.js_asset_proxy_candidate_count, 1); assert_eq!( + // The template's own example is a comment, so only an entry at the + // start of a line is one the audit wrote. draft .toml - .matches("[[integrations.js_asset_proxy.assets]]") + .matches( + " +[[proxy.js_asset_proxy.assets]]" + ) .count(), 1, "should only emit one candidate entry" @@ -2630,14 +2697,14 @@ mod tests { .contains("No eligible third-party HTTPS script assets") ); assert!( - !draft - .toml - .contains("[[integrations.js_asset_proxy.assets]]"), + !draft.toml.contains("\n[[proxy.js_asset_proxy.assets]]"), "should not emit asset array entries without candidates" ); assert!( - !draft.toml.contains("example-vendor-loader"), - "should remove starter-template placeholder asset" + draft + .toml + .contains("# path = \"/assets/example-vendor-loader.js\""), + "should leave the template's commented example as a comment" ); toml::from_str::(&draft.toml).expect("draft should parse as TOML"); } @@ -2691,11 +2758,167 @@ mod tests { assert!(draft.contains("domain = \"www.publisher.example\"")); assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); - assert!(draft.contains("[integrations.gpt]\nenabled = true")); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); - assert!(draft.contains("container_id = \"GTM-ABC123\"")); assert!(draft.contains("Detected prebid")); - toml::from_str::(&draft).expect("draft should parse as TOML"); + let parsed = toml::from_str::(&draft).expect("draft should parse as TOML"); + let selected = |section: &str| { + parsed[section]["modules"] + .as_array() + .expect("should write the section's selection") + .iter() + .filter_map(|name| name.as_str()) + .map(str::to_owned) + .collect::>() + }; + assert_eq!(selected("tag"), vec!["google-tag-manager"]); + assert_eq!(selected("ad-tag"), vec!["google"]); + assert!( + parsed + .get("auction") + .and_then(|auction| auction.get("modules")) + .is_none(), + "should leave Prebid to manual review" + ); + assert_eq!( + parsed["tag"]["google-tag-manager"]["container_id"].as_str(), + Some("GTM-ABC123"), + "should write the container it found" + ); + } + + /// What the draft's entries of `phase` hold, each as its media type, + /// whether it has a path, and the middleware it names. + fn drafted_entries(parsed: &toml::Value, phase: &str) -> Vec<(String, bool, Vec)> { + parsed + .get(phase) + .and_then(toml::Value::as_array) + .map(|entries| { + entries + .iter() + .map(|entry| { + ( + entry["media_type"] + .as_str() + .expect("should write a media type") + .to_owned(), + entry.get("path").is_some(), + entry["middleware"] + .as_array() + .expect("should write the middleware") + .iter() + .filter_map(toml::Value::as_str) + .map(str::to_owned) + .collect(), + ) + }) + .collect() + }) + .unwrap_or_default() + } + + #[test] + fn build_draft_config_places_the_page_changes_of_the_modules_it_selects() { + let url = Url::parse("https://publisher.example.com/page").expect("should parse URL"); + let mut detected_integrations: Vec = ["gpt", "datadome", "didomi"] + .into_iter() + .map(|id| DetectedIntegration { + id: id.to_string(), + evidence: format!("https://{id}.example.com/tag.js"), + }) + .collect(); + detected_integrations.push(DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "GTM-ABC123".to_string(), + }); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations, + assets: vec![audited_asset( + "https://cdn.vendor.example.com/sdk.js", + AssetParty::ThirdParty, + None, + )], + warnings: Vec::new(), + }; + let mut generator = FixedPathGenerator::new(&["/assets/aaaaaaaaaaaaaaaaaaaaaaaa.js"]); + + let draft = build_draft_config_with_generator( + &url, + &artifact, + &gpt_slots::DiscoveredSlots::default(), + &mut generator, + ) + .expect("should build draft config"); + + let parsed = + toml::from_str::(&draft.toml).expect("draft should parse as TOML"); + assert_eq!( + drafted_entries(&parsed, "fetch"), + vec![( + "text/html".to_owned(), + false, + vec![ + "js_asset_proxy".to_owned(), + "cmp.didomi".to_owned(), + "tag.google-tag-manager".to_owned(), + "bot-protection.datadome".to_owned(), + "ad-tag.google".to_owned(), + ], + )], + "should run the fetch middleware of every module it selected on every HTML \ + page, in the order the stock modules register" + ); + assert_eq!( + drafted_entries(&parsed, "serve"), + vec![( + "text/html".to_owned(), + false, + vec!["bot-protection.datadome.tag".to_owned()], + )], + "should run the one serve middleware the selected modules supply" + ); + trusted_server_core::settings::Settings::from_toml( + &draft + .toml + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", + ) + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"test-proxy-secret-32-bytes-minimum\"", + ), + ) + .expect("should load the draft, its entries included, as settings"); + } + + #[test] + fn build_draft_config_writes_no_entry_when_it_selects_no_module() { + let url = Url::parse("https://publisher.example/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 0, + third_party_asset_count: 0, + detected_integrations: vec![DetectedIntegration { + id: "prebid".to_string(), + evidence: "inline script matched `prebid`".to_string(), + }], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + let parsed = toml::from_str::(&draft).expect("draft should parse as TOML"); + assert!( + drafted_entries(&parsed, "fetch").is_empty() + && drafted_entries(&parsed, "serve").is_empty(), + "should place nothing for a module it leaves to manual review" + ); } #[test] @@ -2717,12 +2940,12 @@ mod tests { let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) .expect("should build draft config"); let config = toml::from_str::(&draft).expect("should parse draft config"); - let gtm = &config["integrations"]["google_tag_manager"]; + let gtm = &config["tag"]["google-tag-manager"]; assert_eq!( - gtm["enabled"].as_bool(), - Some(true), - "should enable GTM when URL evidence contains a container ID" + config["tag"]["modules"][0].as_str(), + Some("google-tag-manager"), + "should select GTM when URL evidence contains a container ID" ); assert_eq!( gtm["container_id"].as_str(), @@ -2767,13 +2990,12 @@ mod tests { .expect("should build draft config"); let config = toml::from_str::(&draft).expect("should parse draft config"); - assert_eq!( - config["integrations"]["google_tag_manager"]["enabled"].as_bool(), - Some(false), - "should not enable GTM with an invalid ID in {evidence}" + assert!( + config.get("tag").is_none(), + "should not select GTM with an invalid ID in {evidence}" ); assert!( - draft.contains("# - Detected google_tag_manager;"), + draft.contains("Detected google_tag_manager"), "should retain manual-review guidance for {evidence}" ); let warnings = @@ -2786,7 +3008,7 @@ mod tests { } #[test] - fn build_draft_config_does_not_enable_gtm_without_container_id() { + fn build_draft_config_does_not_name_gtm_without_container_id() { let url = Url::parse("https://publisher.example/path").expect("should parse URL"); let artifact = AuditArtifact { audited_url: url.to_string(), @@ -2804,7 +3026,11 @@ mod tests { let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) .expect("should build draft config"); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); + let parsed = toml::from_str::(&draft).expect("draft should parse as TOML"); + assert!( + parsed.get("tag").is_none(), + "should not select GTM without a container to configure it with" + ); assert!(draft.contains("Detected google_tag_manager")); } @@ -3799,10 +4025,6 @@ mod tests { /// values, so both source validation and runtime loading can be exercised. fn loadable_config() -> String { EXAMPLE_CONFIG - .replace( - "password = \"handler_password\"", - "password = \"test-admin-password-32-bytes-minimum\"", - ) .replace( "passphrase = \"ec_passphrase\"", "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", @@ -4245,10 +4467,6 @@ mod tests { fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") .expect("should write manifest"); let config = EXAMPLE_CONFIG - .replace( - "password = \"handler_password\"", - "password = \"test-admin-password-32-bytes-minimum\"", - ) .replace( "passphrase = \"ec_passphrase\"", "passphrase = \"test-ec-passphrase-32-bytes-minimum\"", @@ -4322,7 +4540,7 @@ mod tests { assert!(output.starts_with("--- configured creative opportunities\n")); assert!(output.contains("+++ generated creative opportunities\n")); assert!( - !output.contains("test-admin-password-32-bytes-minimum"), + !output.contains("test-proxy-secret-32-bytes-minimum"), "dry run must not expose unrelated secrets" ); assert!( diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs index 4334cb398d..5fa246bc43 100644 --- a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -53,6 +53,7 @@ pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult CliResult<()> { + crate::app_config::register_stock_modules(); let config: TrustedServerAppConfig = toml::from_str(source).map_err(|error| { error.span().map_or_else( || "config could not be loaded".to_string(), @@ -125,6 +126,7 @@ mod tests { ); let parsed: TrustedServerAppConfig = toml::from_str(&invalid).expect("should accept source syntax and schema"); + crate::app_config::register_stock_modules(); assert!( TrustedServerAppConfig::new(parsed.into_settings()).is_err(), "should exercise deploy validation" diff --git a/crates/trusted-server-cli/src/commands/cache/mod.rs b/crates/trusted-server-cli/src/commands/cache/mod.rs index 889852e10e..79bdb2b8ff 100644 --- a/crates/trusted-server-cli/src/commands/cache/mod.rs +++ b/crates/trusted-server-cli/src/commands/cache/mod.rs @@ -1,63 +1,54 @@ //! `ts cache` — operator control over shared template and origin caches. -pub mod purge; +pub(crate) mod purge; use clap::Subcommand; use crate::error::CliResult; +use crate::fastly_cli::FastlyProcess; /// Subcommands under `ts cache`. #[derive(Debug, Subcommand)] -pub enum CacheCommand { - /// Purge cached templates and tagged origin responses through a deployed service's admin endpoint. +pub(crate) enum CacheCommand { + /// Purge cached templates and tagged origin responses through the hosting platform's purge API. Purge(PurgeArgs), } /// Arguments for `ts cache purge`. /// -/// # Why this calls the service rather than the Fastly purge API +/// The platform is asked, so the credential that can empty the cache stays off the +/// publisher's domain. /// -/// The recorded Fastly API token is config- and secret-store write only, with no purge -/// permission (`adapter-fastly/src/management_api.rs`). The service's own admin endpoint -/// purges from inside the running service, where the platform SDK needs no API token at -/// all, so this path works with the credentials an operator already has. -/// -/// It also leaves one implementation of the purge. The surrogate key is derived -/// server-side from the URL, so the CLI cannot drift out of agreement with the cache it is -/// purging — a class of bug that a second client-side derivation would reintroduce. +/// The surrogate key is derived here by the function the service tags its cached objects +/// with, so the command cannot drift out of agreement with the cache it is purging. #[derive(Debug, clap::Args)] -pub struct PurgeArgs { - /// HTTPS base URL of the service; HTTP is permitted only for loopback development. - #[arg(long)] - pub service: String, +pub(crate) struct PurgeArgs { + /// The Fastly service whose cache is purged. + #[arg(long, value_name = "ID")] + pub(crate) service_id: String, /// Purge every cached template and tagged origin response. #[arg(long, conflicts_with = "page")] - pub all: bool, + pub(crate) all: bool, /// Purge one reader-facing page URL, including its exact scheme, host, and port. #[arg(long, conflicts_with = "all")] - pub page: Option, - - /// Admin username for the service's `^/_ts/admin` basic auth. - #[arg(long, default_value = "admin")] - pub username: String, + pub(crate) page: Option, } -/// Environment variable carrying the admin password. -/// -/// Read from the environment and never accepted as a flag: an argument is visible to every -/// other process on the host through `ps`, and lands in shell history. -pub const ADMIN_PASSWORD_ENVIRONMENT_VARIABLE: &str = "TRUSTED_SERVER_ADMIN_PASSWORD"; - -/// Run a `ts cache` subcommand. +/// Run a `ts cache` subcommand with the token from [`crate::fastly_cli::TOKEN_VARIABLE`]. /// /// # Errors /// -/// Returns an error when neither scope is given, when the admin password is absent from -/// the environment, when the service cannot be reached, or when it refuses the purge. -pub fn run(command: CacheCommand, out: &mut impl std::io::Write) -> CliResult<()> { +/// Returns an error when neither scope is given, when the token is absent from the +/// environment, or when the platform refuses the purge. +pub(crate) fn run(command: &CacheCommand, out: &mut impl std::io::Write) -> CliResult<()> { match command { - CacheCommand::Purge(args) => purge::run_purge(&args, out), + CacheCommand::Purge(args) => { + // Checked before the token is asked for, so a mistyped command is corrected + // without one. + purge::purge_target(args)?; + purge::run_purge(args, &FastlyProcess::from_environment()?, out) + } } } diff --git a/crates/trusted-server-cli/src/commands/cache/purge.rs b/crates/trusted-server-cli/src/commands/cache/purge.rs index dcfa38880b..ec41dbf2a9 100644 --- a/crates/trusted-server-cli/src/commands/cache/purge.rs +++ b/crates/trusted-server-cli/src/commands/cache/purge.rs @@ -1,268 +1,279 @@ -//! Calling a deployed service's cache-purge endpoint. +//! Asking the hosting platform to purge a surrogate key. -use std::time::Duration; +use trusted_server_core::platform::{ + TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, reader_url_surrogate_key, +}; -use serde::Deserialize; - -use crate::commands::cache::{ADMIN_PASSWORD_ENVIRONMENT_VARIABLE, PurgeArgs}; +use crate::commands::cache::PurgeArgs; use crate::error::{CliResult, cli_error}; +use crate::fastly_cli::{FastlyRunner, args}; -const PURGE_PATH: &str = "/_ts/admin/cache/purge"; -const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// What one purge names: the scope it is reported under and the surrogate key purged. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct PurgeTarget { + scope: &'static str, + surrogate_key: String, +} -/// The purge request body, as the endpoint's parser expects it. +/// Works out what `purge` asks for. +/// +/// # Errors /// -/// `scope: "all"` must carry no `url` field: the endpoint refuses that combination, on the -/// grounds that an operator who meant one page and mistyped the scope should not be handed -/// a silent full flush. -fn request_body(args: &PurgeArgs) -> CliResult { - match (args.all, args.page.as_deref()) { - (true, _) => Ok(r#"{"scope":"all"}"#.to_owned()), +/// Returns a message when neither scope is given, when `--page` is not an absolute +/// http(s) URL, or when the service id could be read as anything but an id. +pub(crate) fn purge_target(purge: &PurgeArgs) -> CliResult { + // A Fastly service id holds letters and digits only. + if purge.service_id.is_empty() || !purge.service_id.bytes().all(|b| b.is_ascii_alphanumeric()) { + return cli_error("--service-id must be a Fastly service id, of letters and digits only"); + } + match (purge.all, purge.page.as_deref()) { + (true, _) => Ok(PurgeTarget { + scope: "all", + surrogate_key: TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY.to_owned(), + }), + // A page that is not an absolute URL names a key nothing was stored with. (false, Some(page)) => { - if !reqwest::Url::parse(page).is_ok_and(|parsed| { + if !url::Url::parse(page).is_ok_and(|parsed| { matches!(parsed.scheme(), "http" | "https") && parsed.host_str().is_some() }) { return cli_error("--page must be an absolute http(s) URL with a host"); } - Ok(serde_json::json!({ "scope": "url", "url": page }).to_string()) + Ok(PurgeTarget { + scope: "url", + surrogate_key: reader_url_surrogate_key(page), + }) } + // A bare `ts cache purge` purges nothing. (false, None) => cli_error("specify --all or --page "), } } -/// Join the service base URL and the purge path without doubling or dropping a slash. -fn purge_endpoint(service: &str) -> CliResult { - let mut url = - reqwest::Url::parse(service).map_err(|_| "--service must be an absolute HTTPS URL")?; - crate::url_guard::require_credential_safe_transport(&url, "--service")?; - if url.host().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.query().is_some() - || url.fragment().is_some() - || !url.path().trim_matches('/').is_empty() - { - return cli_error( - "--service must be a base HTTPS URL without credentials, a path, query, or fragment", - ); - } - url.set_path(PURGE_PATH); - Ok(url.to_string()) -} - -/// The acknowledgment returned by the purge endpoint. -#[derive(Deserialize)] -struct PurgeAcknowledgment { - purged: bool, - scope: String, - surrogate_key: Option, -} - -/// Execute `ts cache purge`. +/// Execute `ts cache purge` through `runner`. /// /// # Errors /// -/// Returns an error when no scope is given, the admin password is missing from the -/// environment, the service cannot be reached, or the service answers with a non-success -/// status, redirects, or does not acknowledge the requested purge scope. -pub fn run_purge(args: &PurgeArgs, out: &mut impl std::io::Write) -> CliResult<()> { - let body = request_body(args)?; - let endpoint = purge_endpoint(&args.service)?; - - let password = std::env::var(ADMIN_PASSWORD_ENVIRONMENT_VARIABLE).map_err(|_| { - format!( - "set {ADMIN_PASSWORD_ENVIRONMENT_VARIABLE} to the service's admin password \ - (it is read from the environment, never a flag, so it does not reach `ps` \ - output or shell history)" +/// Returns an error when `purge` names nothing to purge, when the platform refuses the +/// purge, or when the result cannot be written. +pub(crate) fn run_purge( + purge: &PurgeArgs, + runner: &dyn FastlyRunner, + out: &mut impl std::io::Write, +) -> CliResult<()> { + let target = purge_target(purge)?; + + // By surrogate key for both scopes, never `fastly service purge --all`. + runner + .run( + &args(&[ + "service", + "purge", + &format!("--key={}", target.surrogate_key), + &format!("--service-id={}", purge.service_id), + "--non-interactive", + ]), + None, ) - })?; - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| format!("failed to start the HTTP runtime: {error}"))?; - - // Without this the first HTTPS request panics with "No provider set". - crate::tls::install_crypto_provider(); - - let (status, response_body) = runtime.block_on(async { - let client = reqwest::Client::builder() - .timeout(REQUEST_TIMEOUT) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|error| format!("failed to build the HTTP client: {error}"))?; - let response = client - .post(&endpoint) - .basic_auth(&args.username, Some(&password)) - .header("content-type", "application/json") - .body(body) - .send() - .await - .map_err(|error| format!("could not reach {endpoint}: {error}"))?; - let status = response.status(); - let text = response - .text() - .await - .map_err(|error| format!("could not read the purge acknowledgment: {error}"))?; - Ok::<_, String>((status, text)) - })?; - - if status.is_success() { - let acknowledgment: PurgeAcknowledgment = serde_json::from_str(&response_body) - .map_err(|error| format!("invalid purge acknowledgment: {error}"))?; - let expected_scope = if args.all { "all" } else { "url" }; - if !acknowledgment.purged || acknowledgment.scope != expected_scope { - return cli_error(format!( - "service did not acknowledge a successful {expected_scope} purge" - )); - } - if !args.all - && acknowledgment - .surrogate_key - .as_deref() - .is_none_or(|key| key.trim().is_empty()) - { - return cli_error("URL purge acknowledgment is missing its surrogate key"); - } - writeln!(out, "{response_body}") - .map_err(|error| format!("failed to write the purge result: {error}"))?; - return Ok(()); - } + .map_err(|message| format!("purge failed: {message}"))?; - // Named individually, because each one sends the operator somewhere different and a - // purge is usually run mid-incident. - let hint = match status.as_u16() { - 401 => " — check the admin username and $TRUSTED_SERVER_ADMIN_PASSWORD", - 404 => " — this service may predate the purge endpoint", - 501 => " — the template cache is Fastly-backed; this adapter cannot purge", - _ => "", - }; - cli_error(format!( - "purge failed: {status}{hint}\n{}", - response_body.trim() - )) + // A success acknowledges invalidation of the key, not that an object existed. + let report = serde_json::json!({ + "purged": true, + "scope": target.scope, + "surrogate_key": target.surrogate_key, + "service_id": purge.service_id, + }); + writeln!(out, "{report}").map_err(|error| format!("failed to write the purge result: {error}")) } #[cfg(test)] mod tests { + use std::sync::Mutex; + use super::*; - fn args(all: bool, page: Option<&str>) -> PurgeArgs { + const SERVICE: &str = "ExampleServiceId0123456"; + + /// Plays the `fastly` CLI, recording every invocation and failing when told to. + #[derive(Debug, Default)] + struct FakeFastly { + calls: Mutex>>, + failure: Option<&'static str>, + } + + impl FakeFastly { + fn calls(&self) -> Vec> { + self.calls.lock().expect("should lock").clone() + } + } + + impl FastlyRunner for FakeFastly { + fn run(&self, args: &[String], stdin: Option<&str>) -> Result { + assert!(stdin.is_none(), "a purge sends nothing on standard input"); + self.calls.lock().expect("should lock").push(args.to_vec()); + self.failure + .map_or_else(|| Ok(String::new()), |message| Err(message.to_owned())) + } + } + + fn purge_args(all: bool, page: Option<&str>) -> PurgeArgs { PurgeArgs { - service: "https://edge.example.com".to_owned(), + service_id: SERVICE.to_owned(), all, page: page.map(str::to_owned), - username: "admin".to_owned(), } } + /// The value joined to `name` with `=`. + fn flag<'a>(args: &'a [String], name: &str) -> Option<&'a str> { + args.iter() + .find_map(|arg| arg.strip_prefix(name)?.strip_prefix('=')) + } + + fn purge(fake: &FakeFastly, purge: &PurgeArgs) -> CliResult { + let mut out = Vec::new(); + run_purge(purge, fake, &mut out)?; + Ok(serde_json::from_slice(&out).expect("should report JSON")) + } + #[test] - fn invalid_page_urls_are_rejected_locally() { - for page in [ - "/article", - "example.com/article", - "ftp://example.com/article", - "", - ] { - assert!( - request_body(&args(false, Some(page))).is_err(), - "should reject {page:?}" - ); - } + fn purge_all_names_the_key_every_cached_object_is_tagged_with() { + let fake = FakeFastly::default(); + + let report = purge(&fake, &purge_args(true, None)).expect("should purge"); + + let calls = fake.calls(); + assert_eq!(calls.len(), 1, "one purge is one call: {calls:?}"); + assert_eq!(calls[0][..2], ["service", "purge"]); + assert_eq!(flag(&calls[0], "--key"), Some("ts-template")); + assert_eq!(flag(&calls[0], "--service-id"), Some(SERVICE)); + assert!( + !calls[0].iter().any(|arg| arg == "--all"), + "a purge of everything the service cached would reach untagged objects: {calls:?}" + ); assert!( - request_body(&args(false, Some("http://example.com/article"))).is_ok(), - "should allow HTTP reader URLs" + calls[0].iter().any(|arg| arg == "--non-interactive"), + "fastly must never wait on a prompt: {calls:?}" ); + assert_eq!(report["purged"], true); + assert_eq!(report["scope"], "all"); + assert_eq!(report["surrogate_key"], "ts-template"); } #[test] - fn purge_all_sends_no_url_field() { - // The endpoint refuses scope "all" carrying a url, so emitting one would make - // every --all run fail. + fn purge_page_names_the_key_the_service_tags_that_page_with() { + let fake = FakeFastly::default(); + let page = "https://example.com/article?b=2&a=1"; + + let report = purge(&fake, &purge_args(false, Some(page))).expect("should purge"); + + let expected = reader_url_surrogate_key(page); assert_eq!( - request_body(&args(true, None)).expect("should build"), - r#"{"scope":"all"}"# + flag(&fake.calls()[0], "--key"), + Some(expected.as_str()), + "should purge the key the service derives for the page" ); + assert_eq!(report["scope"], "url"); + assert_eq!(report["surrogate_key"], expected.as_str()); } #[test] - fn purge_page_sends_the_url_verbatim() { - // Canonicalization is the endpoint's job. Normalizing here too would give the two - // sides separate rules to drift apart. - let body = request_body(&args(false, Some("https://example.com/a?b=2&a=1"))) - .expect("should build"); + fn two_spellings_of_one_page_purge_one_key() { + let key_of = |page: &str| { + purge_target(&purge_args(false, Some(page))) + .expect("should name a key") + .surrogate_key + }; + assert_eq!( - body, - r#"{"scope":"url","url":"https://example.com/a?b=2&a=1"}"# + key_of("https://EXAMPLE.com:443/article/?b=2&a=1"), + key_of("https://example.com/article?a=1&b=2"), + "should canonicalize as the service does when it tags the page" + ); + assert_ne!( + key_of("http://example.com/article"), + key_of("https://example.com/article"), + "the scheme is part of the page a reader addresses" ); } #[test] - fn a_url_with_json_punctuation_is_escaped_rather_than_breaking_the_body() { - let body = request_body(&args(false, Some(r#"https://example.com/"; drop"#))) - .expect("should build"); - let parsed: serde_json::Value = serde_json::from_str(&body).expect("should stay valid"); - assert_eq!(parsed["url"], r#"https://example.com/"; drop"#); - } + fn invalid_page_urls_are_rejected_before_fastly_is_run() { + for page in [ + "/article", + "example.com/article", + "ftp://example.com/article", + "", + ] { + let fake = FakeFastly::default(); - #[test] - fn neither_scope_is_an_error_rather_than_a_default() { - // Defaulting to --all would make a bare `ts cache purge` flush production. - assert!(request_body(&args(false, None)).is_err()); - } + let error = + purge(&fake, &purge_args(false, Some(page))).expect_err("should refuse the page"); - #[test] - fn the_endpoint_url_survives_a_trailing_slash() { - for service in [ - "https://edge.example.com", - "https://edge.example.com/", - "https://edge.example.com///", - ] { - assert_eq!( - purge_endpoint(service).expect("should accept a secure base URL"), - "https://edge.example.com/_ts/admin/cache/purge", - "{service} must resolve to one well-formed endpoint" + assert!( + error.contains("--page"), + "should name the flag for {page:?}: {error}" ); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); } } #[test] - fn only_loopback_services_may_use_plaintext_http() { - for service in [ - "http://127.0.0.1:8080", - "http://[::1]:8080", - "http://localhost:8080", - ] { - assert!( - purge_endpoint(service).is_ok(), - "should permit loopback development: {service}" - ); - } - for service in [ - "http://192.0.2.1", - "http://[2001:db8::1]", - "http://localhost.example.com", - ] { + fn every_value_is_joined_to_its_flag() { + let fake = FakeFastly::default(); + + purge( + &fake, + &purge_args(false, Some("https://example.com/article")), + ) + .expect("should purge"); + + for arg in &fake.calls()[0][2..] { assert!( - purge_endpoint(service).is_err(), - "should refuse remote plaintext transport: {service}" + arg.starts_with("--"), + "a value standing alone can be read as a flag: {:?}", + fake.calls() ); } } #[test] - fn service_urls_cannot_redirect_the_purge_path_or_embed_credentials() { - for service in [ - "not a URL", - "https://example.com/path", - "https://example.com?query=1", - "https://example.com#fragment", - "https://user:example-password@example.com", - ] { - assert!( - purge_endpoint(service).is_err(), - "should require an unambiguous service base URL" - ); + fn neither_scope_is_an_error_rather_than_a_default() { + let fake = FakeFastly::default(); + + let error = purge(&fake, &purge_args(false, None)).expect_err("should refuse"); + + assert!(error.contains("--all"), "should name the choices: {error}"); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); + } + + #[test] + fn a_service_id_that_fastly_could_read_as_a_flag_is_refused() { + for service_id in ["", "--all", "-s", "abc def", "abc/def"] { + let fake = FakeFastly::default(); + let refused = PurgeArgs { + service_id: service_id.to_owned(), + all: true, + page: None, + }; + + let error = purge(&fake, &refused).expect_err("should refuse the service id"); + + assert!(error.contains("--service-id"), "{error}"); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); } } + + #[test] + fn a_refused_purge_reports_what_the_platform_said_and_prints_no_result() { + let fake = FakeFastly { + failure: Some("`fastly service purge` failed: 403 Forbidden"), + ..FakeFastly::default() + }; + let mut out = Vec::new(); + + let error = run_purge(&purge_args(true, None), &fake, &mut out).expect_err("should fail"); + + assert!(error.contains("403 Forbidden"), "{error}"); + assert!(out.is_empty(), "should not report a successful purge"); + } } diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs index cfb2d13972..20f96b5f51 100644 --- a/crates/trusted-server-cli/src/commands/config/ad_templates.rs +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -183,18 +183,21 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin .map_err(output_error)?; writeln!( out, - "auction.providers: {}", - if loaded.settings.auction.providers.is_empty() { + "demand.modules: {}", + if loaded.settings.demand.selected().is_empty() { "(none)".to_string() } else { - loaded + // Sorted rather than in selection order, so the report reads the + // same however the document happens to be written. + let mut names = loaded .settings - .auction - .providers - .keys() - .map(|id| escape_terminal_text(id.as_str()).into_owned()) - .collect::>() - .join(", ") + .demand + .selected() + .into_iter() + .map(|name| escape_terminal_text(name).into_owned()) + .collect::>(); + names.sort_unstable(); + names.join(", ") } ) .map_err(output_error)?; @@ -213,10 +216,10 @@ fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), Strin "status: slots are configured, but [auction].enabled is false" ) .map_err(output_error)?; - } else if loaded.settings.auction.providers.is_empty() { + } else if loaded.settings.demand.selected().is_empty() { writeln!( out, - "status: slots are configured, but [auction].providers is empty" + "status: slots are configured, but [demand] modules is empty" ) .map_err(output_error)?; } else { @@ -351,7 +354,7 @@ fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), .creative_opportunities .as_ref() .is_some_and(|config| config.enabled); - let providers_configured = !loaded.settings.auction.providers.is_empty(); + let providers_configured = !loaded.settings.demand.selected().is_empty(); let gate = evaluate_ad_stack_gate(AdStackGateInput { method_get: method_pass, @@ -666,10 +669,14 @@ mod tests { .as_table_mut() .expect("should find the auction table"); auction.insert("enabled".to_string(), toml::Value::Boolean(true)); - // Remove the provider map and its bidder references structurally to - // isolate the runtime verdict from the provider advisory. - auction.remove("providers"); + // Remove the bidder references, and the demand sources the auction + // would route to, structurally, to isolate the runtime verdict from the + // provider advisory. auction.remove("bidders"); + config_value + .as_table_mut() + .expect("should find the document root") + .remove("demand"); let config_text = toml::to_string(&config_value).expect("should serialize the fixture config"); let (_temp, config) = project_with_config(&config_text); @@ -702,18 +709,35 @@ mod tests { #[test] fn lint_reports_configured_slot_count_and_auction_state() { - // Insert the alphabetically earlier provider after pbs-main in source - // order, so the output must sort identifiers rather than preserve input. - let config_text = format!( - "{}\n[auction.providers.aps-main]\n\ - protocol = \"openrtb-2.6\"\n\ - profile = \"aps\"\n\ - endpoint = \"https://aps.example.com/e/pb/bid\"\n\ - routing = \"all_eligible\"\n\ - [auction.providers.aps-main.profile_config]\n\ - account_id = \"example-aps-account-id\"\n", - config_with_slots() - ); + // The example config selects `pbs_main`. Naming the alphabetically + // earlier `aps_main` after it means the output must sort the names + // rather than preserve the order they were selected in. + let mut config_value: toml::Value = + toml::from_str(&config_with_slots()).expect("should parse the fixture config"); + let root = config_value + .as_table_mut() + .expect("should find the document root"); + let demand = root + .get_mut("demand") + .and_then(toml::Value::as_table_mut) + .expect("should find the demand table"); + demand + .get_mut("modules") + .and_then(toml::Value::as_array_mut) + .expect("should find the demand selection") + .push(toml::Value::String("aps_main".to_string())); + let mut aps = toml::value::Table::new(); + for (key, value) in [ + ("implementation", "aps"), + ("endpoint", "https://aps.example.com/e/pb/bid"), + ("routing", "all_eligible"), + ("account_id", "example-aps-account-id"), + ] { + aps.insert(key.to_string(), toml::Value::String(value.to_string())); + } + demand.insert("aps_main".to_string(), toml::Value::Table(aps)); + let config_text = + toml::to_string(&config_value).expect("should serialize the fixture config"); let (_temp, config) = project_with_config(&config_text); let mut out = Vec::new(); @@ -735,9 +759,9 @@ mod tests { assert_eq!( output .lines() - .find(|line| line.starts_with("auction.providers:")), - Some("auction.providers: aps-main, pbs-main"), - "should report provider identifiers in deterministic order: {output}" + .find(|line| line.starts_with("demand.modules:")), + Some("demand.modules: aps_main, pbs_main"), + "should report demand names in deterministic order: {output}" ); assert!(!output.contains("legacy fallback")); } @@ -750,8 +774,11 @@ mod tests { .as_table_mut() .expect("should find auction table"); auction.insert("enabled".to_string(), toml::Value::Boolean(true)); - auction.remove("providers"); auction.remove("bidders"); + config_value + .as_table_mut() + .expect("should find the document root") + .remove("demand"); let config_text = toml::to_string(&config_value).expect("should serialize the fixture config"); let (_temp, config) = project_with_config(&config_text); @@ -767,14 +794,14 @@ mod tests { assert_eq!( output .lines() - .find(|line| line.starts_with("auction.providers:")), - Some("auction.providers: (none)"), - "should report no providers" + .find(|line| line.starts_with("demand.modules:")), + Some("demand.modules: (none)"), + "should report no demand sources" ); assert!( - output.lines().any( - |line| line == "status: slots are configured, but [auction].providers is empty" - ), + output + .lines() + .any(|line| line == "status: slots are configured, but [demand] modules is empty"), "should explain why configured slots are ineligible: {output}" ); } diff --git a/crates/trusted-server-cli/src/commands/config/init.rs b/crates/trusted-server-cli/src/commands/config/init.rs index 72c31e65ae..ed31702b1e 100644 --- a/crates/trusted-server-cli/src/commands/config/init.rs +++ b/crates/trusted-server-cli/src/commands/config/init.rs @@ -82,15 +82,23 @@ mod tests { let config = fs::read_to_string(&path).expect("should read initialized config"); let parsed = toml::from_str::(&config).expect("config should parse as TOML"); - assert!( - parsed["integrations"]["js_asset_proxy"] - .get("cache_ttl_seconds") - .is_none(), - "JS asset proxy should preserve upstream cache policy by default" - ); + for section in [ + "cmp", + "tag", + "ad-tag", + "bot-protection", + "identity", + "audience", + "framework", + ] { + assert!( + parsed.get(section).is_none(), + "no module should run until an operator selects one, but [{section}] is present" + ); + } assert!( config.contains( - "[integrations.js_asset_proxy]\nenabled = false\n# Uncomment to override upstream cache headers for every asset below.\n# This replaces upstream directives, including private and no-store.\n# Use only when each asset's bytes are identical for every visitor.\n# cache_ttl_seconds = 3600\n# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n# that vary by browser User-Agent or use integrity hashes for UA-specific bytes." + "# [proxy.js_asset_proxy]\n# Uncomment to override upstream cache headers for every asset below.\n# This replaces upstream directives, including private and no-store.\n# Use only when each asset's bytes are identical for every visitor.\n# cache_ttl_seconds = 3600\n# Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets\n# that vary by browser User-Agent or use integrity hashes for UA-specific bytes." ), "initialized config should document the optional cache override" ); diff --git a/crates/trusted-server-cli/src/commands/keys.rs b/crates/trusted-server-cli/src/commands/keys.rs new file mode 100644 index 0000000000..0da57125eb --- /dev/null +++ b/crates/trusted-server-cli/src/commands/keys.rs @@ -0,0 +1,917 @@ +//! `ts keys`, the request signing key commands. +//! +//! An operator holding a Fastly API token rotates and retires the keys the +//! service signs with. The commands write the stores through the `fastly` +//! CLI, with the token taken from `FASTLY_API_TOKEN` and passed to the +//! `fastly` process in its environment, and a private key reaches the +//! `fastly` process on its standard input, never in its arguments. +//! +//! The rotation library in core takes a read that fails for an entry that +//! does not exist. Every read here is a call to the platform, which can fail +//! for other reasons, so the stores tell the two apart and write nothing once +//! a read has failed. + +use std::io::Write as _; +use std::sync::{Arc, OnceLock}; + +use error_stack::Report; +use trusted_server_core::platform::{ + BackendNamingPolicy, ClientInfo, DisabledGeo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformSecretStore, RuntimeServices, StoreId, StoreName, + UnavailableHttpClient, UnavailableKvStore, +}; +use trusted_server_core::request_signing::rotation::KeyRotationManager; +use trusted_server_core::request_signing::{kid_is_creatable, kid_is_well_formed}; + +use crate::fastly_cli::{FastlyProcess, FastlyRunner}; + +/// How the `fastly` CLI begins its report that the API found nothing at the +/// address it was asked for. +/// +/// Only the test fake is known to produce this wording. No recorded run of the +/// `fastly` CLI has, and the live check saw `401` alone. +const API_NOT_FOUND: &str = "the Fastly API returned 404"; + +/// The entries of the config store that are not keys. A key named as one of +/// them would be written over, or would take the entry with it when deleted. +const BOOKKEEPING_ENTRIES: [&str; 2] = ["current-kid", "active-kids"]; + +/// Request signing key commands. +#[derive(Debug, clap::Subcommand)] +pub(crate) enum KeysCommand { + /// Create a signing key and make it the one the service signs with. + Rotate(RotateArgs), + /// Take a key out of the active set, and delete it with `--delete`. + Deactivate(DeactivateArgs), +} + +/// The two stores request signing keeps its keys in. +#[derive(Debug, clap::Args)] +pub(crate) struct StoreArgs { + /// The Fastly config store the service links as `jwks_store`, holding the + /// public keys, `current-kid` and `active-kids`. + #[arg(long, value_name = "ID")] + config_store_id: String, + /// The Fastly secret store the service links as `signing_keys`, holding + /// the private keys. + #[arg(long, value_name = "ID")] + secret_store_id: String, +} + +/// Arguments for `ts keys rotate`. +#[derive(Debug, clap::Args)] +pub(crate) struct RotateArgs { + #[command(flatten)] + stores: StoreArgs, + /// The new key's id, `ts-` when not given. It starts with a lower + /// case letter, holds only letters, digits, `-`, `_`, `.` and `:`, and is + /// at most 128 characters. + #[arg(long)] + kid: Option, +} + +/// Arguments for `ts keys deactivate`. +#[derive(Debug, clap::Args)] +pub(crate) struct DeactivateArgs { + #[command(flatten)] + stores: StoreArgs, + /// The key to take out of the active set. The current key is refused, so + /// rotate first. + #[arg(long)] + kid: String, + /// Also delete the key from both stores. + #[arg(long)] + delete: bool, +} + +/// The `fastly` CLI as the two stores of one command use it. +struct Platform { + runner: Arc, + /// The first read that failed for a reason other than a missing entry. + failed_read: OnceLock, +} + +impl Platform { + /// Runs `fastly ` on the store `store_id`, with `flags` + /// after the store id. + /// + /// A value is joined to its flag with `=`, so that `fastly` never takes a + /// value beginning with `-` for a flag of its own. + fn run( + &self, + command: &str, + action: &str, + store_id: &str, + flags: &[String], + stdin: Option<&str>, + ) -> Result { + let mut arguments = vec![ + command.to_owned(), + action.to_owned(), + format!("--store-id={store_id}"), + ]; + arguments.extend_from_slice(flags); + arguments.push("--non-interactive".to_owned()); + self.runner.run(&arguments, stdin) + } + + /// Refuses a write once a read has failed. + /// + /// The rotation library would otherwise take the failed read for an empty + /// store, and write a key list that drops the keys already active or + /// retire the key the service signs with. + fn writable(&self, kind: PlatformError) -> Result<(), Report> { + match self.failed_read.get() { + Some(message) => Err(Report::new(kind).attach(format!( + "nothing is written, because the config store could not be read: {message}" + ))), + None => Ok(()), + } + } + + /// Deletes one entry of a store, and takes an entry that is already gone + /// for deleted, so that a delete tried again after a partial failure + /// finishes. + /// + /// The API answers that it found nothing for a missing store as well, so + /// the store is asked for before a missing entry is taken for deleted. + fn delete(&self, store: &str, store_id: &str, entry: String) -> Result<(), String> { + let deleted = self.run( + &format!("{store}-entry"), + "delete", + store_id, + &[entry, "--auto-yes".to_owned()], + None, + ); + match deleted { + Ok(_) => Ok(()), + Err(message) if message.contains(API_NOT_FOUND) => self + .run(store, "describe", store_id, &["--json".to_owned()], None) + .map(drop) + .map_err(|_| message), + Err(message) => Err(message), + } + } +} + +/// The value in what `fastly config-store-entry describe --json` printed. +fn item_value(output: &str, key: &str) -> Result { + let entry: serde_json::Value = serde_json::from_str(output).map_err(|error| { + format!("`fastly config-store-entry describe` did not answer JSON: {error}") + })?; + entry + .get("item_value") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| format!("the entry `{key}` has no value")) +} + +/// The config store, read and written through the `fastly` CLI. +/// +/// Reads name the store the service links, which is not an id the API +/// knows, so every read goes to the store the command was given. +struct FastlyConfigStore { + platform: Arc, + store_id: String, +} + +impl PlatformConfigStore for FastlyConfigStore { + fn get(&self, _store_name: &StoreName, key: &str) -> Result> { + self.platform + .run( + "config-store-entry", + "describe", + &self.store_id, + &[format!("--key={key}"), "--json".to_owned()], + None, + ) + .and_then(|output| item_value(&output, key)) + .map_err(|message| { + if !message.contains(API_NOT_FOUND) { + let _ = self.platform.failed_read.set(message.clone()); + } + Report::new(PlatformError::ConfigStore).attach(message) + }) + } + + fn put(&self, store_id: &StoreId, key: &str, value: &str) -> Result<(), Report> { + self.platform.writable(PlatformError::ConfigStore)?; + self.platform + .run( + "config-store-entry", + "update", + store_id.as_ref(), + &[ + format!("--key={key}"), + "--stdin".to_owned(), + "--upsert".to_owned(), + ], + Some(value), + ) + .map(drop) + .map_err(|message| Report::new(PlatformError::ConfigStore).attach(message)) + } + + fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + self.platform.writable(PlatformError::ConfigStore)?; + self.platform + .delete("config-store", store_id.as_ref(), format!("--key={key}")) + .map_err(|message| Report::new(PlatformError::ConfigStore).attach(message)) + } +} + +/// The secret store, written through the `fastly` CLI and never read, because +/// the API does not give a secret back. +struct FastlySecretStore { + platform: Arc, +} + +impl PlatformSecretStore for FastlySecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + Err(Report::new(PlatformError::Unsupported) + .attach("a secret cannot be read back from the platform")) + } + + fn create( + &self, + store_id: &StoreId, + name: &str, + value: &str, + ) -> Result<(), Report> { + self.platform.writable(PlatformError::SecretStore)?; + self.platform + .run( + "secret-store-entry", + "create", + store_id.as_ref(), + &[format!("--name={name}"), "--stdin".to_owned()], + Some(value), + ) + .map(drop) + .map_err(|message| Report::new(PlatformError::SecretStore).attach(message)) + } + + fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + self.platform.writable(PlatformError::SecretStore)?; + self.platform + .delete("secret-store", store_id.as_ref(), format!("--name={name}")) + .map_err(|message| Report::new(PlatformError::SecretStore).attach(message)) + } +} + +/// A backend nothing here asks for. +struct NoBackend; + +impl PlatformBackend for NoBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// The services the rotation library writes through, for these stores. +fn services(runner: Arc, stores: &StoreArgs) -> RuntimeServices { + let platform = Arc::new(Platform { + runner, + failed_read: OnceLock::new(), + }); + RuntimeServices::builder() + .config_store(Arc::new(FastlyConfigStore { + platform: Arc::clone(&platform), + store_id: stores.config_store_id.clone(), + })) + .secret_store(Arc::new(FastlySecretStore { platform })) + .kv_store(Arc::new(UnavailableKvStore)) + .backend(Arc::new(NoBackend)) + .http_client(Arc::new(UnavailableHttpClient)) + .geo(Arc::new(DisabledGeo)) + .client_info(ClientInfo::default()) + .build() +} + +/// Refuses a key id that names one of the store's bookkeeping entries. +fn refuse_bookkeeping_entry(kid: &str) -> Result<(), String> { + if BOOKKEEPING_ENTRIES.contains(&kid) { + return Err(format!( + "kid `{kid}` names an entry the store keeps its key list in, and cannot name a key" + )); + } + Ok(()) +} + +/// Runs a `ts keys` command with the token from +/// [`crate::fastly_cli::TOKEN_VARIABLE`]. +/// +/// # Errors +/// +/// A message when the token is not set or the command fails. +pub(crate) fn run(command: &KeysCommand) -> Result<(), String> { + let report = run_with(command, Arc::new(FastlyProcess::from_environment()?))?; + writeln!(std::io::stdout().lock(), "{report}") + .map_err(|error| format!("the report could not be written: {error}")) +} + +/// Runs a `ts keys` command through `runner` and returns what it reports, as +/// JSON. +/// +/// # Errors +/// +/// A message when the key id is refused, a store cannot be read or a store +/// write fails. +pub(crate) fn run_with( + command: &KeysCommand, + runner: Arc, +) -> Result { + match command { + KeysCommand::Rotate(rotate) => { + if let Some(kid) = rotate.kid.as_deref() { + if !kid_is_creatable(kid) { + return Err(format!( + "kid `{kid}` must start with a lower case letter, hold only letters, \ + digits, `-`, `_`, `.` and `:`, and be at most 128 characters" + )); + } + refuse_bookkeeping_entry(kid)?; + } + let services = services(runner, &rotate.stores); + let manager = KeyRotationManager::new( + &rotate.stores.config_store_id, + &rotate.stores.secret_store_id, + ); + let result = manager + .rotate_key(&services, rotate.kid.clone()) + .map_err(|report| format!("the key was not rotated: {report:?}"))?; + let jwk = serde_json::to_value(&result.jwk) + .map_err(|error| format!("the new public key could not be written: {error}"))?; + Ok(serde_json::json!({ + "new_kid": result.new_kid, + "previous_kid": result.previous_kid, + "active_kids": result.active_kids, + "jwk": jwk, + }) + .to_string()) + } + KeysCommand::Deactivate(deactivate) => { + if !kid_is_well_formed(&deactivate.kid) { + return Err(format!( + "kid `{}` must hold only letters, digits, `-`, `_`, `.` and `:`, and be 1 \ + to 128 characters", + deactivate.kid + )); + } + refuse_bookkeeping_entry(&deactivate.kid)?; + let services = services(runner, &deactivate.stores); + let manager = KeyRotationManager::new( + &deactivate.stores.config_store_id, + &deactivate.stores.secret_store_id, + ); + if deactivate.delete { + manager.delete_key(&services, &deactivate.kid) + } else { + manager.deactivate_key(&services, &deactivate.kid) + } + .map_err(|report| format!("the key was not deactivated: {report:?}"))?; + let remaining = manager.list_active_keys(&services).map_err(|report| { + format!( + "the key was retired, and the active keys could not then be read: {report:?}" + ) + })?; + Ok(serde_json::json!({ + "deactivated_kid": deactivate.kid, + "deleted": deactivate.delete, + "remaining_active_kids": remaining, + }) + .to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + use std::sync::Mutex; + + use super::*; + + const CONFIG_STORE: &str = "config-store-id"; + const SECRET_STORE: &str = "secret-store-id"; + + /// One `fastly` invocation as the fake saw it. + #[derive(Debug, Clone)] + struct Call { + args: Vec, + stdin: Option, + } + + impl Call { + /// Whether the invocation changes a store. + fn writes(&self) -> bool { + matches!(self.args[1].as_str(), "update" | "create" | "delete") + } + } + + /// Plays the `fastly` CLI against two stores held in memory, recording + /// every invocation. It answers a missing store or entry as the real one + /// does, with the API's 404. + #[derive(Debug, Default)] + struct FakeFastly { + config: Mutex>, + secrets: Mutex>, + calls: Mutex>, + /// Config keys whose next read fails as a network fault would. + failing_reads: Mutex>, + } + + impl FakeFastly { + fn with_config(entries: &[(&str, &str)]) -> Self { + let fake = Self::default(); + fake.config.lock().expect("should lock").extend( + entries + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())), + ); + fake + } + + fn config(&self, key: &str) -> Option { + self.config.lock().expect("should lock").get(key).cloned() + } + + fn has_secret(&self, name: &str) -> bool { + self.secrets.lock().expect("should lock").contains_key(name) + } + + fn add_secret(&self, name: &str) { + self.secrets + .lock() + .expect("should lock") + .insert(name.to_owned(), "private".to_owned()); + } + + /// Makes the next read of `key` fail for a reason other than the + /// entry being absent. + fn fail_next_read_of(&self, key: &str) { + self.failing_reads + .lock() + .expect("should lock") + .insert(key.to_owned()); + } + + fn calls(&self) -> Vec { + self.calls.lock().expect("should lock").clone() + } + + fn writes(&self) -> Vec { + self.calls().into_iter().filter(Call::writes).collect() + } + } + + /// The value joined to `name` with `=`, which is the only form the + /// command may pass a value in. + fn flag<'a>(args: &'a [String], name: &str) -> Option<&'a str> { + args.iter() + .find_map(|arg| arg.strip_prefix(name)?.strip_prefix('=')) + } + + fn not_found(what: &str) -> String { + format!("`fastly` failed: ERROR: the Fastly API returned 404 Not Found: {what}") + } + + impl FastlyRunner for FakeFastly { + fn run(&self, args: &[String], stdin: Option<&str>) -> Result { + self.calls.lock().expect("should lock").push(Call { + args: args.to_vec(), + stdin: stdin.map(str::to_owned), + }); + let command = (args[0].as_str(), args[1].as_str()); + let store = flag(args, "--store-id").unwrap_or_default(); + match command { + ("config-store-entry" | "config-store", _) if store != CONFIG_STORE => { + Err(not_found("no such config store")) + } + ("secret-store-entry" | "secret-store", _) if store != SECRET_STORE => { + Err(not_found("no such secret store")) + } + ("config-store" | "secret-store", "describe") => Ok("{}".to_owned()), + ("config-store-entry", "describe") => { + let key = flag(args, "--key").unwrap_or_default(); + if self.failing_reads.lock().expect("should lock").remove(key) { + return Err("`fastly` failed: ERROR: network is unreachable".to_owned()); + } + self.config(key) + .map(|value| serde_json::json!({ "item_value": value }).to_string()) + .ok_or_else(|| not_found("no such entry")) + } + ("config-store-entry", "update") => { + let key = flag(args, "--key").unwrap_or_default(); + self.config + .lock() + .expect("should lock") + .insert(key.to_owned(), stdin.unwrap_or_default().to_owned()); + Ok(String::new()) + } + ("config-store-entry", "delete") => { + let key = flag(args, "--key").unwrap_or_default(); + self.config + .lock() + .expect("should lock") + .remove(key) + .map(|_| String::new()) + .ok_or_else(|| not_found("no such entry")) + } + ("secret-store-entry", "create") => { + let name = flag(args, "--name").unwrap_or_default(); + self.secrets + .lock() + .expect("should lock") + .insert(name.to_owned(), stdin.unwrap_or_default().to_owned()); + Ok(String::new()) + } + ("secret-store-entry", "delete") => { + let name = flag(args, "--name").unwrap_or_default(); + self.secrets + .lock() + .expect("should lock") + .remove(name) + .map(|_| String::new()) + .ok_or_else(|| not_found("no such secret")) + } + _ => Err(format!("unexpected fastly {args:?}")), + } + } + } + + fn stores() -> StoreArgs { + StoreArgs { + config_store_id: CONFIG_STORE.to_owned(), + secret_store_id: SECRET_STORE.to_owned(), + } + } + + fn rotate(fake: &Arc, kid: Option<&str>) -> Result { + let command = KeysCommand::Rotate(RotateArgs { + stores: stores(), + kid: kid.map(str::to_owned), + }); + run_with(&command, Arc::clone(fake) as Arc) + .map(|report| serde_json::from_str(&report).expect("should report JSON")) + } + + fn deactivate_in( + fake: &Arc, + stores: StoreArgs, + kid: &str, + delete: bool, + ) -> Result { + let command = KeysCommand::Deactivate(DeactivateArgs { + stores, + kid: kid.to_owned(), + delete, + }); + run_with(&command, Arc::clone(fake) as Arc) + .map(|report| serde_json::from_str(&report).expect("should report JSON")) + } + + fn deactivate( + fake: &Arc, + kid: &str, + delete: bool, + ) -> Result { + deactivate_in(fake, stores(), kid, delete) + } + + /// Two active keys, `ts-new` the current one, with `ts-old` held in both + /// stores. + fn two_keys() -> Arc { + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-new"), + ("active-kids", "ts-old,ts-new"), + ("ts-old", "{}"), + ])); + fake.add_secret("ts-old"); + fake + } + + #[test] + fn rotate_stores_a_new_key_and_makes_it_current() { + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-old"), + ("active-kids", "ts-old"), + ("ts-old", "{}"), + ])); + + let report = rotate(&fake, Some("ts-new")).expect("should rotate"); + + assert_eq!(report["new_kid"], "ts-new"); + assert_eq!(report["previous_kid"], "ts-old"); + assert_eq!(fake.config("current-kid").as_deref(), Some("ts-new")); + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-old,ts-new")); + let jwk: serde_json::Value = + serde_json::from_str(&fake.config("ts-new").expect("should store the public key")) + .expect("should store the public key as JSON"); + assert_eq!(jwk["kid"], "ts-new"); + assert_eq!(report["jwk"], jwk, "should report the stored public key"); + assert!( + fake.secrets + .lock() + .expect("should lock") + .get("ts-new") + .is_some_and(|secret| !secret.is_empty()), + "should store the private key in the secret store" + ); + } + + #[test] + fn the_first_key_of_an_empty_store_has_no_previous_key() { + let fake = Arc::new(FakeFastly::default()); + + let report = rotate(&fake, Some("ts-new")).expect("should rotate"); + + assert_eq!(report["previous_kid"], serde_json::Value::Null); + assert_eq!(fake.config("current-kid").as_deref(), Some("ts-new")); + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-new")); + } + + #[test] + fn the_private_key_reaches_fastly_only_on_its_standard_input() { + let fake = Arc::new(FakeFastly::default()); + + rotate(&fake, Some("ts-new")).expect("should rotate"); + + let secret = fake + .secrets + .lock() + .expect("should lock") + .get("ts-new") + .cloned() + .expect("should store the private key"); + for call in fake.calls() { + assert!( + call.args.iter().all(|arg| !arg.contains(&secret)), + "a private key in the arguments is visible to every process: {:?}", + call.args + ); + assert!( + call.args.iter().any(|arg| arg == "--non-interactive"), + "fastly must never wait on a prompt: {:?}", + call.args + ); + } + let create = fake + .calls() + .into_iter() + .find(|call| call.args[..2] == ["secret-store-entry", "create"]) + .expect("should create the secret"); + assert_eq!(create.stdin.as_deref(), Some(secret.as_str())); + } + + #[test] + fn rotate_names_a_key_by_date_when_given_no_kid() { + let fake = Arc::new(FakeFastly::default()); + + let report = rotate(&fake, None).expect("should rotate"); + + let kid = report["new_kid"].as_str().expect("should name the key"); + assert!(kid.starts_with("ts-"), "{kid}"); + assert!(kid_is_creatable(kid), "{kid}"); + } + + #[test] + fn rotate_refuses_a_kid_no_platform_can_store_before_writing_anything() { + let too_long = "a".repeat(129); + for kid in ["Kid", "_kid", "-kid", "kid with space", "", &too_long] { + let fake = Arc::new(FakeFastly::default()); + + let error = rotate(&fake, Some(kid)).expect_err("should refuse the kid"); + + assert!( + error.contains("must start with a lower case letter"), + "{error}" + ); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); + } + } + + #[test] + fn rotate_refuses_a_kid_that_is_already_active() { + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-old"), + ("active-kids", "ts-old"), + ])); + + let error = rotate(&fake, Some("ts-old")).expect_err("should refuse a used kid"); + + assert!(error.contains("already exists"), "{error}"); + assert_eq!(fake.config("current-kid").as_deref(), Some("ts-old")); + } + + #[test] + fn a_key_is_never_named_as_one_of_the_bookkeeping_entries() { + // On an empty store nothing else would refuse these. The public key + // would be stored as `current-kid` and then written over by the + // pointer of the same name. + for kid in BOOKKEEPING_ENTRIES { + let fake = Arc::new(FakeFastly::default()); + + let error = rotate(&fake, Some(kid)).expect_err("should refuse the kid"); + + assert!(error.contains("cannot name a key"), "{error}"); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); + } + } + + #[test] + fn a_bookkeeping_entry_is_never_retired_as_a_key() { + // Deleting either would take the service's key list with it. + for kid in BOOKKEEPING_ENTRIES { + let fake = two_keys(); + + let error = deactivate(&fake, kid, true).expect_err("should refuse the kid"); + + assert!(error.contains("cannot name a key"), "{error}"); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); + assert!(fake.config(kid).is_some(), "should keep `{kid}`"); + } + } + + #[test] + fn deactivate_refuses_a_kid_that_is_not_a_key_id_before_asking_fastly_anything() { + let too_long = "a".repeat(129); + for kid in ["", "ts old", "ts-old,ts-new", "ts/old", &too_long] { + let fake = two_keys(); + + let error = deactivate(&fake, kid, true).expect_err("should refuse the kid"); + + assert!(error.contains("must hold only"), "{error}"); + assert!(fake.calls().is_empty(), "nothing should be asked of fastly"); + } + } + + #[test] + fn a_key_made_under_an_earlier_rule_can_still_be_retired() { + // Neither id could be given to a new key. Both begin with a + // character `fastly` would read as a flag if it stood alone. + for kid in ["-legacy", "9legacy"] { + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-new"), + ("active-kids", &format!("{kid},ts-new")), + (kid, "{}"), + ])); + fake.add_secret(kid); + + deactivate(&fake, kid, true).expect("should retire the key"); + + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-new")); + assert!(fake.config(kid).is_none(), "should delete the public key"); + assert!(!fake.has_secret(kid), "should delete the private key"); + } + } + + #[test] + fn every_value_is_joined_to_its_flag() { + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-new"), + ("active-kids", "-legacy,ts-new"), + ("-legacy", "{}"), + ])); + fake.add_secret("-legacy"); + + rotate(&fake, Some("ts-newer")).expect("should rotate"); + deactivate(&fake, "-legacy", true).expect("should retire the key"); + + for call in fake.calls() { + for arg in &call.args[2..] { + assert!( + arg.starts_with("--"), + "a value standing alone can be read as a flag: {:?}", + call.args + ); + } + } + } + + #[test] + fn deactivate_takes_a_key_out_of_the_active_set_and_keeps_its_material() { + let fake = two_keys(); + + let report = deactivate(&fake, "ts-old", false).expect("should deactivate"); + + assert_eq!( + report["remaining_active_kids"], + serde_json::json!(["ts-new"]) + ); + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-new")); + assert!( + fake.config("ts-old").is_some(), + "should keep the public key" + ); + assert!(fake.has_secret("ts-old"), "should keep the private key"); + } + + #[test] + fn deactivate_with_delete_removes_the_key_from_both_stores() { + let fake = two_keys(); + + deactivate(&fake, "ts-old", true).expect("should delete"); + + assert!( + fake.config("ts-old").is_none(), + "should delete the public key" + ); + assert!(!fake.has_secret("ts-old"), "should delete the private key"); + } + + #[test] + fn deactivate_refuses_the_current_key() { + let fake = two_keys(); + + let error = deactivate(&fake, "ts-new", false).expect_err("should refuse the current key"); + + assert!(error.contains("current signing key"), "{error}"); + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-old,ts-new")); + } + + #[test] + fn a_delete_tried_again_finishes_when_part_of_the_key_is_already_gone() { + // The first attempt deleted the private key and failed before the + // public one. + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-new"), + ("active-kids", "ts-new"), + ("ts-old", "{}"), + ])); + + deactivate(&fake, "ts-old", true).expect("should finish the delete"); + + assert!( + fake.config("ts-old").is_none(), + "should delete the public key" + ); + + deactivate(&fake, "ts-old", true).expect("should have nothing left to delete"); + } + + #[test] + fn a_missing_store_is_not_taken_for_a_deleted_key() { + let fake = two_keys(); + let wrong_secret_store = StoreArgs { + config_store_id: CONFIG_STORE.to_owned(), + secret_store_id: "another-store-id".to_owned(), + }; + + let error = deactivate_in(&fake, wrong_secret_store, "ts-old", true) + .expect_err("should report the store that was not found"); + + assert!(error.contains("404"), "{error}"); + assert!(fake.has_secret("ts-old"), "should keep the private key"); + assert!( + fake.config("ts-old").is_some(), + "should keep the public key while the private key is still stored" + ); + } + + #[test] + fn rotate_writes_nothing_when_the_active_keys_cannot_be_read() { + // Read as an empty list, the rotation would write a list holding the + // new key alone and drop `ts-old` while its signatures are in flight. + let fake = Arc::new(FakeFastly::with_config(&[ + ("current-kid", "ts-old"), + ("active-kids", "ts-old"), + ("ts-old", "{}"), + ])); + fake.fail_next_read_of("active-kids"); + + let error = rotate(&fake, Some("ts-new")).expect_err("should not rotate"); + + assert!(error.contains("nothing is written"), "{error}"); + assert!(error.contains("network is unreachable"), "{error}"); + assert!(fake.writes().is_empty(), "{:?}", fake.writes()); + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-old")); + } + + #[test] + fn the_current_key_is_not_retired_when_the_current_key_cannot_be_read() { + // Read as no current key, the check that refuses the current key + // would pass, and the key the service signs with would leave the + // active set. + let fake = two_keys(); + fake.fail_next_read_of("current-kid"); + + let error = deactivate(&fake, "ts-new", false).expect_err("should not retire the key"); + + assert!(error.contains("nothing is written"), "{error}"); + assert!(fake.writes().is_empty(), "{:?}", fake.writes()); + assert_eq!(fake.config("active-kids").as_deref(), Some("ts-old,ts-new")); + } +} diff --git a/crates/trusted-server-cli/src/commands/mod.rs b/crates/trusted-server-cli/src/commands/mod.rs index 0f4354f007..2cdc395b0a 100644 --- a/crates/trusted-server-cli/src/commands/mod.rs +++ b/crates/trusted-server-cli/src/commands/mod.rs @@ -1,11 +1,11 @@ pub(crate) mod audit; +pub(crate) mod cache; pub(crate) mod config; +pub(crate) mod keys; pub(crate) mod pbs; // `dev` is `pub` so the macOS-gated `tests/proxy_e2e.rs` suite can reach -// `commands::dev::proxy`, `origin` is `pub` so `tests/origin_probe.rs` can drive the -// shareability probe against a local fixture, and `cache` is `pub` so -// `tests/cache_purge.rs` can drive a purge against one; the other command modules are +// `commands::dev::proxy`, and `origin` is `pub` so `tests/origin_probe.rs` can drive the +// shareability probe against a local fixture; the other command modules are // crate-internal. -pub mod cache; pub mod dev; pub mod origin; diff --git a/crates/trusted-server-cli/src/commands/origin/mod.rs b/crates/trusted-server-cli/src/commands/origin/mod.rs index c385881c26..c0e9f021ea 100644 --- a/crates/trusted-server-cli/src/commands/origin/mod.rs +++ b/crates/trusted-server-cli/src/commands/origin/mod.rs @@ -89,7 +89,7 @@ pub fn run(command: OriginCommand, out: &mut impl std::io::Write) -> CliResult<( /// value: `name=value; name=value`. /// /// Read from the environment and never accepted as a flag, for the same reason as the -/// admin password in `ts cache purge`: an argument is visible to every other process on +/// Fastly token in `ts cache purge`: an argument is visible to every other process on /// the host through `ps`, and lands in shell history. The report asks operators to probe /// with a genuine session's cookies, so these values are credentials. pub const PROBE_COOKIES_ENVIRONMENT_VARIABLE: &str = "TRUSTED_SERVER_PROBE_COOKIES"; diff --git a/crates/trusted-server-cli/src/commands/pbs/inspect.rs b/crates/trusted-server-cli/src/commands/pbs/inspect.rs index 0bfd39df12..0814f568e4 100644 --- a/crates/trusted-server-cli/src/commands/pbs/inspect.rs +++ b/crates/trusted-server-cli/src/commands/pbs/inspect.rs @@ -4,52 +4,66 @@ use std::path::Path; use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Value, json}; +use trusted_server_auction_prebid_server::MODULE as PREBID_SERVER_MODULE; use trusted_server_core::auction_config_types::{BidderId, ProviderId}; use super::{Output, PbsError, Result, identifier, read_text}; +/// The name `[auction] modules` selects the Prebid module by. +const PREBID_MODULE: &str = "prebid"; + /// Deliberately partial: inspecting PBS requirements must not require unrelated TS settings. #[derive(Default, Deserialize)] struct Source { auction: Option, #[serde(default)] - integrations: Integrations, + demand: Demand, } #[derive(Default, Deserialize)] struct Auction { enabled: Option, #[serde(default)] - providers: BTreeMap, - #[serde(default)] bidders: BTreeMap, + /// The modules `[auction]` selects, `prebid` among them. + #[serde(default)] + modules: Vec, + /// The Prebid module's own table, `[auction.prebid]`. + prebid: Option, +} + +/// The `[demand]` table: `modules` selects what runs, and every other key is +/// one demand source's settings table. +#[derive(Default, Deserialize)] +struct Demand { + #[serde(default)] + modules: Vec, + #[serde(flatten)] + sources: BTreeMap, } #[derive(Deserialize)] -struct AuctionProvider { - profile: Option, +struct DemandSource { + implementation: Option, endpoint: Option, timeout_ms: Option, - profile_config: Option, + test_mode: Option, + debug: Option, + #[serde(default)] + bid_param_override_rules: Vec, } -impl AuctionProvider { - fn profile_bool(&self, key: &str) -> Option { - self.profile_config.as_ref()?.get(key)?.as_bool() - } - - fn override_rule_count(&self) -> usize { - self.profile_config - .as_ref() - .and_then(|config| config.get("bid_param_override_rules")) - .and_then(toml::Value::as_array) - .map_or(0, Vec::len) +impl DemandSource { + /// A demand table runs the implementation its `implementation` line + /// names by module path. + fn runs_prebid_server(&self) -> bool { + self.implementation.as_deref() == Some(PREBID_SERVER_MODULE) } } #[derive(Deserialize)] struct AuctionBidder { - provider: ProviderId, + module: ProviderId, } #[derive(Serialize)] @@ -63,6 +77,7 @@ struct ServerBidderCandidate { #[derive(Serialize)] struct ServerProviderReport { provider: String, + selected: bool, endpoint_configured: bool, timeout_ms_explicit: Option, test_mode_explicit: Option, @@ -71,14 +86,8 @@ struct ServerProviderReport { server_bidder_candidates: Vec, } -#[derive(Default, Deserialize)] -struct Integrations { - prebid: Option, -} - -#[derive(Default, Deserialize)] +#[derive(Default, Clone, Deserialize)] struct Prebid { - enabled: Option, account_id: Option, timeout_ms: Option, debug: Option, @@ -88,14 +97,14 @@ struct Prebid { bundle: Bundle, } -#[derive(Default, Deserialize)] +#[derive(Default, Clone, Deserialize)] struct Bundle { #[serde(default)] modules: BundleModules, } /// Explicit selections from core's bundle module schema; never expand generator presets. -#[derive(Default, Deserialize)] +#[derive(Default, Clone, Deserialize)] struct BundleModules { #[serde(default)] bidder: Vec, @@ -179,8 +188,12 @@ pub(super) fn inspect(path: &Path) -> Result { })?; let auction_present = source.auction.is_some(); let auction = source.auction.unwrap_or_default(); - let prebid_present = source.integrations.prebid.is_some(); - let prebid = source.integrations.prebid.unwrap_or_default(); + let prebid_present = auction.prebid.is_some(); + let prebid_selected = auction + .modules + .iter() + .any(|name| name == PREBID_MODULE || name == "auction.prebid"); + let prebid = auction.prebid.clone().unwrap_or_default(); for name in prebid .client_side_bidders .iter() @@ -194,29 +207,31 @@ pub(super) fn inspect(path: &Path) -> Result { ))); } } - let server_providers: Vec<_> = auction - .providers + let server_providers: Vec<_> = source + .demand + .sources .iter() - .filter(|(_, provider)| provider.profile.as_deref() == Some("prebid-server")) - .map(|(provider_id, provider)| { + .filter(|(_, demand_source)| demand_source.runs_prebid_server()) + .map(|(name, demand_source)| { let requirements: Vec<_> = auction .bidders .iter() - .filter(|(_, bidder)| &bidder.provider == provider_id) + .filter(|(_, bidder)| &bidder.module == name) .map(|(bidder_id, _)| ServerBidderCandidate { bidder: bidder_id.as_str().to_owned(), - source_key: format!("auction.bidders.{}.provider", bidder_id.as_str()), + source_key: format!("auction.bidders.{}.module", bidder_id.as_str()), host_secret_requirement: "unresolved", partner_authorization: "unresolved", }) .collect(); ServerProviderReport { - provider: provider_id.as_str().to_owned(), - endpoint_configured: provider.endpoint.is_some(), - timeout_ms_explicit: provider.timeout_ms, - test_mode_explicit: provider.profile_bool("test_mode"), - debug_explicit: provider.profile_bool("debug"), - bid_param_override_rule_count: provider.override_rule_count(), + provider: name.as_str().to_owned(), + selected: source.demand.modules.contains(name), + endpoint_configured: demand_source.endpoint.is_some(), + timeout_ms_explicit: demand_source.timeout_ms, + test_mode_explicit: demand_source.test_mode, + debug_explicit: demand_source.debug, + bid_param_override_rule_count: demand_source.bid_param_override_rules.len(), server_bidder_candidates: requirements, } }) @@ -224,7 +239,7 @@ pub(super) fn inspect(path: &Path) -> Result { let warnings = [ "Local file only: confirm environment, remote configuration, and request-time overrides.", "Omitted fields/defaults are not expanded; empty candidate lists are not proof of no demand.", - "Disabled auctions, providers, and browser bundle adapters do not authorize PBS activation.", + "A disabled auction, a demand source [demand] modules does not select, and browser bundle adapters do not authorize PBS activation.", "Host secret requirements need adapter metadata verified against the selected PBS release.", ]; let mut details = vec![ @@ -233,10 +248,9 @@ pub(super) fn inspect(path: &Path) -> Result { auction.enabled ), format!( - "Prebid browser section present: {prebid_present}; enabled explicitly: {:?}", - prebid.enabled + "Prebid browser section present: {prebid_present}; selected in [auction] modules: {prebid_selected}" ), - format!("Prebid Server providers: {}", server_providers.len()), + format!("Prebid Server demand sources: {}", server_providers.len()), ]; details.extend(server_providers.iter().map(|provider| { let bidders = provider @@ -246,8 +260,9 @@ pub(super) fn inspect(path: &Path) -> Result { .collect::>() .join(", "); format!( - "Server provider {}: bidders: {}; endpoint configured: {}; timeout explicit: {:?}; test mode explicit: {:?}; debug explicit: {:?}; bid-parameter rules: {}; values withheld", + "Demand source {}: selected: {}; bidders: {}; endpoint configured: {}; timeout explicit: {:?}; test mode explicit: {:?}; debug explicit: {:?}; bid-parameter rules: {}; values withheld", provider.provider, + provider.selected, bidders, provider.endpoint_configured, provider.timeout_ms_explicit, @@ -282,14 +297,14 @@ pub(super) fn inspect(path: &Path) -> Result { data: json!({ "source": path, "source_sections": { - "server": ["auction.providers", "auction.bidders"], - "browser": "integrations.prebid" + "server": ["demand", "auction.bidders"], + "browser": "auction.prebid" }, "auction_section_present": auction_present, "auction_enabled_explicit": auction.enabled, "server_providers": server_providers, "section_present": prebid_present, - "enabled_explicit": prebid.enabled, + "selected": prebid_selected, "account_id_configured": prebid.account_id.is_some(), "timeout_ms_explicit": prebid.timeout_ms, "debug_explicit": prebid.debug, @@ -316,36 +331,38 @@ mod tests { [auction] enabled = true -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand] +modules = ["pbs_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://user:NEVER_PRINT_ME@pbs.example.com/path?token=NEVER_PRINT_ME" timeout_ms = 900 routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = true bid_param_override_rules = [{ when = { bidder = "serverbidder" }, set = { placementId = "NEVER_PRINT_ME" } }] -[auction.bidders.serverbidder] -provider = "pbs-main" - -[auction.providers.pbs-secondary] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand.pbs_secondary] +implementation = "auction.prebid-server" endpoint = "https://NEVER_PRINT_ME@secondary.example.com/openrtb2/auction" routing = "explicit" +[demand.house] +implementation = "auction-protocol.openrtb" +endpoint = "https://house.example.com/openrtb2/auction" + +[auction.bidders.serverbidder] +module = "pbs_main" + [auction.bidders.otherbidder] -provider = "pbs-secondary" +module = "pbs_secondary" -[integrations.prebid] -enabled = false +[auction.prebid] account_id = "NEVER_PRINT_ME" client_side_bidders = ["browserbidder"] -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["exampleBidAdapter"] user_id = ["sharedIdSystem"] analytics = ["exampleAnalyticsAdapter"] @@ -354,19 +371,25 @@ analytics = ["exampleAnalyticsAdapter"] let report = inspect(&path).expect("should inspect config"); assert_eq!(report.data["auction_section_present"], true); assert_eq!(report.data["auction_enabled_explicit"], true); - assert_eq!(report.data["enabled_explicit"], false); + assert_eq!(report.data["selected"], false); assert_eq!( report.data["server_providers"].as_array().map(Vec::len), - Some(2) + Some(2), + "should report the Prebid Server tables and leave the OpenRTB one out" ); - assert_eq!(report.data["server_providers"][0]["provider"], "pbs-main"); + assert_eq!(report.data["server_providers"][0]["provider"], "pbs_main"); + assert_eq!(report.data["server_providers"][0]["selected"], true); assert_eq!( report.data["server_providers"][0]["server_bidder_candidates"][0]["bidder"], "serverbidder" ); assert_eq!( report.data["server_providers"][1]["provider"], - "pbs-secondary" + "pbs_secondary" + ); + assert_eq!( + report.data["server_providers"][1]["selected"], false, + "a table [demand] modules does not name is reported as not selected" ); assert_eq!( report.data["server_providers"][1]["server_bidder_candidates"][0]["bidder"], @@ -374,7 +397,7 @@ analytics = ["exampleAnalyticsAdapter"] ); assert_eq!( report.data["server_providers"][0]["server_bidder_candidates"][0]["source_key"], - "auction.bidders.serverbidder.provider" + "auction.bidders.serverbidder.module" ); assert_eq!( report.data["server_providers"][0]["endpoint_configured"], @@ -409,7 +432,7 @@ analytics = ["exampleAnalyticsAdapter"] .write(false, &mut human) .expect("should render human report"); let human = String::from_utf8(human).expect("should emit UTF-8"); - assert!(human.contains("pbs-main")); + assert!(human.contains("pbs_main")); assert!(human.contains("serverbidder")); assert!(human.contains("Browser bundle adapters: exampleBidAdapter")); assert!(human.contains("Browser identity modules: sharedIdSystem")); @@ -441,7 +464,7 @@ analytics = ["exampleAnalyticsAdapter"] fs::write( file.path(), r#" -[integrations.prebid] +[auction.prebid] client_side_bidders = 'examplebidder\' "#, ) @@ -469,10 +492,10 @@ client_side_bidders = 'examplebidder\' "'example\\u0062idder,otherbidder'", "{ '10' = 'otherbidder', '2' = 'examplebidder' }", ] { - let text = format!("[integrations.prebid]\nclient_side_bidders={input}\n"); + let text = format!("[auction.prebid]\nclient_side_bidders={input}\n"); let file = tempfile::NamedTempFile::new().expect("should create config"); fs::write(file.path(), &text).expect("should write config"); - let runtime: trusted_server_core::integrations::prebid::PrebidIntegrationConfig = + let runtime: trusted_server_auction_prebid::PrebidIntegrationConfig = toml::from_str(&format!("client_side_bidders={input}")) .expect("runtime should accept encoding"); let output = inspect(file.path()).expect("inspect should accept runtime encoding"); @@ -494,11 +517,11 @@ client_side_bidders = 'examplebidder\' "[bundle.modules]\nuser_id = ['sharedIdSystem']", "[bundle.modules]\nanalytics = ['exampleAnalyticsAdapter']", ] { - let runtime: trusted_server_core::integrations::prebid::PrebidIntegrationConfig = + let runtime: trusted_server_auction_prebid::PrebidIntegrationConfig = toml::from_str(input).expect("should parse current core bundle schema"); let text = format!( - "[integrations.prebid]\n{}", - input.replace("[bundle", "[integrations.prebid.bundle") + "[auction.prebid]\n{}", + input.replace("[bundle", "[auction.prebid.bundle") ); let file = tempfile::NamedTempFile::new().expect("should create config"); fs::write(file.path(), &text).expect("should write config"); @@ -523,16 +546,14 @@ client_side_bidders = 'examplebidder\' let input = "[bundle]\nadapters = ['exampleBidAdapter']\nuser_id_modules = ['sharedIdSystem']"; assert!( - toml::from_str::( - input - ) - .is_err(), + toml::from_str::(input) + .is_err(), "core should reject the retired bundle schema" ); let file = tempfile::NamedTempFile::new().expect("should create config"); fs::write( file.path(), - input.replace("[bundle]", "[integrations.prebid.bundle]"), + input.replace("[bundle]", "[auction.prebid.bundle]"), ) .expect("should write config"); // Inspection is deliberately partial, not full runtime validation. @@ -550,7 +571,7 @@ client_side_bidders = 'examplebidder\' let file = tempfile::NamedTempFile::new().expect("should create config"); fs::write( file.path(), - format!("[integrations.prebid.bundle.modules]\n{field} = {value}\n"), + format!("[auction.prebid.bundle.modules]\n{field} = {value}\n"), ) .expect("should write config"); let error = inspect(file.path()) @@ -568,6 +589,6 @@ client_side_bidders = 'examplebidder\' fs::write(&path, "").expect("should write fixture"); let output = inspect(&path).expect("should inspect empty file"); assert_eq!(output.data["section_present"], false); - assert!(output.data["enabled_explicit"].is_null()); + assert_eq!(output.data["selected"], false); } } diff --git a/crates/trusted-server-cli/src/fastly_cli.rs b/crates/trusted-server-cli/src/fastly_cli.rs new file mode 100644 index 0000000000..bd40b7a5ec --- /dev/null +++ b/crates/trusted-server-cli/src/fastly_cli.rs @@ -0,0 +1,146 @@ +//! Running the `fastly` CLI for the commands that administer a deployed +//! service. +//! +//! Those commands go through the hosting platform's own tool, with the +//! operator's own token. The token is read from `FASTLY_API_TOKEN` and reaches +//! the `fastly` process in its environment, never in its arguments, which +//! every other process on the machine can read. + +use std::io::Write as _; +use std::process::{Command, Stdio}; + +/// The environment variable the Fastly API token is read from. +pub(crate) const TOKEN_VARIABLE: &str = "FASTLY_API_TOKEN"; + +/// Runs one invocation of the `fastly` CLI. +pub(crate) trait FastlyRunner: Send + Sync { + /// Runs `fastly` with `args`, writing `stdin` to it when given, and + /// returns its standard output. + /// + /// # Errors + /// + /// A message when the process cannot be run or exits unsuccessfully. + fn run(&self, args: &[String], stdin: Option<&str>) -> Result; +} + +/// Runs the `fastly` binary on the `PATH`, with the token in its environment. +pub(crate) struct FastlyProcess { + token: String, +} + +impl FastlyProcess { + /// Takes the token from [`TOKEN_VARIABLE`]. + /// + /// # Errors + /// + /// A message naming the variable when it is unset or blank. + pub(crate) fn from_environment() -> Result { + Self::with_token(std::env::var(TOKEN_VARIABLE).ok()) + } + + fn with_token(token: Option) -> Result { + token + .filter(|token| !token.trim().is_empty()) + .map(|token| Self { token }) + .ok_or_else(|| format!("set {TOKEN_VARIABLE} to a Fastly API token")) + } +} + +impl FastlyRunner for FastlyProcess { + fn run(&self, args: &[String], stdin: Option<&str>) -> Result { + let mut child = Command::new("fastly") + .args(args) + .env(TOKEN_VARIABLE, &self.token) + .stdin(if stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| format!("could not run `fastly`: {error}"))?; + let written = match stdin { + Some(input) => match child.stdin.take() { + Some(mut pipe) => pipe + .write_all(input.as_bytes()) + .map_err(|error| error.to_string()), + None => Err("its standard input was not open".to_owned()), + }, + None => Ok(()), + }; + let output = child + .wait_with_output() + .map_err(|error| format!("`fastly` did not finish: {error}"))?; + // What `fastly` said comes first. A process that stopped before it + // read its input also fails the write, and its own message says why. + if !output.status.success() { + return Err(format!( + "`fastly {}` failed: {}", + command_words(args), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + written.map_err(|error| format!("could not write to `fastly`: {error}"))?; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } +} + +/// The words of an invocation that name its command, which are those before +/// its first flag. +fn command_words(args: &[String]) -> String { + args.iter() + .map(String::as_str) + .take_while(|arg| !arg.starts_with('-')) + .collect::>() + .join(" ") +} + +/// The arguments of one `fastly` invocation, owned. +pub(crate) fn args(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_missing_or_blank_token_is_refused_by_name() { + for token in [None, Some(String::new()), Some(" ".to_owned())] { + let Err(error) = FastlyProcess::with_token(token.clone()) else { + panic!("should refuse the token {token:?}"); + }; + assert!( + error.contains(TOKEN_VARIABLE), + "should name the variable to set: {error}" + ); + } + } + + #[test] + fn a_token_is_kept_for_the_process_environment() { + let process = + FastlyProcess::with_token(Some("example-token".to_owned())).expect("should accept"); + + assert_eq!(process.token, "example-token"); + } + + #[test] + fn a_command_is_named_by_the_words_before_its_first_flag() { + assert_eq!( + command_words(&args(&[ + "secret-store-entry", + "create", + "--store-id=example", + "--name=ts-key", + "--stdin", + ])), + "secret-store-entry create" + ); + assert_eq!( + command_words(&args(&["service", "purge", "--key=ts-template"])), + "service purge" + ); + } +} diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 24e001d853..4eb3d09058 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -5,6 +5,8 @@ mod app_config; #[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] +mod fastly_cli; +#[cfg(not(target_arch = "wasm32"))] mod prebid_bundle; #[cfg(not(target_arch = "wasm32"))] mod run; diff --git a/crates/trusted-server-cli/src/prebid_bundle.rs b/crates/trusted-server-cli/src/prebid_bundle.rs index 609c950531..86d2ce0320 100644 --- a/crates/trusted-server-cli/src/prebid_bundle.rs +++ b/crates/trusted-server-cli/src/prebid_bundle.rs @@ -300,12 +300,12 @@ fn run_bundle_with_context( if config.external_bundle_url.is_none() { writeln!( out, - "Next: upload {bundle_filename} and set integrations.prebid.external_bundle_url to its HTTPS URL." + "Next: upload {bundle_filename} and set auction.prebid.external_bundle_url to its HTTPS URL." ) } else { writeln!( out, - "Next: upload {bundle_filename} and update integrations.prebid.external_bundle_url if the hosted filename changed." + "Next: upload {bundle_filename} and update auction.prebid.external_bundle_url if the hosted filename changed." ) } .map_err(|error| report_error(format!("failed to write command output: {error}")))?; @@ -326,7 +326,7 @@ fn validate_managed_user_id_modules( .any(|module| module == &requirement.module_name) { return cli_error(format!( - "{} configures managed User ID {:?}, which requires Prebid module {:?}, but the generated manifest omits it; add {:?} to integrations.prebid.bundle.modules.user_id and rerun `ts prebid client`", + "{} configures managed User ID {:?}, which requires Prebid module {:?}, but the generated manifest omits it; add {:?} to auction.prebid.bundle.modules.user_id and rerun `ts prebid client`", config_path.display(), requirement.config_name, requirement.module_name, @@ -343,7 +343,7 @@ fn read_managed_user_id_names(prebid: &toml::Value, config_path: &Path) -> CliRe }; let entries = value.as_array().ok_or_else(|| { report_error(format!( - "{} integrations.prebid.managed_user_ids must be an array of tables", + "{} auction.prebid.managed_user_ids must be an array of tables", config_path.display() )) })?; @@ -354,11 +354,11 @@ fn read_managed_user_id_names(prebid: &toml::Value, config_path: &Path) -> CliRe .map(|(index, entry)| { let table = entry.as_table().ok_or_else(|| { report_error(format!( - "{} integrations.prebid.managed_user_ids[{index}] must be a table", + "{} auction.prebid.managed_user_ids[{index}] must be a table", config_path.display() )) })?; - let field = format!("integrations.prebid.managed_user_ids[{index}].name"); + let field = format!("auction.prebid.managed_user_ids[{index}].name"); let name = table .get("name") .and_then(toml::Value::as_str) @@ -486,48 +486,45 @@ pub(crate) fn load_bundle_config(config_path: &Path) -> CliResult CliResult CliResult<()> { if modules.bidder.is_empty() { return cli_error(format!( - "{} integrations.prebid.bundle.modules.bidder must contain at least one module stem", + "{} auction.prebid.bundle.modules.bidder must contain at least one module stem", config_path.display() )); } let selections = [ ( - "integrations.prebid.bundle.modules.bidder", + "auction.prebid.bundle.modules.bidder", Some(modules.bidder.as_slice()), ), ( - "integrations.prebid.bundle.modules.user_id", + "auction.prebid.bundle.modules.user_id", modules.user_id.as_deref(), ), ( - "integrations.prebid.bundle.modules.analytics", + "auction.prebid.bundle.modules.analytics", modules.analytics.as_deref(), ), ]; @@ -754,25 +751,23 @@ fn patch_config_metadata(config_path: &Path, sha256: &str, sri: &str) -> CliResu )) })?; - if !document.contains_key("integrations") { - document.insert("integrations", table()); + if !document.contains_key("auction") { + document.insert("auction", table()); } - let integrations = table_like_mut( + let auction = table_like_mut( document - .get_mut("integrations") - .expect("should have integrations table"), - "integrations", + .get_mut("auction") + .expect("should have the auction table"), + "auction", config_path, )?; - if !integrations.contains_key("prebid") { - integrations.insert("prebid", table()); + if !auction.contains_key("prebid") { + auction.insert("prebid", table()); } let prebid = table_like_mut( - integrations - .get_mut("prebid") - .expect("should have prebid table"), - "integrations.prebid", + auction.get_mut("prebid").expect("should have prebid table"), + "auction.prebid", config_path, )?; @@ -838,12 +833,11 @@ mod tests { fn managed_config(managed: &str, user_id: &str) -> String { format!( r#" -[integrations.prebid] -enabled = true +[auction.prebid] server_url = "https://prebid.example.com/openrtb2/auction" {managed} -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] user_id = [{user_id}] "# @@ -868,7 +862,7 @@ user_id = [{user_id}] #[test] fn bundle_config_loader_reads_managed_user_id_names_in_order() { let (_temp, path) = write_config(&managed_config( - "\n[[integrations.prebid.managed_user_ids]]\nname = \"identityLink\"\n\n[[integrations.prebid.managed_user_ids]]\nname = \"pubCommonId\"\n", + "\n[[auction.prebid.managed_user_ids]]\nname = \"identityLink\"\n\n[[auction.prebid.managed_user_ids]]\nname = \"pubCommonId\"\n", "\"identityLinkIdSystem\", \"sharedIdSystem\"", )); @@ -904,7 +898,7 @@ user_id = [{user_id}] let error = load_bundle_config(&path).expect_err("should require an array"); assert!( - error.contains("integrations.prebid.managed_user_ids must be an array of tables"), + error.contains("auction.prebid.managed_user_ids must be an array of tables"), "should identify the malformed managed list: {error}" ); } @@ -920,7 +914,7 @@ user_id = [{user_id}] let error = load_bundle_config(&path).expect_err("should require managed tables"); assert!( - error.contains("integrations.prebid.managed_user_ids[0] must be a table"), + error.contains("auction.prebid.managed_user_ids[0] must be a table"), "should identify the malformed managed entry: {error}" ); } @@ -941,7 +935,7 @@ user_id = [{user_id}] let error = load_bundle_config(&path).expect_err("should reject malformed name"); assert!( - error.contains("integrations.prebid.managed_user_ids[0].name"), + error.contains("auction.prebid.managed_user_ids[0].name"), "should identify the malformed managed name: {error}" ); } @@ -1015,7 +1009,7 @@ user_id = [{user_id}] #[test] fn run_bundle_rejects_managed_name_when_bundle_omits_required_module() { let (_temp, config_path) = write_config(&managed_config( - "\n[[integrations.prebid.managed_user_ids]]\nname = \"identityLink\"\n", + "\n[[auction.prebid.managed_user_ids]]\nname = \"identityLink\"\n", "\"sharedIdSystem\"", )); let original = fs::read_to_string(&config_path).expect("should read original config"); @@ -1039,7 +1033,7 @@ user_id = [{user_id}] "should name the managed entry and its required module: {error}" ); assert!( - error.contains("integrations.prebid.bundle.modules.user_id"), + error.contains("auction.prebid.bundle.modules.user_id"), "should identify the corrective field: {error}" ); assert_eq!( @@ -1052,7 +1046,7 @@ user_id = [{user_id}] #[test] fn run_bundle_accepts_bundle_with_required_managed_module() { let (_temp, config_path) = write_config(&managed_config( - "\n[[integrations.prebid.managed_user_ids]]\nname = \"identityLink\"\n", + "\n[[auction.prebid.managed_user_ids]]\nname = \"identityLink\"\n", "\"sharedIdSystem\", \"identityLinkIdSystem\"", )); let output_root = tempfile::tempdir().expect("should create output root"); @@ -1127,7 +1121,7 @@ user_id = [{user_id}] #[test] fn run_bundle_rejects_ambiguous_managed_name_before_generation() { let (_temp, config_path) = write_config(&managed_config( - "\n[[integrations.prebid.managed_user_ids]]\nname = \"ambiguousId\"\n", + "\n[[auction.prebid.managed_user_ids]]\nname = \"ambiguousId\"\n", "\"sharedIdSystem\"", )); let original = fs::read_to_string(&config_path).expect("should read original config"); @@ -1189,7 +1183,7 @@ user_id = [{user_id}] #[test] fn run_bundle_rejects_unknown_managed_name_before_generation() { let (_temp, config_path) = write_config(&managed_config( - "\n[[integrations.prebid.managed_user_ids]]\nname = \"unknownId\"\n", + "\n[[auction.prebid.managed_user_ids]]\nname = \"unknownId\"\n", "\"sharedIdSystem\"", )); let original = fs::read_to_string(&config_path).expect("should read original config"); @@ -1263,7 +1257,7 @@ user_id = [{user_id}] #[test] fn run_bundle_requires_every_managed_module() { let (_temp, config_path) = write_config(&managed_config( - "\n[[integrations.prebid.managed_user_ids]]\nname = \"identityLink\"\n\n[[integrations.prebid.managed_user_ids]]\nname = \"sharedId\"\n", + "\n[[auction.prebid.managed_user_ids]]\nname = \"identityLink\"\n\n[[auction.prebid.managed_user_ids]]\nname = \"sharedId\"\n", "\"identityLinkIdSystem\"", )); let original = fs::read_to_string(&config_path).expect("should read original config"); @@ -1302,12 +1296,11 @@ user_id = [{user_id}] fn valid_config() -> String { r#" -[integrations.prebid] -enabled = true +[auction.prebid] server_url = "https://prebid.example.com/openrtb2/auction" external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-old.js" -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter", "kargoBidAdapter"] user_id = ["sharedIdSystem", "uid2IdSystem"] analytics = ["atsAnalyticsAdapter"] @@ -1366,7 +1359,7 @@ analytics = ["atsAnalyticsAdapter"] fn bundle_config_loader_preserves_omitted_and_empty_optional_lists() { let (_omitted_temp, omitted_path) = write_config( r#" -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] "#, ); @@ -1376,7 +1369,7 @@ bidder = ["rubiconBidAdapter"] let (_empty_temp, empty_path) = write_config( r#" -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] user_id = [] analytics = [] @@ -1394,7 +1387,7 @@ analytics = [] let error = load_bundle_config(&path).expect_err("should reject missing prebid block"); assert!( - error.contains("missing [integrations.prebid]"), + error.contains("missing [auction.prebid]"), "error should explain missing prebid block: {error:?}" ); } @@ -1403,10 +1396,10 @@ analytics = [] fn bundle_config_loader_rejects_missing_bundle_or_modules() { for (contents, expected) in [ ( - "[integrations.prebid]\nenabled = true\n", - "missing [integrations.prebid.bundle]", + "[auction.prebid]\nenabled = true\n", + "missing [auction.prebid.bundle]", ), - ("[integrations.prebid.bundle]\n", "missing field `modules`"), + ("[auction.prebid.bundle]\n", "missing field `modules`"), ] { let (_temp, path) = write_config(contents); let error = load_bundle_config(&path).expect_err("should reject missing table"); @@ -1421,15 +1414,15 @@ analytics = [] fn bundle_config_loader_rejects_empty_or_malformed_bidder_lists() { for (contents, expected) in [ ( - "[integrations.prebid.bundle.modules]\nbidder = []\n", + "[auction.prebid.bundle.modules]\nbidder = []\n", "must contain at least one", ), ( - "[integrations.prebid.bundle.modules]\nbidder = [\"rubiconBidAdapter\", 123]\n", + "[auction.prebid.bundle.modules]\nbidder = [\"rubiconBidAdapter\", 123]\n", "invalid type", ), ( - "[integrations.prebid.bundle.modules]\nbidder = \"rubiconBidAdapter\"\n", + "[auction.prebid.bundle.modules]\nbidder = \"rubiconBidAdapter\"\n", "invalid type", ), ] { @@ -1455,7 +1448,7 @@ analytics = [] "rubiconBidAdapter'", "rubicon\nBidAdapter", ] { - let contents = format!("[integrations.prebid.bundle.modules]\nbidder = [{stem:?}]\n"); + let contents = format!("[auction.prebid.bundle.modules]\nbidder = [{stem:?}]\n"); let (_temp, path) = write_config(&contents); let error = load_bundle_config(&path).expect_err("should reject invalid stem"); assert!( @@ -1469,11 +1462,11 @@ analytics = [] fn bundle_config_loader_rejects_duplicates_within_and_across_kinds() { for contents in [ r#" -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter", "rubiconBidAdapter"] "#, r#" -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["exampleModule"] analytics = ["exampleModule"] "#, @@ -1491,12 +1484,12 @@ analytics = ["exampleModule"] fn bundle_config_loader_rejects_removed_fields_in_fixed_order() { let (_temp, path) = write_config( r#" -[integrations.prebid.bundle] +[auction.prebid.bundle] adapters = ["rubicon"] user_id_modules = ["sharedIdSystem"] analytics_adapters = ["atsAnalyticsAdapter"] -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] "#, ); @@ -1516,10 +1509,10 @@ bidder = ["rubiconBidAdapter"] ] { let contents = format!( r#" -[integrations.prebid.bundle] +[auction.prebid.bundle] {field} = ["exampleModule"] -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] "# ); @@ -1542,7 +1535,7 @@ bidder = ["rubiconBidAdapter"] fn bundle_config_loader_rejects_unknown_module_kinds() { let (_temp, path) = write_config( r#" -[integrations.prebid.bundle.modules] +[auction.prebid.bundle.modules] bidder = ["rubiconBidAdapter"] real_time_data = ["exampleRtdProvider"] "#, @@ -1551,7 +1544,7 @@ real_time_data = ["exampleRtdProvider"] let error = load_bundle_config(&path).expect_err("should reject unknown kind"); assert!(error.contains("unknown field `real_time_data`")); - assert!(error.contains("integrations.prebid.bundle")); + assert!(error.contains("auction.prebid.bundle")); } #[test] @@ -1645,8 +1638,8 @@ real_time_data = ["exampleRtdProvider"] let contents = fs::read_to_string(&path).expect("should read patched config"); let value: toml::Value = toml::from_str(&contents).expect("should parse patched config"); let prebid = value - .get("integrations") - .and_then(|integrations| integrations.get("prebid")) + .get("auction") + .and_then(|auction| auction.get("prebid")) .expect("should have prebid table"); assert_eq!( prebid @@ -1752,7 +1745,7 @@ real_time_data = ["exampleRtdProvider"] assert!(output.contains("generator stdout")); assert!( output.contains(&format!( - "Next: upload trusted-prebid-{}.js and update integrations.prebid.external_bundle_url", + "Next: upload trusted-prebid-{}.js and update auction.prebid.external_bundle_url", "b".repeat(64) )), "should tell operators which content-addressed filename to host: {output}" diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index f77ebb17d2..f0427b0291 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -40,6 +40,9 @@ enum Command { Deploy(DeployArgs), /// Probe a deployed version until it reports healthy. Healthcheck(HealthcheckArgs), + /// Rotate and retire request signing keys, with a Fastly API token. + #[command(subcommand)] + Keys(crate::commands::keys::KeysCommand), /// Trusted Server Prebid commands. Prebid(PrebidArgs), /// Provision platform resources through a target adapter. @@ -118,6 +121,9 @@ pub fn run_from_env() -> Result { } fn dispatch(args: Args) -> Result { + // The config commands validate through `EdgeZero`, which reaches the + // settings' own validation with no way to hand it a list of modules. + crate::app_config::register_stock_modules(); match args.command { Command::ActiveVersion(args) => { edgezero_cli::run_active_version(&args).map(|()| RunOutcome::Success) @@ -152,6 +158,9 @@ fn dispatch(args: Args) -> Result { Command::Healthcheck(args) => { edgezero_cli::run_healthcheck(&args).map(|()| RunOutcome::Success) } + Command::Keys(command) => { + crate::commands::keys::run(&command).map(|()| RunOutcome::Success) + } Command::Prebid(prebid) => match prebid.command { PrebidCommand::Client(args) => { let mut generator = NpmPrebidBundleGenerator; @@ -173,7 +182,7 @@ fn dispatch(args: Args) -> Result { Command::Cache(command) => { let stdout = std::io::stdout(); let mut out = stdout.lock(); - crate::commands::cache::run(command, &mut out).map(|()| RunOutcome::Success) + crate::commands::cache::run(&command, &mut out).map(|()| RunOutcome::Success) } Command::Origin(command) => { let stdout = std::io::stdout(); @@ -196,6 +205,63 @@ mod tests { Args::try_parse_from(args).expect("should parse args") } + #[test] + fn parses_cache_purge_with_its_service_and_one_scope() { + let args = parse(&[ + "ts", + "cache", + "purge", + "--service-id", + "ExampleServiceId0123456", + "--page", + "https://example.com/article", + ]); + let Command::Cache(crate::commands::cache::CacheCommand::Purge(purge)) = args.command + else { + panic!("expected cache purge command"); + }; + assert_eq!(purge.service_id, "ExampleServiceId0123456"); + assert_eq!(purge.page.as_deref(), Some("https://example.com/article")); + assert!(!purge.all, "should not also purge everything"); + + Args::try_parse_from([ + "ts", + "cache", + "purge", + "--service-id", + "ExampleServiceId0123456", + "--all", + "--page", + "https://example.com/article", + ]) + .expect_err("should refuse both scopes at once"); + } + + #[test] + fn parses_keys_rotate_with_its_stores_and_kid() { + let args = parse(&[ + "ts", + "keys", + "rotate", + "--config-store-id", + "config-123", + "--secret-store-id", + "secret-456", + "--kid", + "ts-next", + ]); + let Command::Keys(crate::commands::keys::KeysCommand::Rotate(rotate)) = args.command else { + panic!("expected keys rotate command"); + }; + let rendered = format!("{rotate:?}"); + assert!( + rendered.contains("config-123") + && rendered.contains("secret-456") + && rendered.contains("ts-next"), + "should carry both stores and the key id, got {rendered}" + ); + } + #[test] fn top_level_version_flag_is_available() { let err = Args::try_parse_from(["ts", "--version"]) diff --git a/crates/trusted-server-cli/src/url_guard.rs b/crates/trusted-server-cli/src/url_guard.rs index 240f10d543..2fc961ff31 100644 --- a/crates/trusted-server-cli/src/url_guard.rs +++ b/crates/trusted-server-cli/src/url_guard.rs @@ -1,8 +1,8 @@ -//! Transport checks shared by the commands that send an operator's credentials to a URL. +//! Transport checks for any command that sends an operator's credentials to a URL. //! //! One implementation rather than one per command: a second copy is a second place for the //! loopback exemption to drift, and every command that gets this wrong sends a session -//! cookie or an admin password over the wire in the clear. +//! cookie over the wire in the clear. use crate::error::{CliResult, cli_error}; diff --git a/crates/trusted-server-cli/tests/audit_config_diagnostics.rs b/crates/trusted-server-cli/tests/audit_config_diagnostics.rs index 87df5a447f..7e04c15b28 100644 --- a/crates/trusted-server-cli/tests/audit_config_diagnostics.rs +++ b/crates/trusted-server-cli/tests/audit_config_diagnostics.rs @@ -12,6 +12,13 @@ use std::time::Duration; use trusted_server_core::config::TrustedServerAppConfig; +/// Registers the modules a stock build ships with deploy validation, as the +/// `ts` tool does before it validates, so a configuration validated here is +/// judged against the modules the tool has. +fn register_stock_modules() { + trusted_server_core::config::register_deploy_integrations(trusted_server_modules::builders()); +} + #[test] fn generate_rejects_invalid_config_without_echoing_source() { for (source, guidance) in [ @@ -218,6 +225,7 @@ fn generate_deploy_validation_warning_does_not_echo_baseline_values() { ); let parsed: TrustedServerAppConfig = toml::from_str(&source).expect("should pass syntax and schema"); + register_stock_modules(); assert!( TrustedServerAppConfig::new(parsed.into_settings()).is_err(), "should fail deploy validation" @@ -274,6 +282,7 @@ fn generate_deploy_validation_refusal_preserves_config_and_safe_output() { let source = format!("{}\n# FICTIONAL_SECRET_SENTINEL\n", baseline()); let parsed: TrustedServerAppConfig = toml::from_str(&source).expect("should parse valid baseline"); + register_stock_modules(); TrustedServerAppConfig::new(parsed.into_settings()).expect("should pass deploy validation"); for dry_run in [false, true] { let directory = tempfile::tempdir().expect("should create temporary config directory"); diff --git a/crates/trusted-server-cli/tests/cache_purge.rs b/crates/trusted-server-cli/tests/cache_purge.rs deleted file mode 100644 index 9db1938ef3..0000000000 --- a/crates/trusted-server-cli/tests/cache_purge.rs +++ /dev/null @@ -1,276 +0,0 @@ -//! Tests for `ts cache purge` against a local fixture standing in for a deployed service. -//! -//! Run with `./scripts/test-cli.sh`. - -mod support_origin; - -use support_origin::{FixtureResponse, FixtureServer}; -use trusted_server_cli::commands::cache::{CacheCommand, PurgeArgs, run}; - -/// Serializes the environment across this binary's tests. -/// -/// The CLI reads the password from the environment on purpose, so exercising that path -/// means writing to it — and the test harness runs these in parallel threads of one -/// process, where one test clearing the variable races another that just set it. -static ENVIRONMENT: std::sync::Mutex<()> = std::sync::Mutex::new(()); - -#[test] -fn plaintext_remote_services_are_rejected_before_sending_credentials() { - for service in [ - "http://edge.example.com", - "http://localhost.example.com", - "ftp://127.0.0.1", - ] { - let mut out = Vec::new(); - let error = with_password(Some("example-password"), || { - run( - CacheCommand::Purge(PurgeArgs { - service: service.to_owned(), - all: true, - page: None, - username: "admin".to_owned(), - }), - &mut out, - ) - }) - .expect_err("should refuse insecure remote transport"); - assert!( - error.to_string().contains("HTTPS"), - "should explain the transport requirement: {error}" - ); - assert!(out.is_empty(), "should not report a successful purge"); - } -} - -/// Set the admin password for one call and clear it afterwards. -fn with_password(password: Option<&str>, body: impl FnOnce() -> T) -> T { - // A panicking test poisons the lock; the data is `()`, so recovering it loses nothing - // and keeps one failure from cascading into every other test in the file. - let _guard = ENVIRONMENT - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - // SAFETY: the guard above makes this the only thread touching the environment. - unsafe { - match password { - Some(value) => std::env::set_var("TRUSTED_SERVER_ADMIN_PASSWORD", value), - None => std::env::remove_var("TRUSTED_SERVER_ADMIN_PASSWORD"), - } - } - let result = body(); - // SAFETY: as above; still holding the guard. - unsafe { - std::env::remove_var("TRUSTED_SERVER_ADMIN_PASSWORD"); - } - result -} - -fn args(server: &FixtureServer, all: bool, page: Option<&str>) -> PurgeArgs { - PurgeArgs { - service: server.url(""), - all, - page: page.map(str::to_owned), - username: "admin".to_owned(), - } -} - -#[test] -fn a_successful_purge_prints_the_services_answer_and_exits_zero() { - let server = FixtureServer::start(|_request| { - FixtureResponse::html(r#"{"purged":true,"scope":"all"}"#) - .with_header("content-type", "application/json") - }); - - let mut out = Vec::new(); - let outcome = with_password(Some("admin-pass"), || { - run(CacheCommand::Purge(args(&server, true, None)), &mut out) - }); - - assert!(outcome.is_ok(), "a 200 from the service must exit zero"); - let rendered = String::from_utf8(out).expect("output should be UTF-8"); - assert!( - rendered.contains(r#""purged":true"#), - "the service's own answer is what an operator needs, got: {rendered}" - ); -} - -#[test] -fn the_request_carries_credentials_json_and_the_expected_body() { - // Pins the wire format against the endpoint's guards: it requires POST, rejects any - // Content-Type but application/json, and authenticates on ^/_ts/admin. - let server = FixtureServer::start(|request| { - FixtureResponse::html(format!( - r#"{{"purged":true,"scope":"all","method":"{}","auth":{},"type":"{}","path":"{}"}}"#, - request.method, - request.header("authorization").is_some(), - request.header("content-type").unwrap_or("none"), - request.path - )) - }); - - let mut out = Vec::new(); - with_password(Some("admin-pass"), || { - run(CacheCommand::Purge(args(&server, true, None)), &mut out).expect("should succeed") - }); - - let rendered = String::from_utf8(out).expect("output should be UTF-8"); - assert!(rendered.contains(r#""method":"POST""#), "got: {rendered}"); - assert!(rendered.contains(r#""auth":true"#), "got: {rendered}"); - assert!( - rendered.contains(r#""type":"application/json""#), - "got: {rendered}" - ); - assert!( - rendered.contains(r#""path":"/_ts/admin/cache/purge""#), - "got: {rendered}" - ); -} - -#[test] -fn a_missing_password_fails_before_the_service_is_touched() { - let server = FixtureServer::start(|_request| FixtureResponse::html("{}")); - - let mut out = Vec::new(); - let outcome = with_password(None, || { - run(CacheCommand::Purge(args(&server, true, None)), &mut out) - }); - - assert!(outcome.is_err()); - assert_eq!( - server.request_count(), - 0, - "a purge must not be attempted without a credential" - ); -} - -#[test] -fn a_rejected_purge_exits_non_zero_and_explains_the_status() { - for (status, expected_hint) in [ - (401u16, "admin username"), - (404, "predate the purge endpoint"), - (501, "cannot purge"), - ] { - let server = - FixtureServer::start(move |_request| FixtureResponse::html("{}").with_status(status)); - - let mut out = Vec::new(); - let outcome = with_password(Some("admin-pass"), || { - run(CacheCommand::Purge(args(&server, true, None)), &mut out) - }); - - let error = outcome.expect_err("a refused purge must not exit zero"); - let message = error.to_string(); - assert!( - message.contains(expected_hint), - "a {status} must say what to do next, got: {message}" - ); - } -} - -#[test] -fn a_page_purge_sends_the_url_the_operator_typed() { - let server = FixtureServer::start(|request| { - let body: serde_json::Value = - serde_json::from_slice(&request.body).expect("should parse purge body"); - assert_eq!( - body, - serde_json::json!({"scope": "url", "url": "https://example.com/article"}), - "should send the requested reader URL" - ); - FixtureResponse::html( - serde_json::json!({"purged": true, "scope": "url", "surrogate_key": "example-key"}) - .to_string(), - ) - }); - - let mut out = Vec::new(); - with_password(Some("admin-pass"), || { - run( - CacheCommand::Purge(args(&server, false, Some("https://example.com/article"))), - &mut out, - ) - .expect("should succeed") - }); - - assert_eq!(server.request_count(), 1); -} - -#[test] -fn redirects_cannot_turn_a_login_page_into_a_successful_purge() { - let server = FixtureServer::start(|request| { - if request.path == "/_ts/admin/cache/purge" { - FixtureResponse::html("") - .with_status(302) - .with_header("location", "/login") - } else { - FixtureResponse::html("login") - } - }); - let mut out = Vec::new(); - - let outcome = with_password(Some("admin-pass"), || { - run(CacheCommand::Purge(args(&server, true, None)), &mut out) - }); - - assert!(outcome.is_err(), "should refuse a redirected purge"); - assert!(out.is_empty(), "should print no success response"); - assert_eq!(server.request_count(), 1, "should not follow the redirect"); -} - -#[test] -fn success_requires_a_valid_acknowledgment_for_the_requested_scope() { - for body in [ - "login".to_owned(), - serde_json::json!({"purged": false, "scope": "all"}).to_string(), - serde_json::json!({"purged": true, "scope": "url"}).to_string(), - serde_json::json!({"scope": "all"}).to_string(), - ] { - let server = FixtureServer::start(move |_request| FixtureResponse::html(body.clone())); - let mut out = Vec::new(); - - let outcome = with_password(Some("admin-pass"), || { - run(CacheCommand::Purge(args(&server, true, None)), &mut out) - }); - - assert!( - outcome.is_err(), - "should refuse a missing or mismatched purge acknowledgment" - ); - assert!(out.is_empty(), "should print no success response"); - } -} - -#[test] -fn a_url_purge_requires_a_surrogate_key_in_its_acknowledgment() { - for key in [ - serde_json::Value::Null, - serde_json::json!(""), - serde_json::json!(" "), - ] { - let server = FixtureServer::start(move |_request| { - FixtureResponse::html( - serde_json::json!({ - "purged": true, "scope": "url", "surrogate_key": key - }) - .to_string(), - ) - }); - let mut out = Vec::new(); - - let outcome = with_password(Some("admin-pass"), || { - run( - CacheCommand::Purge(args(&server, false, Some("https://example.com/article"))), - &mut out, - ) - }); - - assert!( - outcome.is_err(), - "should require the service to name the purged URL key" - ); - assert!( - out.is_empty(), - "should not print an incomplete acknowledgment as success" - ); - } -} diff --git a/crates/trusted-server-cli/tests/config_env_overlay.rs b/crates/trusted-server-cli/tests/config_env_overlay.rs index d81b0366a9..3e18ef4729 100644 --- a/crates/trusted-server-cli/tests/config_env_overlay.rs +++ b/crates/trusted-server-cli/tests/config_env_overlay.rs @@ -29,10 +29,10 @@ ids = ["trusted_server_secrets"] "#; const REWRITE_ENV: &str = "TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES"; const SANITIZE_ENV: &str = "TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES"; -const GAM_ATTRIBUTION_ENV: &str = "TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED"; +const GAM_ATTRIBUTION_ENV: &str = "TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED"; const AD_TEMPLATES_ENABLED_ENV: &str = "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED"; -const PROVIDER_ENDPOINT_ENV: &str = "TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT"; -const BIDDER_PROVIDER_ENV: &str = "TRUSTED_SERVER__AUCTION__BIDDERS__EXAMPLE-BIDDER__PROVIDER"; +const PROVIDER_ENDPOINT_ENV: &str = "TRUSTED_SERVER__DEMAND__PBS_MAIN__ENDPOINT"; +const BIDDER_MODULE_ENV: &str = "TRUSTED_SERVER__AUCTION__BIDDERS__EXAMPLE-BIDDER__MODULE"; struct MigratedProject { directory: TempDir, @@ -54,13 +54,18 @@ fn migrated_project() -> MigratedProject { document["auction"]["sanitize_creatives"] = value(false); document["creative_opportunities"]["enabled"] = value(true); document["creative_opportunities"]["gam_network_id"] = value("123456789"); - document["auction"]["providers"]["pbs-main"] = toml_edit::table(); - document["auction"]["providers"]["pbs-main"]["protocol"] = value("openrtb-2.6"); - document["auction"]["providers"]["pbs-main"]["profile"] = value("standard"); - document["auction"]["providers"]["pbs-main"]["endpoint"] = - value("https://original.example/openrtb2/auction"); - document["auction"]["bidders"]["example-bidder"] = toml_edit::table(); - document["auction"]["bidders"]["example-bidder"]["provider"] = value("pbs-main"); + // The demand source `pbs_main` and the bidder route `example-bidder` + // that names it come from the fixture itself, so the map-shaped overlays + // below have leaves to replace. + // A module reads its settings only when its section selects it, and the + // overlay cannot create a leaf, so the Google ad tag's table carries the + // one the override replaces. + let mut modules = Array::new(); + modules.push("google.diagnostics"); + modules.push("google"); + document["ad-tag"]["modules"] = value(modules); + document["ad-tag"]["google"] = toml_edit::table(); + document["ad-tag"]["google"]["gam_attribution_enabled"] = value(false); fs::write(&config_path, document.to_string()).expect("should write migrated config"); fs::write(&manifest_path, MANIFEST).expect("should write test manifest"); MigratedProject { @@ -114,8 +119,8 @@ fn config_validate_explains_legacy_provider_list_migration() { "error should identify the removed field: {stderr}" ); assert!( - stderr.contains("CHANGELOG.md"), - "error should direct operators to migration guidance: {stderr}" + stderr.contains("[demand] modules"), + "error should name where the setting moved to: {stderr}" ); } @@ -200,7 +205,7 @@ fn migrated_config_applies_boolean_environment_overrides() { serde_json::from_str(envelope_json).expect("should parse blob envelope"); assert_eq!( - envelope["data"]["integrations"]["gpt"]["gam_attribution_enabled"], + envelope["data"]["ad-tag"]["google"]["gam_attribution_enabled"], serde_json::Value::Bool(true), "pushed config should contain the GAM attribution environment override" ); @@ -354,7 +359,7 @@ fn map_shaped_provider_and_bidder_environment_overlays_apply() { PROVIDER_ENDPOINT_ENV, "https://overlay.example/openrtb2/auction", ) - .env(BIDDER_PROVIDER_ENV, "pbs-main") + .env(BIDDER_MODULE_ENV, "pbs_main") .output() .expect("should run ts config push with map overlays"); @@ -380,12 +385,12 @@ fn map_shaped_provider_and_bidder_environment_overlays_apply() { serde_json::from_str(envelope_json).expect("should parse blob envelope"); assert_eq!( - envelope["data"]["auction"]["providers"]["pbs-main"]["endpoint"], + envelope["data"]["demand"]["pbs_main"]["endpoint"], "https://overlay.example/openrtb2/auction" ); assert_eq!( - envelope["data"]["auction"]["bidders"]["example-bidder"]["provider"], - "pbs-main" + envelope["data"]["auction"]["bidders"]["example-bidder"]["module"], + "pbs_main" ); } diff --git a/crates/trusted-server-cli/tests/pbs_cli.rs b/crates/trusted-server-cli/tests/pbs_cli.rs index 07536965ae..7deea99529 100644 --- a/crates/trusted-server-cli/tests/pbs_cli.rs +++ b/crates/trusted-server-cli/tests/pbs_cli.rs @@ -92,7 +92,7 @@ fn assert_payload_cleanup(dir: &Path) { #[test] fn local_commands_never_execute_aws_and_preserve_the_source() { let dir = fixture(); - let toml = "[auction]\nenabled=true\n[auction.providers.pbs-main]\nprofile='prebid-server'\nendpoint='https://DUMMY_SECRET@pbs.example.com/openrtb2/auction'\n[auction.bidders.examplebidder]\nprovider='pbs-main'\n[integrations.prebid]\nenabled=false\naccount_id='DUMMY_SECRET'\n[integrations.prebid.bundle.modules]\nbidder=['exampleBidAdapter']\nuser_id=['sharedIdSystem']\nanalytics=['exampleAnalyticsAdapter']\n"; + let toml = "[auction]\nenabled=true\n[demand]\nmodules=['pbs_main']\n[demand.pbs_main]\nimplementation='auction.prebid-server'\nendpoint='https://DUMMY_SECRET@pbs.example.com/openrtb2/auction'\n[auction.bidders.examplebidder]\nmodule='pbs_main'\n[auction.prebid]\naccount_id='DUMMY_SECRET'\n[auction.prebid.bundle.modules]\nbidder=['exampleBidAdapter']\nuser_id=['sharedIdSystem']\nanalytics=['exampleAnalyticsAdapter']\n"; fs::write(dir.path().join("trusted-server.toml"), toml).expect("should write TOML"); let output = command(dir.path()) .args(["prebid", "server", "inspect", "--json"]) @@ -101,14 +101,15 @@ fn local_commands_never_execute_aws_and_preserve_the_source() { assert!(output.status.success()); assert_no_secret(&output); let report: Value = serde_json::from_slice(&output.stdout).expect("should emit JSON"); - assert_eq!(report["enabled_explicit"], false); + assert_eq!(report["selected"], false); assert_eq!(report["bundle_adapters"], json!(["exampleBidAdapter"])); assert_eq!(report["identity_modules"], json!(["sharedIdSystem"])); assert_eq!( report["analytics_modules"], json!(["exampleAnalyticsAdapter"]) ); - assert_eq!(report["server_providers"][0]["provider"], "pbs-main"); + assert_eq!(report["server_providers"][0]["provider"], "pbs_main"); + assert_eq!(report["server_providers"][0]["selected"], true); assert_eq!( report["server_providers"][0]["server_bidder_candidates"][0]["bidder"], "examplebidder" diff --git a/crates/trusted-server-core/.gitattributes b/crates/trusted-server-core/.gitattributes new file mode 100644 index 0000000000..6e7a419fbc --- /dev/null +++ b/crates/trusted-server-core/.gitattributes @@ -0,0 +1,4 @@ +# The recording is compared byte for byte with the page made from the input +# beside it, so both must arrive with the line endings they were committed +# with on every platform. +src/integrations/fixtures/*.html text eol=lf diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index 0a8d052805..e68b1eca70 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -37,10 +37,12 @@ log = { workspace = true } lol_html = { workspace = true } matchit = { workspace = true } mime = { workspace = true } +p256 = { workspace = true } rand = { workspace = true } regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +serde_yaml_ng = { workspace = true } sha2 = { workspace = true } subtle = { workspace = true } toml = { workspace = true } @@ -63,10 +65,16 @@ uuid = { workspace = true, features = ["js"] } chrono = { workspace = true, features = ["wasmbind"] } [features] +# Compiles in the `client_fixed` demonstration Edge Cookie module. Off by +# default so production builds cannot select a fixed shared word as identity. +client-fixed-demo = [] default = [] # Exposes test-only constructors (e.g. `IntegrationRegistry::from_request_filters`) -# so downstream crates can build registries with stub integrations in their tests. -test-utils = [] +# and the platform test doubles, so downstream crates can build registries with +# stub integrations and run against stub services in their tests. The doubles +# are built on the platform library's own, which its feature of the same name +# offers. +test-utils = ["edgezero-core/test-utils"] [dev-dependencies] config = { workspace = true } diff --git a/crates/trusted-server-core/README.md b/crates/trusted-server-core/README.md index 1d8c54ea93..067f9785a0 100644 --- a/crates/trusted-server-core/README.md +++ b/crates/trusted-server-core/README.md @@ -12,7 +12,7 @@ on an edge SDK, Tokio runtime, filesystem, socket, or host-only process API. service boundary adapters implement. - `publisher`, `router`, `handlers`, and `response` dispatch publisher and administrative requests through platform-neutral request/response types. -- `auction` builds plans, invokes providers and mediators, selects bids, and +- `auction` builds plans, invokes demand sources and the ad server, selects bids, and emits bounded telemetry events. - `integrations` registers explicit proxy, rewrite, injection, filter, post-processing, provider, and browser-module capabilities. diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 19aa0b82f3..e650dc0782 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -8,19 +8,21 @@ use trusted_server_core::streaming_processor::StreamProcessor as _; fn make_config() -> HtmlProcessorConfig { HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, origin_host: "origin.bench.example.com".to_string(), request_host: "proxy.bench.example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, + prepared: std::sync::Arc::new(trusted_server_core::middleware::Prepared::default()), + request_state: trusted_server_core::integrations::IntegrationRequestState::default(), // The benchmark measures URL rewriting, not ad injection, and // `ad_slots_script` is `None` here — matching the previous behaviour, // which inferred no body-close work from that. body_close: BodyCloseInjection::None, - suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index 4cc29466f7..8ceaae0fd4 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -103,6 +103,16 @@ fn resolve_from_local_git() -> Option { }) } +mod build_time; +mod key_schedule; + +/// Source file that must stay out of the generated list. +/// +/// `migration_guards.rs` holds the banned pattern itself, as the regex literal +/// the guard matches with, so including it would fail the guard on its own +/// text. +const EXCLUDED: &str = "migration_guards.rs"; + fn main() { // Watching the directory also catches newly added and removed source files. println!("cargo:rerun-if-changed=src"); @@ -151,9 +161,84 @@ fn main() { ) .expect("should write the template build digest"); + write_migration_guard_sources(); + write_build_time(); + write_build_identity(); + write_attestation_keys(); emit_git_version(); } +/// Embeds the attestation signing key schedule in the file +/// `TRUSTED_SERVER_ATTESTATION_KEYS` names, or an empty schedule. +/// +/// The schedule is compiled into the binary, so no signing key sits in a +/// store the service reads when it runs. See [`key_schedule::constant_source`] +/// for what a schedule holds. +fn write_attestation_keys() { + println!("cargo:rerun-if-env-changed=TRUSTED_SERVER_ATTESTATION_KEYS"); + let schedule = env::var_os("TRUSTED_SERVER_ATTESTATION_KEYS") + .filter(|path| !path.is_empty()) + .map(|path| { + let path = PathBuf::from(path); + println!("cargo:rerun-if-changed={}", path.display()); + fs::read_to_string(&path).expect("should read the attestation key schedule") + }); + let source = match key_schedule::constant_source(schedule.as_deref()) { + Ok(source) => source, + Err(reason) => { + println!("cargo::error=the attestation key schedule is refused, because {reason}"); + std::process::exit(1); + } + }; + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("should read OUT_DIR")); + fs::write(out_dir.join("attestation_keys.rs"), source) + .expect("should write the attestation key schedule"); +} + +/// Bakes the build time into the binary as `TRUSTED_SERVER_BUILT_AT`. +/// +/// `SOURCE_DATE_EPOCH` is honored where it is set, so a reproducible build +/// stays reproducible rather than differing only by this string. +#[allow( + clippy::disallowed_methods, + reason = "a build script runs on the host, where the std clock is sound" +)] +fn write_build_time() { + println!("cargo:rerun-if-env-changed=SOURCE_DATE_EPOCH"); + let seconds = env::var("SOURCE_DATE_EPOCH") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| i64::try_from(elapsed.as_secs()).unwrap_or_default()) + .unwrap_or_default() + }); + println!( + "cargo:rustc-env=TRUSTED_SERVER_BUILT_AT={}", + build_time::rfc3339_utc(seconds) + ); +} + +/// Bakes in the commit, the build run and the tag the builder names, as +/// `TRUSTED_SERVER_COMMIT`, `TRUSTED_SERVER_BUILD_RUN` and +/// `TRUSTED_SERVER_BUILD_TAG`, or `unknown`. +fn write_build_identity() { + for name in [ + "TRUSTED_SERVER_COMMIT", + "TRUSTED_SERVER_BUILD_RUN", + "TRUSTED_SERVER_BUILD_TAG", + ] { + println!("cargo:rerun-if-env-changed={name}"); + let value = env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty() && !value.chars().any(char::is_control)) + .unwrap_or_else(|| "unknown".to_owned()); + println!("cargo:rustc-env={name}={value}"); + } +} + /// Compiles the deployed git version in as `TRUSTED_SERVER__GIT_VERSION`. fn emit_git_version() { println!("cargo:rerun-if-env-changed={OVERRIDE_ENV}"); @@ -263,3 +348,65 @@ fn collect_sources(crate_dir: &Path, relative: &Path, sources: &mut Vec } } } + +/// Lists every Rust source file under `src` for the migration guard, so a +/// file added to or removed from core joins or leaves the guard on its own. +fn write_migration_guard_sources() { + // Cargo walks a directory named here, so adding, renaming or deleting a + // source file reruns this script and the guard list follows the tree. + println!("cargo:rerun-if-changed=src"); + + let manifest_dir = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("should read CARGO_MANIFEST_DIR")); + let src_dir = manifest_dir.join("src"); + + let mut sources = Vec::new(); + collect_rust_sources(&src_dir, &src_dir, &mut sources); + sources.sort(); + + let mut generated = String::from( + "// Generated by build.rs. Every Rust source file under `src`, so a file\n\ + // added to or removed from core joins or leaves the guard on its own.\n\ + const CHECKED_SOURCES: &[(&str, &str)] = &[\n", + ); + for relative in &sources { + let absolute = format!("{}/{relative}", path_literal(&src_dir)); + generated.push_str(&format!( + " (\"{relative}\", include_str!(\"{absolute}\")),\n" + )); + } + generated.push_str("];\n"); + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("should read OUT_DIR")); + fs::write(out_dir.join("migration_guard_sources.rs"), generated) + .expect("should write the migration guard source list"); +} + +/// Collects every `.rs` file under `dir`, as a path relative to `root` with +/// forward slashes, so the generated file reads the same on every platform. +fn collect_rust_sources(root: &Path, dir: &Path, sources: &mut Vec) { + let entries = fs::read_dir(dir).expect("should read a source directory"); + for entry in entries { + let entry = entry.expect("should read a source directory entry"); + let path = entry.path(); + if path.is_dir() { + collect_rust_sources(root, &path, sources); + continue; + } + if path.extension().is_some_and(|extension| extension == "rs") { + let relative = path + .strip_prefix(root) + .expect("should strip the source root prefix"); + let relative = path_literal(relative); + if relative != EXCLUDED { + sources.push(relative); + } + } + } +} + +/// Renders a path for a Rust string literal, with forward slashes so a Windows +/// path needs no escaping. +fn path_literal(path: &Path) -> String { + path.display().to_string().replace('\\', "/") +} diff --git a/crates/trusted-server-core/build_time.rs b/crates/trusted-server-core/build_time.rs new file mode 100644 index 0000000000..661ce19ce4 --- /dev/null +++ b/crates/trusted-server-core/build_time.rs @@ -0,0 +1,30 @@ +//! The build time written as RFC 3339 in UTC. +//! +//! A module of the build script, which bakes the time into the binary, and of +//! the tests in `src/build_info.rs`, which check the arithmetic against known +//! dates. + +/// Formats seconds since the Unix epoch as RFC 3339 in UTC. +/// +/// Days are civil days from the epoch, using the standard days-to-civil +/// algorithm, which is correct for any date this will see. +pub(crate) fn rfc3339_utc(seconds: i64) -> String { + let days = seconds.div_euclid(86_400); + let rem = seconds.rem_euclid(86_400); + let (hour, minute, second) = (rem / 3600, (rem % 3600) / 60, rem % 60); + + // Days-to-civil: shift the era so March is month 0 and leap days land at + // the end of a 400 year era. + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + + format!("{y:04}-{m:02}-{d:02}T{hour:02}:{minute:02}:{second:02}Z") +} diff --git a/crates/trusted-server-core/key_schedule.rs b/crates/trusted-server-core/key_schedule.rs new file mode 100644 index 0000000000..03c46cf46e --- /dev/null +++ b/crates/trusted-server-core/key_schedule.rs @@ -0,0 +1,70 @@ +//! The attestation key schedule a build is given, read into the constant +//! the build script writes. +//! +//! A module of the build script and of the tests in `src/attestation.rs`, +//! which check what a schedule may hold. + +/// The source of the `ATTESTATION_KEYS` constant for the text of a key +/// schedule, or of an empty schedule for `None`. +/// +/// A schedule holds one key on each line, being the key id, the Unix second +/// the key comes into force and the private key as base64 DER, separated by +/// tabs. Blank lines are passed over. +/// +/// A key id and a key may hold only letters, digits and `-_.:+/=`. Each is +/// written into Rust source as a string, so nothing a schedule holds can end +/// the string it is written in. No two lines share a key id or a start, so +/// one key is in force at any second and a key id names one key. +/// +/// # Errors +/// +/// Returns the number of the first line of any other shape, or of the first +/// that repeats an earlier line's key id or start, and what is wrong with it. +/// The line itself is never repeated, because it holds a private key. +pub(crate) fn constant_source(schedule: Option<&str>) -> Result { + let mut source = String::from("const ATTESTATION_KEYS: &[(&str, i64, &str)] = &["); + let mut seen: Vec<(usize, &str, i64)> = Vec::new(); + for (index, line) in schedule.unwrap_or_default().lines().enumerate() { + let line = line.trim_matches([' ', '\r']); + if line.is_empty() { + continue; + } + let number = index + 1; + let fields: Vec<&str> = line.split('\t').collect(); + let [key_id, starts_at, private_key] = fields.as_slice() else { + return Err(format!( + "line {number} should hold a key id, a start and a key, separated by tabs" + )); + }; + let starts_at: i64 = starts_at + .parse() + .map_err(|_| format!("line {number} should give the key's start as Unix seconds"))?; + if !is_plain(key_id) || !is_plain(private_key) { + return Err(format!( + "line {number} should hold a key id and a key of letters, digits and -_.:+/= only" + )); + } + if let Some((earlier, _, _)) = seen.iter().find(|(_, id, _)| id == key_id) { + return Err(format!( + "line {number} should not repeat the key id of line {earlier}" + )); + } + if let Some((earlier, _, _)) = seen.iter().find(|(_, _, start)| *start == starts_at) { + return Err(format!( + "line {number} should not start at the same second as line {earlier}" + )); + } + seen.push((number, *key_id, starts_at)); + source.push_str(&format!("({key_id:?}, {starts_at}, {private_key:?}),")); + } + source.push_str("];"); + Ok(source) +} + +/// Whether a schedule field holds only characters a key id or base64 uses. +fn is_plain(field: &str) -> bool { + !field.is_empty() + && field + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.:+/=".contains(&b)) +} diff --git a/crates/trusted-server-core/src/attestation.rs b/crates/trusted-server-core/src/attestation.rs new file mode 100644 index 0000000000..686dd7eb27 --- /dev/null +++ b/crates/trusted-server-core/src/attestation.rs @@ -0,0 +1,1996 @@ +//! Signed evidence of who operates this deployment and which build it runs. +//! +//! The endpoint answers with evidence in the sense of RFC 9334, a description +//! of the build and the platform signed with the operator's key. A deployment +//! on a CDN cannot measure its own bytes, so the evidence proves who operates +//! the deployment, not what its code is. A verifier holding the operator's +//! public keys decides whether to trust it, and the page sends the reader there +//! rather than claiming validity itself. +//! +//! The signature is ECDSA P-256 with SHA-256, deterministic per RFC 6979, over +//! the configured context, one line feed, then the evidence bytes. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use base64::Engine as _; +use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderValue, Method, Request, Response, StatusCode, header}; +use p256::ecdsa::signature::Signer as _; +use p256::ecdsa::{Signature, SigningKey}; +use p256::pkcs8::DecodePrivateKey as _; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use validator::{Validate, ValidationError}; + +use crate::inspect::config::{CONFIG_JSON_PATH, CONFIG_PAGE_PATH}; +use crate::inspect::data::DATA_PAGE_PATH; +use crate::inspect::html_escape; +use crate::inspect::permissions::{PERMISSIONS_JSON_PATH, PERMISSIONS_PAGE_PATH}; +use crate::publisher::{PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH}; +use crate::settings::Settings; + +// `ATTESTATION_KEYS`, written by build.rs from the schedule the build names. +include!(concat!(env!("OUT_DIR"), "/attestation_keys.rs")); + +/// The build script's reading of a key schedule, compiled here so a test can +/// call it. +#[cfg(test)] +#[path = "../key_schedule.rs"] +mod key_schedule; + +/// The address of the attestation page when the settings name none. The JSON +/// form is at the same address plus `.json`. +pub const DEFAULT_ENDPOINT: &str = "/_ts/attestation"; + +/// The addresses beneath an underscore prefix that a deployment answers +/// itself whatever the settings say, so an endpoint may take none of them. A +/// segment in braces stands for any one segment, as it does in the router. +pub const RESERVED_PATHS: &[&str] = &[ + CONFIG_PAGE_PATH, + CONFIG_JSON_PATH, + PERMISSIONS_PAGE_PATH, + PERMISSIONS_JSON_PATH, + DATA_PAGE_PATH, + PAGE_BIDS_PATH, + PAGE_BIDS_LEGACY_PATH, + "/_ts/api/v1/batch-sync", + "/_ts/api/v1/identify", + "/_ts/api/v1/ec/resolve", + "/_ts/set-tester", + "/_ts/clear-tester", + // Answered ahead of the router by an adapter whose host exposes the TLS + // fingerprint or serves several requests from one sandbox. + "/_ts/debug/ja4", + "/_ts/debug/sandbox", +]; + +/// Prefixes beneath which common web frameworks serve their assets from the +/// origin. An endpoint under one would own the prefix and answer `404` for +/// every asset there, so an endpoint may take none of them. +pub const ORIGIN_ASSET_PREFIXES: &[&str] = &[ + "_app", + "_astro", + "_content", + "_expo", + "_framework", + "_next", + "_nuxt", + "_vercel", +]; + +/// The signature algorithm, as JOSE names it. +const ALGORITHM: &str = "ES256"; + +/// The version of the evidence format. +const EVIDENCE_VERSION: u8 = 1; + +/// The longest nonce a caller may have signed. +const MAX_NONCE_LEN: usize = 64; + +/// The architecture whose roles and terms the endpoint follows. +const STANDARD_URL: &str = "https://www.rfc-editor.org/rfc/rfc9334"; + +/// A key id, the Unix second the key comes into force, and the private key +/// as base64 of its SEC1 or PKCS #8 DER encoding. +type ScheduledKey<'a> = (&'a str, i64, &'a str); + +/// The `[attestation]` settings section. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct AttestationConfig { + /// The address of the attestation page, with the JSON form at the same + /// address plus `.json`. When it is not the default, reads of the default + /// address redirect here. Neither form may be one of the + /// [`RESERVED_PATHS`]. + #[serde(default = "default_endpoint")] + #[validate(custom(function = validate_endpoint))] + pub endpoint: String, + /// Who the evidence says operates this deployment. + #[validate(length(min = 1, max = 64))] + pub operator: String, + /// The text signed ahead of the evidence, naming what the signature is + /// for. + #[validate(length(min = 1, max = 64), custom(function = validate_single_line))] + pub context: String, + /// Where a relying party checks the evidence. + #[validate(custom(function = validate_https_url))] + pub verify_url: String, +} + +fn default_endpoint() -> String { + DEFAULT_ENDPOINT.to_owned() +} + +/// What the evidence says about the platform, supplied by the adapter. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PlatformIdentity { + /// The platform's name. + pub name: String, + /// The platform's identifier for the service running this build. + pub service_id: String, + /// The platform's version number of that service. + pub service_version: String, + /// Whether the platform's staging environment answered. + pub staging: bool, + /// The location that answered. + pub pop: String, +} + +/// What a fact the platform did not report reads as. +pub const UNKNOWN: &str = "unknown"; + +impl PlatformIdentity { + /// A platform that reports its name and nothing else. + #[must_use] + pub fn named(name: &str) -> Self { + Self { + name: name.to_owned(), + service_id: UNKNOWN.to_owned(), + service_version: UNKNOWN.to_owned(), + staging: false, + pop: UNKNOWN.to_owned(), + } + } +} + +impl Default for PlatformIdentity { + /// A platform nothing has been reported about. + fn default() -> Self { + Self::named(UNKNOWN) + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Build { + version: &'static str, + built_at: &'static str, + commit: &'static str, + run: &'static str, + tag: &'static str, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct AttestationEvidence<'a> { + v: u8, + operator: &'a str, + publisher: &'a str, + host: &'a str, + platform: &'a PlatformIdentity, + build: Build, + #[serde(skip_serializing_if = "Option::is_none")] + nonce: Option<&'a str>, + issued_at: String, +} + +/// The addresses the endpoint answers at, the page first, or `None` when the +/// settings carry no `[attestation]` section. +#[must_use] +pub fn attestation_paths(settings: &Settings) -> Option<[String; 2]> { + Some(paths_of(&settings.attestation.as_ref()?.endpoint)) +} + +/// The page and JSON addresses of an endpoint. +fn paths_of(endpoint: &str) -> [String; 2] { + [endpoint.to_owned(), format!("{endpoint}.json")] +} + +/// The first segment of an endpoint, being the prefix its pages sit under. +fn prefix_of(endpoint: &str) -> &str { + endpoint + .trim_start_matches('/') + .split('/') + .next() + .unwrap_or_default() +} + +/// A route a deployment registers for its attestation pages. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrefixRoute { + /// The path, in the router's own syntax. + pub path: String, + /// Whether every method is answered here rather than by the publisher's + /// origin. True for the routes beneath a prefix the deployment owns. + pub every_method: bool, +} + +/// The routes to register for the attestation pages, empty when the settings +/// carry no `[attestation]` section. +/// +/// A deployment whose endpoint sits under a prefix of its own owns that +/// prefix, so the prefix itself lists the pages beneath it and any other path +/// there is not found. The default prefix is shared with other routes, so +/// only the pages themselves are taken from it. When the endpoint is not the +/// default, the default address is registered too, to redirect. +#[must_use] +pub fn prefix_routes(settings: &Settings) -> Vec { + let Some(config) = settings.attestation.as_ref() else { + return Vec::new(); + }; + let prefix = prefix_of(&config.endpoint); + let owns_prefix = prefix != prefix_of(DEFAULT_ENDPOINT); + let route = |path: String, every_method: bool| PrefixRoute { path, every_method }; + let mut routes: Vec = paths_of(&config.endpoint) + .into_iter() + .map(|path| route(path, owns_prefix)) + .collect(); + if config.endpoint != DEFAULT_ENDPOINT { + routes.extend( + paths_of(DEFAULT_ENDPOINT) + .into_iter() + .map(|path| route(path, false)), + ); + } + if owns_prefix { + routes.extend( + [ + format!("/{prefix}"), + format!("/{prefix}/"), + format!("/{prefix}/{{*rest}}"), + ] + .into_iter() + .map(|path| route(path, true)), + ); + } + routes +} + +/// Answers every request routed for the attestation pages, being the pages +/// themselves, a redirect from the default address when the endpoint has +/// moved, the list of pages at an owned prefix, and not found for anything +/// else. +#[must_use] +pub fn handle_prefix( + settings: &Settings, + platform: &PlatformIdentity, + req: &Request, +) -> Response { + let Some(config) = settings.attestation.as_ref() else { + return error_response( + StatusCode::NOT_FOUND, + "This deployment serves no attestation.", + ); + }; + let read = matches!(*req.method(), Method::GET | Method::HEAD); + let path = req.uri().path(); + let [page, json] = paths_of(&config.endpoint); + if read && (path == page || path == json) { + return handle_attestation(settings, platform, req); + } + if read && config.endpoint != DEFAULT_ENDPOINT { + let [default_page, default_json] = paths_of(DEFAULT_ENDPOINT); + if path == default_page { + return redirect(&page, req.uri().query()); + } + if path == default_json { + return redirect(&json, req.uri().query()); + } + } + let prefix = prefix_of(&config.endpoint); + if read && (path == format!("/{prefix}") || path == format!("/{prefix}/")) { + return response( + StatusCode::OK, + "text/html; charset=utf-8", + render_index(&page, &json), + ); + } + response( + StatusCode::NOT_FOUND, + "text/html; charset=utf-8", + render_not_found(prefix), + ) +} + +/// Sends the caller to `location`, keeping the query it asked with so a +/// nonce survives the move. +fn redirect(location: &str, query: Option<&str>) -> Response { + let target = match query { + Some(query) if !query.is_empty() => format!("{location}?{query}"), + _ => location.to_owned(), + }; + let mut redirect = response( + StatusCode::MOVED_PERMANENTLY, + "application/json", + json!({ "location": target }).to_string(), + ); + if let Ok(value) = HeaderValue::from_str(&target) { + redirect.headers_mut().insert(header::LOCATION, value); + } + redirect +} + +/// Answers with evidence signed by the compiled-in key in force now, as a +/// page or, at the `.json` address, as data. +/// +/// Every failure becomes a JSON body with a status, so an adapter only routes +/// here. The handler makes no outbound request. +#[must_use] +pub fn handle_attestation( + settings: &Settings, + platform: &PlatformIdentity, + req: &Request, +) -> Response { + answer( + settings, + platform, + req, + ATTESTATION_KEYS, + crate::ec::current_timestamp(), + ) +} + +fn answer( + settings: &Settings, + platform: &PlatformIdentity, + req: &Request, + schedule: &[ScheduledKey<'_>], + now: u64, +) -> Response { + let Some(config) = settings.attestation.as_ref() else { + return error_response( + StatusCode::NOT_FOUND, + "This deployment serves no attestation.", + ); + }; + let nonce = match nonce_from_query(req.uri().query()) { + Ok(nonce) => nonce, + Err(message) => return error_response(StatusCode::BAD_REQUEST, message), + }; + let Some(&(key_id, _, private_key)) = key_in_force(schedule, now) else { + log::error!("attestation has no signing key in force at {now}"); + return error_response( + StatusCode::SERVICE_UNAVAILABLE, + "No signing key is built in for this time, so no evidence can be signed.", + ); + }; + let host = request_host(req); + let evidence = AttestationEvidence { + v: EVIDENCE_VERSION, + operator: &config.operator, + publisher: &settings.publisher.domain, + host: &host, + platform, + build: Build { + version: crate::build_info::VERSION, + built_at: crate::build_info::BUILT_AT, + commit: crate::build_info::COMMIT, + run: crate::build_info::BUILD_RUN, + tag: crate::build_info::BUILD_TAG, + }, + nonce: nonce.as_deref(), + issued_at: rfc3339(now), + }; + let Ok(payload) = serde_json::to_vec(&evidence) else { + return error_response( + StatusCode::INTERNAL_SERVER_ERROR, + "The evidence could not be written.", + ); + }; + let signature = match sign(&config.context, &payload, private_key) { + Ok(signature) => signature, + Err(message) => { + log::error!("attestation key `{key_id}` is unusable: {message}"); + return error_response( + StatusCode::SERVICE_UNAVAILABLE, + "The signing key could not be read, so no evidence can be signed.", + ); + } + }; + let envelope = json!({ + "evidence": serde_json::from_slice::(&payload).unwrap_or(Value::Null), + "payload": URL_SAFE_NO_PAD.encode(&payload), + "signature": { + "kid": key_id, + "alg": ALGORITHM, + "value": URL_SAFE_NO_PAD.encode(signature), + }, + "verify": config.verify_url, + }); + if req.uri().path().ends_with(".json") { + let body = serde_json::to_string_pretty(&envelope).unwrap_or_else(|_| "{}".to_owned()); + response(StatusCode::OK, "application/json", body) + } else { + response( + StatusCode::OK, + "text/html; charset=utf-8", + render_page( + &config.operator, + &config.verify_url, + &json_address(req), + &envelope, + ), + ) + } +} + +/// The absolute address of the JSON form of the page being served, from the +/// request's scheme and the host it was addressed to. +/// +/// The scheme follows a `Forwarded` or `X-Forwarded-Proto` header when one is +/// sent. The host is read the way [`request_host`] reads the one the evidence +/// signs, so the link and the signature name the same host, and a forwarded +/// host header is never read. +fn json_address(req: &Request) -> String { + let info = + crate::http_util::RequestInfo::from_request(req, &crate::platform::ClientInfo::default()); + let authority = req + .headers() + .get(header::HOST) + .and_then(|value| value.to_str().ok()) + .or_else(|| req.uri().authority().map(http::uri::Authority::as_str)) + .unwrap_or_default(); + format!("{}://{authority}{}.json", info.scheme, req.uri().path()) +} + +/// Set once the process has warned that it signs with the last key built in. +static LAST_KEY_WARNED: AtomicBool = AtomicBool::new(false); + +/// Whether this is the first time `warned` is asked, setting it. +fn first_time(warned: &AtomicBool) -> bool { + !warned.swap(true, Ordering::Relaxed) +} + +/// The key in force at `now`, being the latest to have started. Warns once +/// for each process when no later key follows it, because the build must +/// then be refreshed. +fn key_in_force<'a>(schedule: &'a [ScheduledKey<'a>], now: u64) -> Option<&'a ScheduledKey<'a>> { + let now = i64::try_from(now).ok()?; + let current = schedule + .iter() + .filter(|(_, starts_at, _)| *starts_at <= now) + .max_by_key(|(_, starts_at, _)| *starts_at)?; + if !schedule + .iter() + .any(|(_, starts_at, _)| *starts_at > current.1) + && first_time(&LAST_KEY_WARNED) + { + log::warn!( + "attestation key `{}` is the last one built in, so the build must be refreshed", + current.0 + ); + } + Some(current) +} + +/// The bytes a signature covers, being the context, one line feed, then the +/// evidence. +#[must_use] +pub fn signed_bytes(context: &str, payload: &[u8]) -> Vec { + let mut message = Vec::with_capacity(context.len() + 1 + payload.len()); + message.extend_from_slice(context.as_bytes()); + message.push(b'\n'); + message.extend_from_slice(payload); + message +} + +/// Signs `payload` under `context` with a key held as base64 DER text, +/// returning the raw 64 byte `r` and `s` pair. +fn sign(context: &str, payload: &[u8], key_text: &str) -> Result, &'static str> { + let der = STANDARD + .decode(key_text.trim()) + .map_err(|_| "the key is not base64")?; + let secret = p256::SecretKey::from_sec1_der(&der) + .or_else(|_| p256::SecretKey::from_pkcs8_der(&der)) + .map_err(|_| "the key is not a P-256 private key in SEC1 or PKCS #8 DER")?; + let signature: Signature = SigningKey::from(secret).sign(&signed_bytes(context, payload)); + Ok(signature.to_bytes().to_vec()) +} + +/// The caller's nonce, when it asked for one and it is well formed. +fn nonce_from_query(query: Option<&str>) -> Result, &'static str> { + let Some(query) = query else { + return Ok(None); + }; + let Some((_, nonce)) = + url::form_urlencoded::parse(query.as_bytes()).find(|(name, _)| name == "nonce") + else { + return Ok(None); + }; + let well_formed = !nonce.is_empty() + && nonce.len() <= MAX_NONCE_LEN + && nonce + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if well_formed { + Ok(Some(nonce.into_owned())) + } else { + Err("A nonce is 1 to 64 characters from A to Z, a to z, 0 to 9, _ and -.") + } +} + +/// The host the request was addressed to, lowercased with any port removed. +fn request_host(req: &Request) -> String { + let raw = req + .headers() + .get(header::HOST) + .and_then(|value| value.to_str().ok()) + .or_else(|| req.uri().host()) + .unwrap_or_default(); + let without_port = match raw.rsplit_once(':') { + Some((host, port)) if port.bytes().all(|b| b.is_ascii_digit()) => host, + _ => raw, + }; + without_port.to_ascii_lowercase() +} + +/// Seconds since the epoch as RFC 3339 in UTC, to the second. +fn rfc3339(seconds: u64) -> String { + i64::try_from(seconds) + .ok() + .and_then(|seconds| chrono::DateTime::from_timestamp(seconds, 0)) + .map(|time| time.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) + .unwrap_or_default() +} + +fn response(status: StatusCode, content_type: &'static str, body: String) -> Response { + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store")) + .header( + header::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("*"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build attestation response") +} + +fn error_response(status: StatusCode, message: &str) -> Response { + response( + status, + "application/json", + json!({ "error": message }).to_string(), + ) +} + +/// The page form: the claim in words, the link to the verifier, the address +/// of the JSON form to copy, the envelope itself, then the standard followed. +fn render_page(operator: &str, verify_url: &str, json_url: &str, envelope: &Value) -> String { + let verifier = url::Url::parse(verify_url) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)) + .unwrap_or_else(|| verify_url.to_owned()); + let pretty = serde_json::to_string_pretty(envelope).unwrap_or_else(|_| "{}".to_owned()); + let body = format!( + "

Attestation

\n\ +

This page is served by a deployment that says it is operated by {operator}. \ +Do not take this page's word for it.

\n\ +

Check this claim at {verifier}

\n\ +

To retrieve this data as JSON for use in your own verification checks use the URL

\n\ +
{json_url}\ +
\n\ +

The JSON response is

\n\ +
{pretty}
\n\ +

The attester, evidence and verifier on this page follow the \ +IETF Remote Attestation Procedures (RATS) Architecture, \ +RFC 9334.

\n", + operator = html_escape(operator), + href = escape_attribute(verify_url), + verifier = html_escape(&verifier), + json_url = html_escape(json_url), + pretty = html_escape(&pretty), + ); + let script = "document.getElementById(\"copy\").addEventListener(\"click\",function(){\ +var button=this;\ +navigator.clipboard.writeText(document.getElementById(\"json-url\").textContent)\ +.then(function(){button.textContent=\"Copied\";});});"; + html_document("Attestation", &body, script) +} + +/// The list of pages served beneath the prefix, at the prefix itself. +fn render_index(page: &str, json: &str) -> String { + let body = format!( + "

Information pages

\n\ +

This deployment serves these pages about itself.

\n\ +
    \n\ +
  • Attestation, who operates this deployment \ +and which build it runs, with a link to check the claim.
    • \n\ +
  • Attestation as JSON, the same signed \ +evidence as data, for your own verification checks.
    • \n\ +
\n", + page = escape_attribute(page), + json = escape_attribute(json), + ); + html_document("Information pages", &body, "") +} + +/// The answer for any other address beneath a prefix the deployment owns. +fn render_not_found(prefix: &str) -> String { + let body = format!( + "

Not found

\n\ +

Nothing is served at this address. The pages this deployment serves are listed at \ +/{prefix}.

\n", + prefix = escape_attribute(prefix), + ); + html_document("Not found", &body, "") +} + +/// The shell every page here shares. It loads nothing from anywhere else. +fn html_document(title: &str, body: &str, script: &str) -> String { + let script = if script.is_empty() { + String::new() + } else { + format!("\n") + }; + format!( + "\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +{title}\n\ +\n\ +\n\ +\n\ +
\n\ +{body}\ +
\n\ +{script}\ +\n\ +\n", + title = html_escape(title), + ) +} + +fn escape_attribute(value: &str) -> String { + html_escape(value).replace('"', """) +} + +/// Validates the endpoint, an absolute path of two or more segments of lower +/// case letters, digits, `_` and `-`, the first beginning with an underscore +/// so the pages sit apart from the publisher's own, and without `.json`, +/// which names the JSON form. Neither the page nor the JSON form may be one +/// of the [`RESERVED_PATHS`], and the prefix may not be one of the +/// [`ORIGIN_ASSET_PREFIXES`]. +/// +/// # Errors +/// +/// Returns `invalid_endpoint` for any other shape, `reserved_endpoint`, +/// naming the address, when the page or the JSON form is reserved, and +/// `origin_prefix`, naming the prefix, when the prefix is one a framework +/// serves its assets from. +pub fn validate_endpoint(value: &str) -> Result<(), ValidationError> { + let segment_ok = |segment: &str| { + !segment.is_empty() + && segment + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_' || b == b'-') + }; + let segments: Vec<&str> = value + .strip_prefix('/') + .map(|rest| rest.split('/').collect()) + .unwrap_or_default(); + let well_formed = value.len() <= 64 + && segments.len() >= 2 + && segments.iter().all(|segment| segment_ok(segment)) + && segments + .first() + .is_some_and(|first| first.len() > 1 && first.starts_with('_')); + if !well_formed { + return Err(ValidationError::new("invalid_endpoint")); + } + if let Some(reserved) = reserved_path_taken(value, RESERVED_PATHS) { + let mut error = ValidationError::new("reserved_endpoint"); + error.message = Some(format!("{reserved} is answered by the deployment itself").into()); + return Err(error); + } + let prefix = prefix_of(value); + if ORIGIN_ASSET_PREFIXES.contains(&prefix) { + let mut error = ValidationError::new("origin_prefix"); + error.message = + Some(format!("/{prefix} is where a web framework serves its assets").into()); + return Err(error); + } + Ok(()) +} + +/// The first of `reserved` that the endpoint's page or JSON form would take. +fn reserved_path_taken(endpoint: &str, reserved: &[&'static str]) -> Option<&'static str> { + let addresses = paths_of(endpoint); + reserved.iter().copied().find(|route| { + addresses + .iter() + .any(|address| route_answers(route, address)) + }) +} + +/// Whether a route answers an address, a segment in braces answering any one +/// segment. +fn route_answers(route: &str, address: &str) -> bool { + let mut route_segments = route.split('/'); + let mut address_segments = address.split('/'); + loop { + match (route_segments.next(), address_segments.next()) { + (None, None) => return true, + (Some(expected), Some(given)) if expected == given || expected.starts_with('{') => {} + _ => return false, + } + } +} + +fn validate_single_line(value: &str) -> Result<(), ValidationError> { + if value.chars().any(char::is_control) { + Err(ValidationError::new("control_character")) + } else { + Ok(()) + } +} + +fn validate_https_url(value: &str) -> Result<(), ValidationError> { + match url::Url::parse(value) { + Ok(url) if url.scheme() == "https" && url.host_str().is_some() => Ok(()), + _ => Err(ValidationError::new("invalid_https_url")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use p256::ecdsa::VerifyingKey; + use p256::ecdsa::signature::Verifier as _; + use p256::pkcs8::{DecodePublicKey as _, EncodePublicKey as _}; + + use crate::test_support::tests::crate_test_settings_str; + + const CONTEXT: &str = "example-attestation:v1"; + const FIRST: [u8; 32] = [0x11; 32]; + const SECOND: [u8; 32] = [0x22; 32]; + const FIRST_STARTS: i64 = 1_000; + const SECOND_STARTS: i64 = 2_000; + + fn secret_key(scalar: &[u8; 32]) -> p256::SecretKey { + p256::SecretKey::from_slice(scalar).expect("a valid scalar") + } + + /// A key as a schedule holds it, base64 of its SEC1 DER form. + fn key_text(scalar: &[u8; 32]) -> String { + STANDARD.encode(secret_key(scalar).to_sec1_der().expect("encodes")) + } + + fn settings_with_attestation() -> Settings { + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("test settings should load"); + settings.attestation = Some(AttestationConfig { + endpoint: "/_ex/attestation".to_owned(), + operator: "Example Operator".to_owned(), + context: CONTEXT.to_owned(), + verify_url: "https://verifier.example/verify?host=test-publisher.com".to_owned(), + }); + settings + } + + fn request(uri: &str) -> Request { + Request::builder() + .uri(uri) + .header(header::HOST, "Test-Publisher.com:443") + .body(EdgeBody::empty()) + .expect("request") + } + + fn platform() -> PlatformIdentity { + PlatformIdentity { + name: "example-platform".to_owned(), + service_id: "svc-1".to_owned(), + service_version: "39".to_owned(), + staging: false, + pop: "LHR".to_owned(), + } + } + + /// Answers from a two key schedule at `now`. + fn answer_at(settings: &Settings, uri: &str, now: u64) -> Response { + let first = key_text(&FIRST); + let second = key_text(&SECOND); + let schedule = [ + ("example-first", FIRST_STARTS, first.as_str()), + ("example-second", SECOND_STARTS, second.as_str()), + ]; + answer(settings, &platform(), &request(uri), &schedule, now) + } + + fn body_bytes(response: Response) -> Vec { + response + .into_body() + .into_bytes() + .expect("a buffered body") + .to_vec() + } + + fn json_body(response: Response) -> Value { + serde_json::from_slice(&body_bytes(response)).expect("a JSON body") + } + + fn verifying_key(scalar: &[u8; 32]) -> VerifyingKey { + let spki = secret_key(scalar) + .public_key() + .to_public_key_der() + .expect("encodes"); + VerifyingKey::from_public_key_der(spki.as_bytes()).expect("decodes") + } + + fn payload_and_signature(envelope: &Value) -> (Vec, Signature) { + let payload = URL_SAFE_NO_PAD + .decode(envelope["payload"].as_str().expect("payload")) + .expect("base64url payload"); + let raw = URL_SAFE_NO_PAD + .decode(envelope["signature"]["value"].as_str().expect("value")) + .expect("base64url signature"); + assert_eq!(raw.len(), 64, "the signature is the raw r and s pair"); + (payload, Signature::from_slice(&raw).expect("64 bytes")) + } + + /// A build given no schedule, an empty one or blank lines compiles a + /// schedule with no key in it, which is what makes the endpoint answer + /// 503. + #[test] + fn a_build_given_no_schedule_compiles_an_empty_one() { + let empty = "const ATTESTATION_KEYS: &[(&str, i64, &str)] = &[];"; + + for (case, schedule) in [ + ("no schedule", None), + ("an empty file", Some("")), + ("blank lines", Some("\n \n\r\n")), + ] { + assert_eq!( + key_schedule::constant_source(schedule).as_deref(), + Ok(empty), + "{case} should compile an empty schedule" + ); + } + } + + /// Each line becomes one key, in the order written, whichever line ending + /// the file was saved with. + #[test] + fn each_line_of_a_schedule_becomes_a_key() { + let first = key_text(&FIRST); + let second = key_text(&SECOND); + let expected = format!( + "const ATTESTATION_KEYS: &[(&str, i64, &str)] = &[\ + (\"example-first\", 1000, {first:?}),(\"example-second\", 2000, {second:?}),];" + ); + + for ending in ["\n", "\r\n"] { + let schedule = format!( + "example-first\t1000\t{first}{ending}example-second\t2000\t{second}{ending}" + ); + assert_eq!( + key_schedule::constant_source(Some(&schedule)), + Ok(expected.clone()), + "should read both keys from a file with {ending:?} line endings" + ); + } + } + + /// A line of any other shape stops the build. Its key is never repeated + /// in the message, and nothing it holds can end the string it would be + /// written in. + #[test] + fn a_schedule_line_of_any_other_shape_is_refused_without_showing_the_key() { + let key = key_text(&FIRST); + let cases = [ + ("two fields", format!("example-first\t{key}")), + ("four fields", format!("example-first\t1000\t{key}\textra")), + ( + "spaces in place of tabs", + format!("example-first 1000 {key}"), + ), + ( + "a start that is not a number", + format!("example-first\tsoon\t{key}"), + ), + ("an empty key id", format!("\t1000\t{key}")), + ("an empty key", "example-first\t1000\t".to_owned()), + ( + "a key id that would end its string", + format!("example\"); evil(\"\t1000\t{key}"), + ), + ( + "a key that would end its string", + format!("example-first\t1000\t{key}\"), (\"x"), + ), + ( + "a key holding a backslash", + format!("example-first\t1000\t{key}\\"), + ), + ]; + for (case, line) in cases { + let schedule = format!("example-zero\t500\t{key}\n{line}\n"); + let refused = key_schedule::constant_source(Some(&schedule)) + .expect_err("should refuse the schedule"); + + assert!( + refused.starts_with("line 2 "), + "{case} should be refused by its line number, got: {refused}" + ); + assert!( + !refused.contains(&key), + "{case} should be refused without repeating the key" + ); + } + } + + /// Two lines sharing a key id or a start stop the build, so the key in + /// force never depends on the order of the lines and a key id names one + /// key. + #[test] + fn a_schedule_repeating_a_key_id_or_a_start_is_refused() { + let first = key_text(&FIRST); + let second = key_text(&SECOND); + for (case, schedule, refusal) in [ + ( + "a repeated key id", + format!("example-first\t1000\t{first}\nexample-first\t2000\t{second}\n"), + "line 2 should not repeat the key id of line 1", + ), + ( + "a repeated start", + format!("example-first\t1000\t{first}\n\nexample-second\t1000\t{second}\n"), + "line 3 should not start at the same second as line 1", + ), + ] { + assert_eq!( + key_schedule::constant_source(Some(&schedule)), + Err(refusal.to_owned()), + "{case} should be refused" + ); + } + } + + /// A verifier with only the public key, in the `SubjectPublicKeyInfo` DER + /// form a key schedule publishes, accepts the signature over exactly the + /// context, one line feed, then the payload. Without the context, without + /// its line feed, or under another context it never verifies, which is + /// what binds a signature to its purpose. + #[test] + fn the_signature_covers_the_context_a_line_feed_and_the_payload() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json?nonce=abc123", + 1_500, + )); + assert_eq!(envelope["signature"]["alg"], "ES256"); + let (payload, signature) = payload_and_signature(&envelope); + let verifier = verifying_key(&FIRST); + + let bytes = signed_bytes(CONTEXT, &payload); + assert!(bytes.starts_with(CONTEXT.as_bytes())); + assert_eq!(bytes[CONTEXT.len()], b'\n'); + assert_eq!(&bytes[CONTEXT.len() + 1..], payload.as_slice()); + verifier + .verify(&bytes, &signature) + .expect("the signature covers context, line feed and payload"); + + for (case, message) in [ + ("no context", payload.clone()), + ( + "the context without its line feed", + [CONTEXT.as_bytes(), payload.as_slice()].concat(), + ), + ( + "another context", + signed_bytes("another-purpose:v1", &payload), + ), + ] { + assert!( + verifier.verify(&message, &signature).is_err(), + "the signature must not verify with {case}", + ); + } + } + + /// Each key signs from its own start until the next key's start, and the + /// evidence of a time verifies only with the key in force then. + #[test] + fn the_key_in_force_is_the_latest_to_have_started() { + let settings = settings_with_attestation(); + for (now, kid, signing, other) in [ + (1_000, "example-first", &FIRST, &SECOND), + (1_500, "example-first", &FIRST, &SECOND), + (1_999, "example-first", &FIRST, &SECOND), + (2_000, "example-second", &SECOND, &FIRST), + (2_500, "example-second", &SECOND, &FIRST), + ] { + let envelope = json_body(answer_at(&settings, "/_ex/attestation.json", now)); + assert_eq!(envelope["signature"]["kid"], kid, "at {now}"); + let (payload, signature) = payload_and_signature(&envelope); + let bytes = signed_bytes(CONTEXT, &payload); + assert!( + verifying_key(signing).verify(&bytes, &signature).is_ok(), + "at {now} the key in force signs" + ); + assert!( + verifying_key(other).verify(&bytes, &signature).is_err(), + "at {now} the other key must not verify" + ); + } + } + + /// The warning that the last key built in is signing is given once for + /// each process, not on every request. + #[test] + fn the_last_key_warning_is_given_once() { + let warned = AtomicBool::new(false); + + assert!(first_time(&warned), "should warn the first time"); + assert!(!first_time(&warned), "should not warn again"); + assert!(!first_time(&warned), "should not warn on any later request"); + } + + /// Nothing is signed before the first key starts, from an empty schedule, + /// with a key that is not a P-256 private key, or in a build given no + /// schedule, which is every build its builder names no schedule for. + #[test] + fn no_usable_key_in_force_answers_service_unavailable() { + assert!(ATTESTATION_KEYS.is_empty(), "test builds carry no keys"); + let settings = settings_with_attestation(); + let first = key_text(&FIRST); + let second = key_text(&SECOND); + let ed25519_seed = STANDARD.encode([0x33u8; 32]); + let no_key = "No signing key is built in for this time"; + let unreadable = "The signing key could not be read"; + let cases: [(&str, Vec>, u64, &str); 5] = [ + ( + "before the first key starts", + vec![ + ("example-first", FIRST_STARTS, first.as_str()), + ("example-second", SECOND_STARTS, second.as_str()), + ], + 999, + no_key, + ), + ("an empty schedule", Vec::new(), 1_500, no_key), + ( + "a key that is not base64", + vec![("example-bad", FIRST_STARTS, "not base64 at all")], + 1_500, + unreadable, + ), + ( + "a key that is not a P-256 private key", + vec![("example-bad", FIRST_STARTS, ed25519_seed.as_str())], + 1_500, + unreadable, + ), + ( + "the schedule this build was given", + ATTESTATION_KEYS.to_vec(), + crate::ec::current_timestamp(), + no_key, + ), + ]; + for (case, schedule, now, says) in cases { + let response = answer( + &settings, + &platform(), + &request("/_ex/attestation.json"), + &schedule, + now, + ); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{case}"); + let body = json_body(response); + assert!( + body["error"] + .as_str() + .is_some_and(|error| error.starts_with(says)), + "{case}: {body}" + ); + } + } + + /// The decoded evidence shown to people is exactly the payload signed. + #[test] + fn the_evidence_shown_is_the_payload_signed() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json?nonce=abc123", + 1_500, + )); + let (payload, _) = payload_and_signature(&envelope); + let decoded: Value = serde_json::from_slice(&payload).expect("JSON payload"); + + assert_eq!(decoded, envelope["evidence"]); + assert_eq!(decoded["v"], 1); + assert_eq!(decoded["operator"], "Example Operator"); + assert_eq!(decoded["host"], "test-publisher.com"); + assert_eq!(decoded["nonce"], "abc123"); + assert_eq!(decoded["platform"]["serviceVersion"], "39"); + assert_eq!(decoded["build"]["version"], crate::build_info::VERSION); + assert_eq!(decoded["issuedAt"], "1970-01-01T00:25:00Z"); + assert_eq!( + envelope["verify"], + "https://verifier.example/verify?host=test-publisher.com" + ); + } + + /// With no nonce asked for, the evidence carries none. + #[test] + fn no_nonce_is_signed_unless_asked_for() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json", + 1_500, + )); + assert!(envelope["evidence"].get("nonce").is_none()); + } + + #[test] + fn a_malformed_nonce_is_refused() { + let settings = settings_with_attestation(); + for uri in [ + "/_ex/attestation.json?nonce=", + "/_ex/attestation.json?nonce=a%20b", + "/_ex/attestation.json?nonce=%3Cscript%3E", + ] { + let response = answer_at(&settings, uri, 1_500); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{uri}"); + } + let long = format!("/_ex/attestation.json?nonce={}", "a".repeat(65)); + let response = answer_at(&settings, &long, 1_500); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + /// A key held as PKCS #8 signs the same way as one held as SEC1. + #[test] + fn a_pkcs8_key_signs_too() { + use p256::pkcs8::EncodePrivateKey as _; + let pkcs8 = STANDARD.encode( + secret_key(&FIRST) + .to_pkcs8_der() + .expect("encodes") + .as_bytes(), + ); + let schedule = [("example-pkcs8", FIRST_STARTS, pkcs8.as_str())]; + let response = answer( + &settings_with_attestation(), + &platform(), + &request("/_ex/attestation.json"), + &schedule, + 1_500, + ); + assert_eq!(response.status(), StatusCode::OK); + } + + /// The page never says the evidence is valid, because only the verifier + /// can, and it links there. + #[test] + fn the_page_links_to_the_verifier_and_claims_nothing() { + let response = answer_at(&settings_with_attestation(), "/_ex/attestation", 1_500); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + let html = String::from_utf8(body_bytes(response)).expect("utf-8"); + assert!(html.contains("Check this claim at verifier.example")); + assert!(html.contains("href=\"https://verifier.example/verify?host=test-publisher.com\"")); + assert!(html.contains("Do not take this page's word for it.")); + // The JSON shown is escaped as text, which leaves quotes as they are. + assert!( + !html.contains("\"valid\""), + "the JSON shown carries no claim of validity" + ); + assert!(html.contains( + "To retrieve this data as JSON for use in your own verification checks use the URL" + )); + assert!( + html.contains("/_ex/attestation.json"), + "the JSON address is offered to copy", + ); + assert!(html.contains("The JSON response is")); + assert!(html.contains("href=\"https://www.rfc-editor.org/rfc/rfc9334\"")); + } + + /// The address offered to copy is absolute, and its scheme is the one the + /// request arrived over. + #[test] + fn the_json_address_carries_the_scheme_and_host_the_request_used() { + let mut req = request("/_ex/attestation"); + assert_eq!( + json_address(&req), + "http://Test-Publisher.com:443/_ex/attestation.json" + ); + req.headers_mut() + .insert("x-forwarded-proto", HeaderValue::from_static("https")); + assert!(json_address(&req).starts_with("https://")); + } + + /// The link names the host the evidence signs, never a forwarded one, on + /// an adapter that leaves forwarded host headers in place. + #[test] + fn the_json_address_never_names_a_forwarded_host() { + for (name, value) in [ + ("x-forwarded-host", "relay.example"), + ("forwarded", "host=relay.example"), + ] { + let mut req = request("/_ex/attestation"); + req.headers_mut() + .insert(name, HeaderValue::from_static(value)); + + let address = json_address(&req); + + assert!( + address.contains("://Test-Publisher.com:443/") && !address.contains("relay"), + "{name} should not reach the link: {address}" + ); + } + } + + #[test] + fn answers_are_never_cached_and_readable_from_any_origin() { + let settings = settings_with_attestation(); + for uri in ["/_ex/attestation", "/_ex/attestation.json"] { + let response = answer_at(&settings, uri, 1_500); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(response.headers()[header::ACCESS_CONTROL_ALLOW_ORIGIN], "*"); + } + } + + #[test] + fn the_paths_follow_the_endpoint_and_default_to_ts() { + let mut settings = settings_with_attestation(); + assert_eq!( + attestation_paths(&settings), + Some([ + "/_ex/attestation".to_owned(), + "/_ex/attestation.json".to_owned() + ]) + ); + if let Some(config) = settings.attestation.as_mut() { + config.endpoint = DEFAULT_ENDPOINT.to_owned(); + } + assert_eq!( + attestation_paths(&settings), + Some([ + "/_ts/attestation".to_owned(), + "/_ts/attestation.json".to_owned() + ]) + ); + settings.attestation = None; + assert_eq!(attestation_paths(&settings), None); + } + + #[test] + fn the_endpoint_is_an_absolute_path_under_an_underscore_prefix() { + for good in [ + "/_ts/attestation", + "/_ex/attestation", + "/_a1/proof-of_operator", + "/_ex/pages/attestation", + ] { + assert!(validate_endpoint(good).is_ok(), "{good}"); + } + let too_long = format!("/_ex/{}", "a".repeat(60)); + for bad in [ + "", + "/", + "_ex/attestation", + "/attestation", + "/ex/attestation", + "/_/attestation", + "/_ex", + "/_ex/", + "/_ex//attestation", + "/_Ex/attestation", + "/_ex/attestation.json", + "/_ex/attestation/", + "/_ex/attestation?nonce=1", + too_long.as_str(), + ] { + assert!(validate_endpoint(bad).is_err(), "{bad}"); + } + } + + /// An endpoint owns its prefix and answers `404` for every other address + /// beneath it, so a prefix a framework serves its assets from is refused, + /// naming it. + #[test] + fn an_endpoint_under_a_framework_asset_prefix_is_refused_naming_it() { + for prefix in ORIGIN_ASSET_PREFIXES { + let endpoint = format!("/{prefix}/attestation"); + let error = validate_endpoint(&endpoint).expect_err("should refuse the endpoint"); + + assert_eq!(error.code, "origin_prefix", "{endpoint} should be refused"); + assert_eq!( + error.message.as_deref(), + Some(format!("/{prefix} is where a web framework serves its assets").as_str()), + "{endpoint} should name its prefix" + ); + } + } + + #[test] + fn the_section_loads_from_a_document_and_refuses_unknown_fields() { + let base = crate_test_settings_str(); + let with_section = format!( + "{base}\n[attestation]\n\ + endpoint = \"/_ex/attestation\"\n\ + operator = \"Example Operator\"\n\ + context = \"example-attestation:v1\"\n\ + verify_url = \"https://verifier.example/verify\"\n" + ); + let settings = Settings::from_toml(&with_section).expect("the section loads"); + let config = settings.attestation.as_ref().expect("the section is read"); + assert_eq!(config.endpoint, "/_ex/attestation"); + assert_eq!(config.operator, "Example Operator"); + + let without_endpoint = with_section.replace("endpoint = \"/_ex/attestation\"\n", ""); + let settings = Settings::from_toml(&without_endpoint).expect("the endpoint has a default"); + assert_eq!( + settings.attestation.as_ref().map(|a| a.endpoint.as_str()), + Some(DEFAULT_ENDPOINT) + ); + + let bad_endpoint = with_section.replace("/_ex/attestation", "/_ex/attestation.json"); + assert!( + Settings::from_toml(&bad_endpoint).is_err(), + "the JSON form is derived, so an endpoint ending in .json is refused", + ); + + let with_key_name = format!("{with_section}signing_key = \"a_key\"\n"); + assert!( + Settings::from_toml(&with_key_name).is_err(), + "keys are built in, so a document naming one is refused", + ); + + let plain_http = with_section.replace("https://verifier", "http://verifier"); + assert!(Settings::from_toml(&plain_http).is_err()); + } + + /// Every field the evidence depends on is checked when the settings load. + /// The context is signed ahead of a line feed, so one holding a control + /// character would make that framing ambiguous, and the operator and the + /// context are each 1 to 64 characters. + #[test] + fn each_field_the_evidence_depends_on_is_checked_at_load() { + let base = crate_test_settings_str(); + let document = |operator: Option<&str>, context: Option<&str>, verify_url: Option<&str>| { + let mut text = format!("{base}\n[attestation]\n"); + for (key, value) in [ + ("operator", operator), + ("context", context), + ("verify_url", verify_url), + ] { + if let Some(value) = value { + text.push_str(&format!("{key} = \"{value}\"\n")); + } + } + text + }; + let operator = Some("Example Operator"); + let context = Some("example-attestation:v1"); + let verify_url = Some("https://verifier.example/verify"); + let longest = "a".repeat(64); + let too_long = "a".repeat(65); + + Settings::from_toml(&document( + Some(longest.as_str()), + Some(longest.as_str()), + verify_url, + )) + .expect("an operator and a context of 64 characters load"); + for (case, (operator, context, verify_url), refusal) in [ + ( + "a context holding a line feed", + (operator, Some("example\\nattestation"), verify_url), + "attestation.context: control_character", + ), + ( + "an empty operator", + (Some(""), context, verify_url), + "attestation.operator: length", + ), + ( + "a 65 character operator", + (Some(too_long.as_str()), context, verify_url), + "attestation.operator: length", + ), + ( + "an empty context", + (operator, Some(""), verify_url), + "attestation.context: length", + ), + ( + "a 65 character context", + (operator, Some(too_long.as_str()), verify_url), + "attestation.context: length", + ), + ( + "no operator", + (None, context, verify_url), + "missing field `operator`", + ), + ( + "no context", + (operator, None, verify_url), + "missing field `context`", + ), + ( + "no verify_url", + (operator, context, None), + "missing field `verify_url`", + ), + ( + "a verify_url with no host", + (operator, context, Some("https://")), + "attestation.verify_url: invalid_https_url", + ), + ] { + let Err(error) = Settings::from_toml(&document(operator, context, verify_url)) else { + panic!("{case} should be refused"); + }; + let text = format!("{error:?}"); + assert!( + text.contains(refusal), + "{case} should be refused as {refusal}: {text}" + ); + } + } + + /// An endpoint whose page or JSON form is an address the deployment + /// answers itself is refused when the settings load, naming that address, + /// because a second route there stops the router being built. + #[test] + fn an_endpoint_at_a_reserved_address_is_refused_naming_it() { + let base = crate_test_settings_str(); + let document = |endpoint: &str| { + format!( + "{base}\n[attestation]\n\ + endpoint = \"{endpoint}\"\n\ + operator = \"Example Operator\"\n\ + context = \"example-attestation:v1\"\n\ + verify_url = \"https://verifier.example/verify\"\n" + ) + }; + for (endpoint, reserved) in [ + ("/_ts/config", "/_ts/config"), + ("/_ts/permissions", "/_ts/permissions"), + ("/_ts/set-tester", "/_ts/set-tester"), + ("/_ts/api/v1/identify", "/_ts/api/v1/identify"), + ("/__ts/page-bids", "/__ts/page-bids"), + ] { + let Err(error) = Settings::from_toml(&document(endpoint)) else { + panic!("{endpoint} should be refused"); + }; + let text = format!("{error:?}"); + assert!( + text.contains("attestation.endpoint: reserved_endpoint"), + "{endpoint} should be refused as reserved: {text}" + ); + assert!( + text.contains(&format!( + "({reserved} is answered by the deployment itself)" + )), + "{endpoint} should name {reserved}: {text}" + ); + } + for endpoint in [ + DEFAULT_ENDPOINT, + "/_ex/attestation", + "/_ts/proof", + "/_ts/configuration", + ] { + let settings = Settings::from_toml(&document(endpoint)) + .unwrap_or_else(|error| panic!("{endpoint} should load: {error:?}")); + assert_eq!( + settings.attestation.as_ref().map(|a| a.endpoint.as_str()), + Some(endpoint) + ); + } + } + + /// The JSON form is checked as well as the page, and a segment in braces + /// stands for any one segment. + #[test] + fn both_forms_are_checked_against_the_reserved_addresses() { + assert_eq!( + reserved_path_taken("/_ex/page", &["/_ex/page.json"]), + Some("/_ex/page.json"), + "the JSON form clashes" + ); + assert_eq!( + reserved_path_taken("/_ex/page", &["/_ex/{name}"]), + Some("/_ex/{name}"), + "a segment in braces answers the page" + ); + assert_eq!( + reserved_path_taken("/_ex/page", &["/_ex", "/_ex/page/more", "/_ex/other.json"]), + None, + "a parent, a child and another JSON form do not clash" + ); + } + + fn request_with(method: Method, uri: &str) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header(header::HOST, "Test-Publisher.com:443") + .body(EdgeBody::empty()) + .expect("request") + } + + fn html(response: Response) -> String { + String::from_utf8(body_bytes(response)).expect("utf-8") + } + + /// The routes as (path, every method) pairs. + fn routes_of(settings: &Settings) -> Vec<(String, bool)> { + prefix_routes(settings) + .into_iter() + .map(|route| (route.path, route.every_method)) + .collect() + } + + /// A deployment whose endpoint sits under a prefix of its own owns all + /// of that prefix and answers the default address with a redirect. The + /// default prefix gives up only the pages themselves, because other + /// routes share it. + #[test] + fn an_own_prefix_is_owned_and_the_default_is_shared() { + let mut settings = settings_with_attestation(); + assert_eq!( + routes_of(&settings), + [ + ("/_ex/attestation".to_owned(), true), + ("/_ex/attestation.json".to_owned(), true), + ("/_ts/attestation".to_owned(), false), + ("/_ts/attestation.json".to_owned(), false), + ("/_ex".to_owned(), true), + ("/_ex/".to_owned(), true), + ("/_ex/{*rest}".to_owned(), true), + ] + ); + + if let Some(config) = settings.attestation.as_mut() { + config.endpoint = "/_ts/proof".to_owned(); + } + assert_eq!( + routes_of(&settings), + [ + ("/_ts/proof".to_owned(), false), + ("/_ts/proof.json".to_owned(), false), + ("/_ts/attestation".to_owned(), false), + ("/_ts/attestation.json".to_owned(), false), + ] + ); + + if let Some(config) = settings.attestation.as_mut() { + config.endpoint = DEFAULT_ENDPOINT.to_owned(); + } + assert_eq!( + routes_of(&settings), + [ + ("/_ts/attestation".to_owned(), false), + ("/_ts/attestation.json".to_owned(), false), + ] + ); + + settings.attestation = None; + assert!(prefix_routes(&settings).is_empty()); + } + + /// When the endpoint is not the default, a read of the default address + /// is sent to the endpoint, page to page and JSON to JSON, with the query + /// kept so a nonce survives. Other methods there are not found. + #[test] + fn reads_of_the_default_address_redirect_to_the_endpoint() { + let settings = settings_with_attestation(); + for (method, uri, location) in [ + (Method::GET, "/_ts/attestation", "/_ex/attestation"), + (Method::HEAD, "/_ts/attestation", "/_ex/attestation"), + ( + Method::GET, + "/_ts/attestation.json?nonce=abc123", + "/_ex/attestation.json?nonce=abc123", + ), + ( + Method::HEAD, + "/_ts/attestation.json", + "/_ex/attestation.json", + ), + ] { + let case = format!("{method} {uri}"); + let response = handle_prefix(&settings, &platform(), &request_with(method, uri)); + assert_eq!(response.status(), StatusCode::MOVED_PERMANENTLY, "{case}"); + assert_eq!(response.headers()[header::LOCATION], location, "{case}"); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "no-store", + "{case}" + ); + assert_eq!(json_body(response)["location"], location, "{case}"); + } + + let post = handle_prefix( + &settings, + &platform(), + &request_with(Method::POST, "/_ts/attestation"), + ); + assert_eq!(post.status(), StatusCode::NOT_FOUND); + } + + /// With the default endpoint the default address is the endpoint, so it + /// is answered there and nothing redirects. + #[test] + fn the_default_address_is_answered_directly_under_the_default_endpoint() { + let mut settings = settings_with_attestation(); + if let Some(config) = settings.attestation.as_mut() { + config.endpoint = DEFAULT_ENDPOINT.to_owned(); + } + for uri in ["/_ts/attestation", "/_ts/attestation.json"] { + let response = handle_prefix(&settings, &platform(), &request(uri)); + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{uri} reaches the handler, which has no key built in", + ); + assert!(response.headers().get(header::LOCATION).is_none(), "{uri}"); + } + let response = handle_prefix(&settings, &platform(), &request("/_ex/attestation")); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + /// With no `[attestation]` section nothing is signed. The prefix handler + /// and the page handler both answer every address not found, in JSON, + /// even with a key in force. + #[test] + fn without_a_section_every_address_is_not_found_in_json() { + let mut settings = settings_with_attestation(); + settings.attestation = None; + for uri in [ + "/_ts/attestation", + "/_ts/attestation.json?nonce=abc123", + "/_ex/attestation", + ] { + for (handler, response) in [ + ( + "prefix", + handle_prefix(&settings, &platform(), &request(uri)), + ), + ("page", answer_at(&settings, uri, 1_500)), + ] { + let case = format!("{handler} {uri}"); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{case}"); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/json", + "{case}" + ); + assert_eq!( + json_body(response)["error"], + "This deployment serves no attestation.", + "{case}" + ); + } + } + } + + /// The prefix itself lists the pages beneath it. + #[test] + fn the_prefix_lists_its_pages() { + let settings = settings_with_attestation(); + for uri in ["/_ex", "/_ex/"] { + let response = handle_prefix(&settings, &platform(), &request(uri)); + assert_eq!(response.status(), StatusCode::OK, "{uri}"); + let page = html(response); + assert!(page.contains("href=\"/_ex/attestation\""), "{uri}"); + assert!(page.contains("href=\"/_ex/attestation.json\""), "{uri}"); + } + } + + /// Any other address beneath an owned prefix is not found, and says where + /// the pages are. + #[test] + fn any_other_address_beneath_the_prefix_is_not_found() { + let settings = settings_with_attestation(); + let response = handle_prefix(&settings, &platform(), &request("/_ex/nothing/here")); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!(html(response).contains("href=\"/_ex\"")); + + let post = handle_prefix( + &settings, + &platform(), + &request_with(Method::POST, "/_ex/attestation"), + ); + assert_eq!( + post.status(), + StatusCode::NOT_FOUND, + "only reads are answered" + ); + } + + /// A read of an attestation address reaches the attestation handler, + /// which with no key built in answers that it cannot sign. + #[test] + fn a_read_of_an_attestation_address_reaches_the_attestation_handler() { + let settings = settings_with_attestation(); + for method in [Method::GET, Method::HEAD] { + let response = handle_prefix( + &settings, + &platform(), + &request_with(method.clone(), "/_ex/attestation.json"), + ); + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{method}" + ); + } + } + + /// The evidence signed for a request whose Host header is `host`, with + /// any further headers given. + fn evidence_for(host: &str, headers: &[(&str, &str)]) -> Value { + let mut builder = Request::builder() + .uri("/_ex/attestation.json") + .header(header::HOST, host); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder.body(EdgeBody::empty()).expect("request"); + let first = key_text(&FIRST); + let schedule = [("example-first", FIRST_STARTS, first.as_str())]; + let envelope = json_body(answer( + &settings_with_attestation(), + &platform(), + &req, + &schedule, + 1_500, + )); + let (payload, _) = payload_and_signature(&envelope); + serde_json::from_slice(&payload).expect("JSON payload") + } + + /// The signed host is the Host header the request arrived with. A + /// forwarded host header, which a relay can set, never replaces it. + #[test] + fn forwarded_host_headers_never_become_the_signed_host() { + let evidence = evidence_for( + "test-publisher.com", + &[ + ("x-forwarded-host", "relay.example"), + ("forwarded", "host=relay.example;proto=https"), + ("x-original-host", "relay.example"), + ], + ); + assert_eq!(evidence["host"], "test-publisher.com"); + } + + /// The signed publisher comes from the settings, so a request under + /// another name changes the signed host and never the publisher. + #[test] + fn the_signed_publisher_comes_from_the_settings() { + let settings = settings_with_attestation(); + let evidence = evidence_for("www.relay.example", &[]); + assert_eq!(evidence["host"], "www.relay.example"); + assert_eq!(evidence["publisher"], settings.publisher.domain.as_str()); + } + + /// Without a Host header the host comes from the request address, lower + /// cased and without its port. An IPv6 literal keeps its brackets. + #[test] + fn the_host_falls_back_to_the_request_address() { + let absolute = Request::builder() + .uri("https://Test-Publisher.com:8443/_ex/attestation.json") + .body(EdgeBody::empty()) + .expect("request"); + assert_eq!(request_host(&absolute), "test-publisher.com"); + let literal = Request::builder() + .uri("/_ex/attestation.json") + .header(header::HOST, "[::1]:8080") + .body(EdgeBody::empty()) + .expect("request"); + assert_eq!(request_host(&literal), "[::1]"); + } + + /// Changing a signed byte, here the host, stops the signature verifying, + /// so a copy cannot be edited to name another site. + #[test] + fn an_altered_payload_fails_verification() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json?nonce=abc123", + 1_500, + )); + let (payload, signature) = payload_and_signature(&envelope); + let text = String::from_utf8(payload).expect("utf-8"); + let altered = text.replacen( + "\"host\":\"test-publisher.com\"", + "\"host\":\"relay.example\"", + 1, + ); + assert_ne!(altered, text, "the host is in the payload"); + assert!( + verifying_key(&FIRST) + .verify(&signed_bytes(CONTEXT, altered.as_bytes()), &signature) + .is_err(), + "an altered payload must not verify", + ); + } + + /// Signing is deterministic, as RFC 6979 describes, so the same evidence + /// always carries the same signature. + #[test] + fn the_same_evidence_carries_the_same_signature() { + let settings = settings_with_attestation(); + let uri = "/_ex/attestation.json?nonce=abc123"; + let first = json_body(answer_at(&settings, uri, 1_500)); + let second = json_body(answer_at(&settings, uri, 1_500)); + assert_eq!(first["payload"], second["payload"]); + assert_eq!(first["signature"]["value"], second["signature"]["value"]); + } + + /// The envelope carries exactly the documented fields. + #[test] + fn the_envelope_carries_exactly_the_documented_fields() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json", + 1_500, + )); + let names = |value: &Value| { + let mut names: Vec = value + .as_object() + .expect("an object") + .keys() + .cloned() + .collect(); + names.sort(); + names + }; + assert_eq!( + names(&envelope), + ["evidence", "payload", "signature", "verify"] + ); + assert_eq!(names(&envelope["signature"]), ["alg", "kid", "value"]); + } + + /// The evidence fields are signed in the documented order. + #[test] + fn the_evidence_fields_are_signed_in_the_documented_order() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json?nonce=abc123", + 1_500, + )); + let (payload, _) = payload_and_signature(&envelope); + let text = String::from_utf8(payload).expect("utf-8"); + let order = [ + "\"v\":", + "\"operator\":", + "\"publisher\":", + "\"host\":", + "\"platform\":", + "\"name\":", + "\"serviceId\":", + "\"serviceVersion\":", + "\"staging\":", + "\"pop\":", + "\"build\":", + "\"version\":", + "\"builtAt\":", + "\"commit\":", + "\"run\":", + "\"nonce\":", + "\"issuedAt\":", + ]; + let positions: Vec = order + .iter() + .map(|field| { + text.find(field) + .unwrap_or_else(|| panic!("{field} is signed: {text}")) + }) + .collect(); + assert!(positions.windows(2).all(|pair| pair[0] < pair[1]), "{text}"); + } + + /// The longest nonce allowed is signed back unchanged, and a nonce sent + /// percent-encoded is signed as its decoded value. + #[test] + fn a_nonce_is_signed_back_as_sent() { + let settings = settings_with_attestation(); + let longest = "a".repeat(64); + let envelope = json_body(answer_at( + &settings, + &format!("/_ex/attestation.json?nonce={longest}"), + 1_500, + )); + let (payload, _) = payload_and_signature(&envelope); + let evidence: Value = serde_json::from_slice(&payload).expect("JSON payload"); + assert_eq!(evidence["nonce"], longest.as_str()); + let encoded = json_body(answer_at( + &settings, + "/_ex/attestation.json?nonce=Ab_9%2Dz", + 1_500, + )); + assert_eq!(encoded["evidence"]["nonce"], "Ab_9-z"); + } + + /// Markup from the document reaches the page as text, in the claim and + /// in the JSON shown. + #[test] + fn the_page_escapes_what_it_shows() { + let mut settings = settings_with_attestation(); + if let Some(config) = settings.attestation.as_mut() { + config.operator = "Example & Co".to_owned(); + } + let page = html(answer_at(&settings, "/_ex/attestation", 1_500)); + assert!(!page.contains("Example"), "{page}"); + assert!(page.contains("<b>Example</b> & Co")); + } + + /// The evidence reports the build identity compiled in and the platform + /// as the adapter describes it, staging included. + #[test] + fn the_evidence_reports_the_build_and_the_platform() { + let envelope = json_body(answer_at( + &settings_with_attestation(), + "/_ex/attestation.json", + 1_500, + )); + let evidence = &envelope["evidence"]; + assert_eq!(evidence["build"]["commit"], crate::build_info::COMMIT); + assert_eq!(evidence["build"]["run"], crate::build_info::BUILD_RUN); + assert_eq!(evidence["build"]["tag"], crate::build_info::BUILD_TAG); + assert_eq!(evidence["build"]["builtAt"], crate::build_info::BUILT_AT); + assert_eq!(evidence["platform"]["name"], "example-platform"); + assert_eq!(evidence["platform"]["serviceId"], "svc-1"); + assert_eq!(evidence["platform"]["staging"], false); + assert_eq!(evidence["platform"]["pop"], "LHR"); + + let mut staged = platform(); + staged.staging = true; + let first = key_text(&FIRST); + let schedule = [("example-first", FIRST_STARTS, first.as_str())]; + let from_staging = json_body(answer( + &settings_with_attestation(), + &staged, + &request("/_ex/attestation.json"), + &schedule, + 1_500, + )); + assert_eq!(from_staging["evidence"]["platform"]["staging"], true); + } +} diff --git a/crates/trusted-server-core/src/auction/README.md b/crates/trusted-server-core/src/auction/README.md index 8a745eabc1..5500547f32 100644 --- a/crates/trusted-server-core/src/auction/README.md +++ b/crates/trusted-server-core/src/auction/README.md @@ -1,11 +1,13 @@ # Auction orchestration The auction module compiles operator configuration into one immutable plan, -routes browser demand to providers, runs provider requests concurrently where -the adapter permits it, and returns normalized OpenRTB bids. +routes browser demand to the configured demand sources, runs their requests +concurrently where the adapter permits it, and returns normalized OpenRTB bids. The maintained operator guide is -[`docs/guide/auction-orchestration.md`](../../../../docs/guide/auction-orchestration.md). +[`docs/guide/auction-orchestration.md`](../../../../docs/guide/auction-orchestration.md), +and the configuration syntax every provider type shares is in +[`docs/guide/configuration-rules.md`](../../../../docs/guide/configuration-rules.md). This file describes the code layout and runtime flow for contributors. ## Runtime flow @@ -16,10 +18,10 @@ flowchart TB B --> C[endpoints::convert_tsjs_to_auction_request] C --> D[routing::route_auction] D --> E[provider::GenericOpenRtbProvider builds requests] - E --> F[orchestrator::AuctionOrchestrator dispatches providers] - F --> G[Provider responses are normalized] - G --> H{Mediator configured?} - H -->|Yes| I[Mediator selects bids] + E --> F[orchestrator::AuctionOrchestrator dispatches demand sources] + F --> G[Demand responses are normalized] + G --> H{Ad server configured?} + H -->|Yes| I[Ad server selects bids] H -->|No| J[Orchestrator ranks bids locally] I --> K[formats::convert_to_openrtb_response] J --> K @@ -27,80 +29,80 @@ flowchart TB ``` Each adapter owns transport routing in its `app.rs`. Core request handling stays -in `auction::endpoints`, so no provider or profile depends on Fastly types. +in `auction::endpoints`, so no demand implementation depends on Fastly types. `handle_auction` performs these steps: 1. Enforce the body limit and parse the Trusted Server ad-unit request. -2. Apply the disabled-auction and consent gates before provider work. +2. Apply the disabled-auction and consent gates before any demand work. 3. Consume the request's existing EC and consent context. The endpoint does not generate an EC ID. 4. Convert the request with `convert_tsjs_to_auction_request`. 5. Route slots and bidder params through the compiled `AuctionPlan`. -6. Run the plan-backed orchestrator and optional mediator. +6. Run the plan-backed orchestrator and the optional ad server. 7. Build the OpenRTB response with `convert_to_openrtb_response`. ## Configuration boundary `auction::compile_auction_plan` is the single settings-to-plan boundary used by -startup and operator validation. It validates: - -- provider IDs, protocols, profiles, endpoints, timeouts, and routing modes; -- bidder-to-provider ownership; -- profile-specific configuration; -- mediator and request-signing references; -- bounded configuration values. +startup and operator validation. It reads `[demand]`, `[ad-server]` and +`[auction.bidders]` and validates: + +- demand source and ad server names, which must be snake_case +- a settings table its type's `modules` or `module` does not select +- an `implementation` no builder registered, naming the ones that are +- endpoints, which must be HTTPS or HTTP to a loopback host, with no + credentials or fragment +- timeouts, routing modes and notification bounds +- the settings each implementation accepts, since each one rejects unknown keys +- bidder-to-demand ownership, and +- request-signing structure. Adapters then call `AuctionPlan::validate_for_target` for backend naming, fan-out support, and target resource limits. -A plan-backed orchestrator contains generic OpenRTB providers compiled from the -plan. `AuctionOrchestrator::register_provider` and the old concrete Prebid +A plan-backed orchestrator contains generic OpenRTB demand sources compiled from +the plan. `AuctionOrchestrator::register_provider` and the old concrete Prebid provider remain test-only parity code. They are not extension APIs. ## Routing `routing::route_auction` normalizes the browser `trustedServer` envelope and -produces one `ProviderAuctionInput` per provider. +produces one `ProviderAuctionInput` per demand source. - `explicit` sends a slot only when it has bidder demand assigned to that - provider, or trusted stored-request demand where the profile supports it. + source, or trusted stored-request demand where the implementation supports it. - `all_eligible` sends every compatible banner slot without copying another - provider's bidder params. -- `prebid-server` requires `explicit`. PBS rejects impressions that have neither + source's bidder params. +- `auction.prebid-server` requires `explicit`. PBS rejects impressions that have neither bidder demand nor a stored-request reference. - APS normally uses `all_eligible` because APS participates across eligible inventory without browser bidder params. -Each `[auction.bidders.]` route has one provider owner. Unlisted page -bidders remain browser demand. +Each `[auction.bidders.]` route has one owner, named by its +`[demand]` table name. Unlisted page bidders remain browser demand. -## Provider execution +## Demand execution `provider::GenericOpenRtbProvider` owns the shared transport path for the -`standard`, `prebid-server`, and `aps` profiles. Profiles receive routed and -privacy-approved facts, not the raw inbound request. - -The orchestrator launches all eligible providers before collecting responses. -It uses adapter `PlatformHttpClient` handles and predicted backend names for -correlation. Provider launch, transport, HTTP, parse, and admission failures are -provider-local when another provider can continue. +`auction-protocol.openrtb`, `auction.prebid-server` and `auction.aps` implementations. An implementation receives +routed and privacy-approved facts, not the raw inbound request. -When no mediator is configured, the orchestrator selects the highest decoded -CPM per slot and applies floors locally. When a mediator is configured, it sends -normalized provider responses to the separately registered mediator and falls -back to local ranking when mediation cannot run. +The orchestrator launches all eligible demand sources before collecting +responses. It uses adapter `PlatformHttpClient` handles and predicted backend +names for correlation. Launch, transport, HTTP, parse and admission failures are +local to one demand source when another can continue. ## Response admission -Providers normalize successful upstream bids into `auction::types::Bid`. +Demand sources normalize successful upstream bids into `auction::types::Bid`. Admission checks keep malformed or unrequested bids out of ranking. Aggregate metadata reports bounded rejection counts without retaining raw upstream bid payloads. Notification suppression runs after normalization and matches exact returned -OpenRTB seats. Provider response identity uses the configured provider ID, such -as `pbs-main`. +OpenRTB seats. Response identity uses the configured demand source name, such +as `pbs_main`. ## Creative delivery @@ -121,55 +123,65 @@ response. enabled = true timeout_ms = 2000 -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand] +modules = ["pbs_main", "aps_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" timeout_ms = 900 routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = false consent_forwarding = "both" -[auction.providers.pbs-main.notifications] +[demand.pbs_main.notifications] suppress_all = false suppress_seats = ["example-seat"] -[auction.bidders.example-server] -provider = "pbs-main" - -[auction.providers.aps-main] -protocol = "openrtb-2.6" -profile = "aps" +[demand.aps_main] +implementation = "auction.aps" endpoint = "https://aps.example.com/e/pb/bid" routing = "all_eligible" -profile_config = { account_id = "example-account" } +account_id = "example-account" + +[auction.bidders.example-server] +module = "pbs_main" + +[ad-server] +module = "mock" + +[ad-server.mock] +endpoint = "https://adserver.example.com/decide" +timeout_ms = 500 ``` -Provider endpoints must be absolute HTTPS URLs. Replace all example values -before enabling an auction. +Every demand table carries an `implementation` line naming the +implementation by its module path, which is how two Prebid Servers run side +by side under names of their own, and `[ad-server]` writes `ad-server.mock` +as `mock`. Endpoints must be HTTPS, or HTTP to `127.0.0.1`, `::1` or +`localhost`. Replace all example values before enabling an auction. ## Code map - `mod.rs` compiles plans and builds the shared orchestrator. - `endpoints.rs` handles `POST /auction` and converts the browser request. - `plan.rs` owns plan validation and target capability checks. -- `profile.rs` owns typed OpenRTB profile configuration. -- `routing.rs` assigns slots and bidder params to providers. +- `demand.rs` owns the demand and ad server implementation seam. +- `routing.rs` assigns slots and bidder params to demand sources. - `openrtb.rs` builds shared requests and parses standard responses. -- `provider.rs` runs plan-backed provider requests and profile-specific parsing. -- `orchestrator.rs` owns fan-out, deadlines, mediation, and local ranking. +- `provider.rs` runs plan-backed demand requests and per-implementation parsing. +- `orchestrator.rs` owns fan-out, deadlines, the ad server call, and local + ranking. - `formats.rs` builds direct endpoint responses and processes creatives. - `types.rs` contains normalized auction request, response, slot, and bid types. ## Testing Use `compile_auction_plan` in tests, then construct the orchestrator and -integration registry from the same `Arc`. Profile tests should -cover typed configuration, exact request output, response admission, routing, -provider-local failures, and target validation. +integration registry from the same `Arc`. Tests for one +implementation should cover its typed settings, exact request output, response +admission, routing, local failures, and target validation. Run target-matched aliases rather than bare workspace tests: diff --git a/crates/trusted-server-core/src/auction/context.rs b/crates/trusted-server-core/src/auction/context.rs index f7fd782942..467801726f 100644 --- a/crates/trusted-server-core/src/auction/context.rs +++ b/crates/trusted-server-core/src/auction/context.rs @@ -1,6 +1,6 @@ //! Context query-parameter forwarding for auction providers. //! -//! Provides a config-driven mechanism for ad-server / mediator providers to +//! Provides a config-driven mechanism for ad server providers to //! forward integration-supplied data (e.g. audience segments) as URL query //! parameters without hard-coding integration-specific knowledge. @@ -26,12 +26,12 @@ pub enum ContextValue { /// Mapping from auction-request context keys to query-parameter names. /// -/// Used by ad-server / mediator providers to forward integration-supplied data +/// Used by ad server providers to forward integration-supplied data /// (e.g. audience segments) as URL query parameters without hard-coding /// integration-specific knowledge. /// /// ```toml -/// [integrations.adserver_mock.context_query_params] +/// [ad-server.mock.context_query_params] /// permutive_segments = "permutive" /// lockr_ids = "lockr" /// ``` diff --git a/crates/trusted-server-core/src/auction/demand.rs b/crates/trusted-server-core/src/auction/demand.rs new file mode 100644 index 0000000000..c9800bb36c --- /dev/null +++ b/crates/trusted-server-core/src/auction/demand.rs @@ -0,0 +1,290 @@ +//! Demand and ad server implementations, the seam through which an integration +//! supplies the providers an auction calls. +//! +//! A deployment selects its demand sources with `[demand] modules = [...]` and +//! its ad server with `[ad-server] module = "..."`. Each selected name resolves +//! to an implementation that an integration builder registers, so core names no +//! vendor. The common `OpenRTB` driver builds every standard request field, sends +//! the request and enforces privacy. An implementation decides only the choices +//! this module exposes, which are its field policy, its own extensions, its +//! outbound headers and how its responses are read. + +use core::any::Any; +use std::net::IpAddr; +use std::sync::Arc; + +use async_trait::async_trait; +use error_stack::Report; +use http::HeaderMap; +use serde_json::{Map, Value}; +use url::Url; + +use crate::consent::ConsentContext; +use crate::error::TrustedServerError; +use crate::platform::PlatformResponse; + +use super::provider::AuctionProvider; +pub use super::routing::{ProviderAuctionInput, ProviderSlotInput, TransportHeaders}; +use super::types::AuctionResponse; + +/// The longest primary `Accept-Language` tag a conservative source receives. +/// +/// A source that sets no limit receives the whole primary tag. +pub const CONSERVATIVE_LANGUAGE_MAX_BYTES: usize = 8; + +/// Where a demand source's timeout comes from when its table sets none. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DemandTimeoutDefault { + /// The whole-auction timeout. + Auction, + /// A fixed timeout, in milliseconds. + Fixed(u32), +} + +/// How the `regs` object is derived from the consent the auction admitted. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum RegsPolicy { + /// No `regs` without consent data, and GDPR set from the applicability bit + /// or from a GDPR jurisdiction. + #[default] + Jurisdiction, + /// `regs` for any admitted consent, and GDPR set from the applicability bit + /// alone. + ApplicabilityBit, +} + +/// The standard `OpenRTB` field choices one demand source makes. +/// +/// The common driver constructs every standard field and reads this policy +/// while it does. A policy can leave data out of what central privacy +/// enforcement approved, and it cannot add data that enforcement removed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DemandFieldPolicy { + /// Send the slot id as `imp.tagid`. + pub imp_tagid: bool, + /// Copy the first banner format into `banner.w` and `banner.h`, and send + /// `banner.topframe` as not in the top frame. + pub primary_banner_size: bool, + /// Send the accepted `Referer` header as `site.ref`. + pub site_ref: bool, + /// Send precise latitude and longitude with the coarse geo. + pub precise_geo: bool, + /// Send Google Additional Consent as `user.ext.ConsentedProvidersSettings`. + pub additional_consent: bool, + /// The longest primary `Accept-Language` tag sent, or `None` for no limit. + pub language_max_bytes: Option, + /// How `regs` is derived from consent. + pub regs: RegsPolicy, + /// Mark the request as a test request. + pub test: bool, + /// Keep the Trusted Server host and scheme in `ext.trusted_server` when the + /// request is not signed. + pub unsigned_request_identity: bool, + /// Send `Accept: application/json`. + pub accept_json: bool, + /// Treat routed bidder parameters as request data rather than reporting + /// them as parameters the source ignored. + pub consumes_bidder_params: bool, +} + +/// Facts an implementation may use when it adjusts the outbound request. +#[derive(Debug, Clone, Copy)] +pub struct DemandTransport<'a> { + /// The request headers admitted for forwarding. + pub headers: &'a TransportHeaders, + /// The client address the platform attested, never one a client supplied. + pub attested_client_ip: Option, +} + +/// Facts an implementation reads while it parses a response. +#[derive(Clone, Copy)] +pub struct DemandResponse<'a> { + /// The configured name of the demand source. + pub provider_id: &'a str, + /// The canonical endpoint the request was sent to. + pub endpoint: &'a str, + /// The routed input the request was built from. + pub input: &'a ProviderAuctionInput, + /// How long the request took, in milliseconds. + pub response_time_ms: u64, + /// State [`CompiledDemand::capture_request`] returned for this request. + pub captured: Option<&'a (dyn Any + Send + Sync)>, +} + +/// The extension objects an implementation may add to a request the driver +/// built, one for the request and one for each impression. +/// +/// This is the only part of the request an implementation writes. Every +/// standard field is the driver's, chosen through [`DemandFieldPolicy`], so an +/// implementation can add what its exchange needs and cannot change what +/// privacy enforcement decided. The `trusted_server` request extension is the +/// driver's too, and a request that claims it is refused. +pub struct RequestExtensions<'a> { + /// The request-level `ext` object. + pub request: &'a mut Option>, + /// The impressions the driver built, each beside the slot it came from. + pub impressions: Vec>, +} + +/// One impression's `ext` object beside the routed slot it was built from. +pub struct ImpressionExtension<'a> { + /// The routed slot, with its bidder parameters and zone. + pub slot: &'a ProviderSlotInput, + /// The impression's `ext` object. + pub ext: &'a mut Option>, + /// Set by an implementation that has no demand to send for this slot. + /// The driver leaves the impression out of the request, and makes no + /// request at all when every impression is left out. + pub omitted: bool, +} + +/// The request and response behavior of one selected demand source, compiled +/// from its `[demand.]` table at startup. +/// +/// Every method except [`augment_request`](Self::augment_request) and +/// [`parse_response`](Self::parse_response) has a default that suits a source +/// with no special handling. +#[async_trait(?Send)] +pub trait CompiledDemand: Send + Sync { + /// The standard field choices this source makes. + fn field_policy(&self) -> DemandFieldPolicy; + + /// The domain sent as `site.domain` and `site.publisher.domain`. + fn site_domain(&self, publisher_domain: &str) -> String { + publisher_domain.to_owned() + } + + /// The page sent as `site.page`, given the publisher's page and the site + /// domain [`site_domain`](Self::site_domain) chose. + fn site_page(&self, publisher_page: Option<&str>, site_domain: &str) -> Option { + let _ = site_domain; + publisher_page.map(str::to_owned) + } + + /// The consent that may travel in the request body. + fn body_consent<'c>(&self, consent: Option<&'c ConsentContext>) -> Option<&'c ConsentContext> { + consent + } + + /// Adds this source's own request and impression extensions. + /// + /// # Errors + /// + /// Returns an error when an extension cannot be built for this request. + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + input: &ProviderAuctionInput, + ) -> Result<(), Report>; + + /// Adds to the outbound headers before the request is sent, for example to + /// forward admitted headers. The driver has already set `Content-Type`, + /// and `Accept` when the field policy asks for it. + fn prepare_outbound(&self, headers: &mut HeaderMap, transport: DemandTransport<'_>) { + let _ = (headers, transport); + } + + /// Keeps request-local state the response parser needs, taken from the + /// serialized body and the outbound headers as they will be sent, after + /// [`prepare_outbound`](Self::prepare_outbound) has run. + fn capture_request( + &self, + body: &[u8], + headers: &HeaderMap, + ) -> Option> { + let _ = (body, headers); + None + } + + /// Reads the upstream response into an auction response. + /// + /// # Errors + /// + /// Returns an error when the response cannot be read at all. A response + /// that arrived but is not usable is reported as an error response rather + /// than as an `Err`. + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result>; + + /// This compiled source as [`Any`], so the implementation that created it + /// can read its own settings back, for example while it registers routes. + fn as_any(&self) -> &dyn Any; +} + +/// Compiles one demand source from its settings. +/// +/// The map holds the table's own settings, without `implementation` and the +/// common keys the driver reads, which are `endpoint`, `timeout_ms`, `routing` +/// and `notifications`. +pub type CompileDemandFn = + fn(&Map) -> Result, Report>; + +/// Builds an ad server from its configured name and its settings. +/// +/// The map holds the table's own settings without `implementation`, and the +/// name is what the ad server reports itself as in responses and telemetry. +pub type BuildAdServerFn = + fn(&str, &Map) -> Result, Report>; + +/// A demand implementation an integration registers, named by +/// `[demand] modules` or by an `implementation` line. +#[derive(Clone, Copy)] +pub struct DemandImplementation { + /// The implementation's name, its module path. + pub id: &'static str, + /// The timeout a source uses when its table sets none. + pub default_timeout: DemandTimeoutDefault, + /// Whether `routing = "all_eligible"` is allowed. + pub allows_all_eligible: bool, + /// Whether a slot that carries no bidder parameters is still sent, as a + /// stored request, with the slot's zone. + pub serves_stored_requests: bool, + /// Checks and canonicalizes the endpoint after the common checks passed. + pub canonicalize_endpoint: fn(&mut Url) -> Result<(), String>, + /// Compiles one source from its settings. + pub compile: CompileDemandFn, +} + +impl core::fmt::Debug for DemandImplementation { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("DemandImplementation") + .field("id", &self.id) + .field("default_timeout", &self.default_timeout) + .field("allows_all_eligible", &self.allows_all_eligible) + .field("serves_stored_requests", &self.serves_stored_requests) + .finish_non_exhaustive() + } +} + +/// Accepts every endpoint the common checks accepted, unchanged. +/// +/// # Errors +/// +/// Never returns an error. +pub fn accept_endpoint(endpoint: &mut Url) -> Result<(), String> { + let _ = endpoint; + Ok(()) +} + +/// An ad server implementation an integration registers, named by +/// `[ad-server] module` or by an `implementation` line. +#[derive(Clone, Copy)] +pub struct AdServerImplementation { + /// The implementation's name, its module path. + pub id: &'static str, + /// Builds the ad server from its configured name and its settings. + pub build: BuildAdServerFn, +} + +impl core::fmt::Debug for AdServerImplementation { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("AdServerImplementation") + .field("id", &self.id) + .finish_non_exhaustive() + } +} diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f364464561..74bda35b40 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -9,8 +9,8 @@ use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::COOKIE_TS_EIDS; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_permissions}; +use crate::constants::COOKIE_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; @@ -26,7 +26,7 @@ use crate::settings::Settings; use super::AuctionOrchestrator; use super::formats::{ - convert_to_openrtb_response, convert_to_openrtb_response_with_report, + attach_device_attributes, convert_to_openrtb_response, convert_to_openrtb_response_with_report, convert_tsjs_to_auction_request, }; use super::telemetry::{ @@ -171,10 +171,11 @@ pub async fn handle_auction( // Story 5 middleware contract: auction is a read-only EC route. // It must not generate EC IDs; it only consumes pre-routed context. - // Only forward the EC ID to auction partners when consent allows it. - // Owned so the identity-graph snapshot can be stored back on `ec_context` - // below without holding a borrow of it across the mutation. - let ec_id = if ec_context.ec_allowed() { + // Forward the EC ID to auction partners only when sharing is permitted, + // being storage plus personalized-ad selection, the same pair that gates + // EIDs. Owned so the identity-graph snapshot can be stored back on + // `ec_context` below without holding a borrow of it across the mutation. + let ec_id = if ec_context.ec_sharing_allowed() { ec_context.ec_value().map(str::to_owned) } else { None @@ -211,7 +212,7 @@ pub async fn handle_auction( let empty_result = OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 0, metadata: HashMap::new(), @@ -264,7 +265,7 @@ pub async fn handle_auction( let empty_result = OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 0, metadata: HashMap::new(), @@ -282,8 +283,9 @@ pub async fn handle_auction( // `ts-eids` cookie so later requests can still forward the browser's // full OpenRTB-style EID structure. // - // Gate this on the same identity-consent condition as the EC ID - // (`ec_id.is_some()`, which is already filtered by `ec_context.ec_allowed()`). + // Gate this on the same identity condition as the EC ID + // (`ec_id.is_some()`, which is already filtered by the sharing pair via + // `ec_context.ec_sharing_allowed()`). // Otherwise a US/GPC or US-Privacy opt-out context — where EC identity use is // denied but a non-personalized auction may still run — could forward // persistent client EIDs from the body/cookie, since `gate_eids_by_consent` @@ -293,7 +295,11 @@ pub async fn handle_auction( let client_eids = if ec_id.is_some() { resolve_client_auction_eids( body.eids.as_ref(), - extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), + extract_cookie_value( + &http_req, + &services.public_prefix().cookie_name(COOKIE_EIDS), + ) + .as_deref(), ) } else { None @@ -309,8 +315,13 @@ pub async fn handle_auction( // EC and both KV and partner stores are available. Gate the read on a // present registry: without one, `resolve_auction_eids` yields no // server-side EIDs, so the snapshot would be an unused billable KV read. + // The row is read under the owning module's canonical form of the + // identifier, the key it is stored under, rather than under the identifier + // as issued. let auction_kv_snapshot = match (kv, ec_id.as_deref(), registry) { - (Some(graph), Some(ec_id), Some(_)) => graph.load_snapshot(ec_id), + (Some(graph), Some(_), Some(_)) => ec_context + .ec_kv_key() + .map_or(EcKvSnapshot::NotRead, |kv_key| graph.load_snapshot(&kv_key)), _ => EcKvSnapshot::NotRead, }; // Hand the loaded row to the request context so response finalization — @@ -325,7 +336,8 @@ pub async fn handle_auction( // Look up geo for device info. let geo = services .geo() - .lookup(services.client_info().client_ip) + .lookup(services.client_info().client_ip, services) + .await .unwrap_or_else(|e| { log::warn!("geo lookup failed: {e}"); None @@ -342,14 +354,17 @@ pub async fn handle_auction( geo, )?; + // Ask the selected device module what the bidder should be told about + // the device. + attach_device_attributes(&mut auction_request, &http_req, settings, services).await; + // Merge current-request client EIDs with KV-resolved EIDs, then apply // consent gating before attaching them to the auction request. let merged_eids = merge_auction_eids(client_eids, eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); - auction_request.user.eids = - gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); + auction_request.user.eids = gate_eids_by_permissions(merged_eids, ec_context.permissions()); if had_eids && auction_request.user.eids.is_none() { - log::warn!("Auction EIDs stripped by TCF consent gating"); + log::warn!("Auction EIDs stripped: bidstream permissions not set"); } // Create auction context @@ -457,7 +472,13 @@ pub(crate) fn resolve_auction_eids( let ec_id = ec_context.ec_value()?; - let Some(entry) = snapshot.entry_for(ec_id) else { + // Callers read the snapshot under the identity-graph key, the owning + // module's canonical form of the identifier, so the entry is looked up + // under that key rather than under the identifier as issued. + let Some(entry) = ec_context + .kv_key_for(ec_id) + .and_then(|kv_key| snapshot.entry_for(&kv_key)) + else { return Some(Vec::new()); }; @@ -630,6 +651,7 @@ mod tests { use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; + use crate::ec::tests::{CANONICAL_COOKIE_VALUE, CANONICAL_KV_KEY, CanonicalizingModule}; use crate::error::IntoHttpResponse as _; use crate::openrtb::Uid; use crate::platform::test_support::{ @@ -677,13 +699,25 @@ mod tests { .build() } - fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { - EcContext::new_for_test( + fn make_ec_context(ec_allowed: bool, ec_value: Option<&str>) -> EcContext { + EcContext::new_for_test_gated( + ec_value.map(str::to_owned), + ConsentContext::default(), + ec_allowed, + ) + } + + /// A context in a non-regulated jurisdiction with the permission gate open, + /// for tests whose subject is dispatch or identity reuse rather than the + /// jurisdiction gate. + fn make_non_regulated_ec_context(ec_value: Option<&str>) -> EcContext { + EcContext::new_for_test_gated( ec_value.map(str::to_owned), ConsentContext { - jurisdiction, + jurisdiction: Jurisdiction::NonRegulated, ..ConsentContext::default() }, + true, ) } @@ -716,9 +750,9 @@ mod tests { let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + provider_names: vec!["eid_capturing_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }); orchestrator.register_provider(Arc::new(EidCapturingProvider { @@ -735,7 +769,7 @@ mod tests { graph.create(&ec_id, &live).expect("should seed live row"); lookups.store(0, std::sync::atomic::Ordering::Relaxed); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let mut ec_context = make_non_regulated_ec_context(Some(&ec_id)); let req = Request::builder() .method("POST") .uri("https://test-publisher.com/auction") @@ -785,7 +819,9 @@ mod tests { None, Some("shared-cookie-id"), &mut response, - ); + &crate::platform::test_support::noop_services(), + ) + .await; assert_eq!( lookups.load(std::sync::atomic::Ordering::Relaxed), @@ -803,6 +839,84 @@ mod tests { ); } + #[tokio::test] + async fn auction_endpoint_loads_the_row_under_the_canonical_key() { + // The identity graph stores a row under the owning module's + // canonical form of the identifier. Loaded and resolved under the + // identifier as issued, a module whose canonical form differs from + // the cookie value found no row, so the auction carried no server-side + // EIDs and the context kept a snapshot bound to the wrong key. + let settings = create_test_settings(); + let had_eids = Arc::new(std::sync::Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + provider_names: vec!["eid_capturing_provider".to_string()], + timeout_ms: 2000, + adserver_name: None, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::clone(&had_eids), + })); + let registry = PartnerRegistry::from_config(&[counting_test_partner("ssp.example.com")]) + .expect("should build partner registry"); + let graph = KvIdentityGraph::in_memory("canonical-auction-store"); + graph + .create( + CANONICAL_KV_KEY, + &crate::ec::kv_types::KvEntry::minimal( + "ssp.example.com", + "partner-uid-123", + 1_741_824_000, + ), + ) + .expect("should seed the row under the canonical key"); + let mut ec_context = make_non_regulated_ec_context(Some(CANONICAL_COOKIE_VALUE)) + .with_module_for_test(Arc::new(CanonicalizingModule)); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + })) + .expect("should serialize body"), + )) + .expect("should build auction request"); + + // The capturing provider records whether the request carried EIDs and + // then fails its launch, which is all this test needs. The request + // carries no client EIDs, so any EID it records came from the graph. + let _ = handle_auction( + &settings, + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + + assert!( + ec_context + .kv_snapshot() + .entry_for(CANONICAL_KV_KEY) + .is_some(), + "the endpoint should load the row stored under the canonical key" + ); + assert_eq!( + *had_eids.lock().expect("should lock captured eids"), + Some(true), + "the auction should carry the canonical row's partner ID as an EID" + ); + } + #[tokio::test] async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { // Regression test for #1184: `/auction` sends every EID it has in the @@ -814,9 +928,9 @@ mod tests { let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + provider_names: vec!["eid_capturing_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }); orchestrator.register_provider(Arc::new(EidCapturingProvider { @@ -831,7 +945,15 @@ mod tests { live.consent.ok = true; graph.create(&ec_id, &live).expect("should seed live row"); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let mut ec_context = EcContext::new_for_test_gated( + Some(ec_id.clone()), + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }, + true, + ) + .with_module_for_test(crate::ec::tests::hmac_module()); ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Auction); let req = Request::builder() .method("POST") @@ -883,7 +1005,9 @@ mod tests { None, // No `ts-eids` cookie on this request at all. None, &mut response, - ); + &noop_services(), + ) + .await; let (stored, _) = graph .get(&ec_id) @@ -940,7 +1064,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for TemplateSwitchProbeProvider { fn provider_name(&self) -> &'static str { - "template-switch-probe" + "template_switch_probe" } async fn request_bids( @@ -992,13 +1116,15 @@ mod tests { #[tokio::test] async fn direct_auction_remains_available_when_templates_are_disabled() { let settings_toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.template-switch-probe]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[demand]\nmodules = [\"template_switch_probe\"]\n\n[demand.template_switch_probe]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); let settings = Settings::from_toml(&settings_toml) .expect("should parse settings with disabled templates"); let calls = Arc::new(Mutex::new(0)); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { calls: Arc::clone(&calls), })); @@ -1014,7 +1140,11 @@ mod tests { .geo(Arc::new(NoopGeo)) .client_info(ClientInfo::default()) .build(); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }; + let mut ec_context = EcContext::new_for_test_gated(None, consent, true); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -1054,16 +1184,16 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: false, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + provider_names: vec!["panic_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(PanicOnBidProvider)); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let mut ec_context = make_non_regulated_ec_context(None); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -1114,8 +1244,8 @@ mod tests { #[tokio::test] async fn all_planned_launch_failures_return_bad_gateway_and_execution_failed_telemetry() { let settings_toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.launch-fail]\nprotocol = \"openrtb-2.6\"\nprofile = \"standard\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n", - crate_test_settings_str() + "{}\n[demand]\nmodules = [\"launch_fail\"]\n\n[demand.launch_fail]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); let settings = Settings::from_toml(&settings_toml).expect("should parse launch-failure settings"); @@ -1126,7 +1256,7 @@ mod tests { let orchestrator = AuctionOrchestrator::from_plan(plan, None); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let mut ec_context = make_non_regulated_ec_context(None); let body = json!({ "adUnits": [{ "code": "div-gpt-ad-1", @@ -1178,9 +1308,9 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + provider_names: vec!["panic_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); @@ -1188,7 +1318,9 @@ mod tests { let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_telemetry(Arc::clone(&telemetry_sink)); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + // The default consent context keeps the jurisdiction unknown, so the + // server-side auction gate fails closed; the EC gate is off to match. + let mut ec_context = make_ec_context(false, Some(&ec_id)); let body = json!({ "adUnits": [ @@ -1306,9 +1438,9 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + provider_names: vec!["eid_capturing_provider".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); @@ -1355,7 +1487,7 @@ mod tests { let req = Request::builder() .method("POST") .uri("https://test-publisher.com/auction") - .header("cookie", format!("{COOKIE_TS_EIDS}={encoded_cookie}")) + .header("cookie", format!("ts-{COOKIE_EIDS}={encoded_cookie}")) .body(EdgeBody::from( serde_json::to_vec(&body).expect("should serialize body"), )) @@ -1385,7 +1517,7 @@ mod tests { fn resolve_auction_eids_returns_empty_without_snapshot() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = make_ec_context(true, Some(&ec_id)); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( @@ -1397,7 +1529,7 @@ mod tests { #[test] fn resolve_auction_eids_returns_none_without_registry() { let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = make_ec_context(true, Some(&ec_id)); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, None, &ec_context); assert!( @@ -1410,7 +1542,7 @@ mod tests { fn resolve_auction_eids_returns_none_when_consent_denied() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + let ec_context = make_ec_context(false, Some(&ec_id)); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( @@ -1422,7 +1554,11 @@ mod tests { #[test] fn resolve_auction_eids_returns_none_when_no_ec() { let registry = PartnerRegistry::empty(); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }; + let ec_context = EcContext::new_for_test_gated(None, consent, true); let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); assert!( @@ -1435,7 +1571,7 @@ mod tests { fn resolve_auction_eids_returns_empty_on_kv_miss() { let registry = PartnerRegistry::empty(); let ec_id = format!("{}.ABC123", "a".repeat(64)); - let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let ec_context = make_ec_context(true, Some(&ec_id)); let snapshot = EcKvSnapshot::Failed { ec_id: ec_id.clone(), diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index a0df63d94a..f4d2c1c839 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -20,6 +20,7 @@ use crate::creative; use crate::ec::eids::encode_eids_header; use crate::error::TrustedServerError; use crate::geo::GeoInfo; +use crate::module_context::{ModuleContext, ModuleRequest}; use crate::openrtb::{ BidExt, BidTrustedServerExt, OpenRtbBid, OpenRtbResponse, ResponseExt, SeatBid, ToExt, to_openrtb_i32, @@ -224,6 +225,8 @@ pub fn convert_tsjs_to_auction_request( .map(str::to_string), ip: services.client_info().client_ip.map(|ip| ip.to_string()), geo, + // Filled by `attach_device_attributes`. + attributes: None, }); // Forward allowed config entries from the JS request into the context map. @@ -481,11 +484,12 @@ pub(crate) fn convert_to_openrtb_response_with_report( delivery.delivered_winner_slots.insert(slot_id.clone()); } - // Determine strategy name for response metadata - let strategy_name = if settings.auction.has_mediator() { - "parallel_mediation" - } else { + // The strategy names what the deployment selected, so it reads the same + // whether or not the ad server answered this auction. + let strategy_name = if settings.adserver.is_empty() { "parallel_only" + } else { + "parallel_adserver" }; // Build per-provider summaries from the orchestration result @@ -551,23 +555,63 @@ pub(crate) fn convert_to_openrtb_response_with_report( Ok(OpenRtbResponseConversion { response, delivery }) } +/// Fills the bid request's device attributes from the selected device module, +/// after each request that makes an outbound auction call is built. +pub async fn attach_device_attributes( + request: &mut AuctionRequest, + http_req: &http::Request, + settings: &Settings, + services: &RuntimeServices, +) { + let Some(module) = services.device_module() else { + return; + }; + let client_ip = services + .client_info() + .client_ip + .map_or_else(String::new, |address| address.to_string()); + let request_info = + crate::evidence::BorrowedRequestInfo::new(&client_ip, Some(http_req.headers())); + let context = ModuleContext::new(ModuleRequest::of(http_req, services.client_info())) + .with_evidence(&request_info) + .with_settings(settings) + .with_services(services); + let Some(attributes) = module + .advertising_attributes(context.call(module.id(), module.required_permissions())) + .await + else { + return; + }; + // An empty object and an absent one say the same to a bidder, and the + // absent one is smaller. + if attributes.is_empty() { + return; + } + + request + .device + .get_or_insert(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + attributes: None, + }) + .attributes = Some(attributes); +} + #[cfg(test)] mod tests { use super::*; - use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, - }; + use crate::auction::plan::AuctionPlan; use crate::auction::routing::route_auction; - use crate::auction::types::{ - ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, BidStatus, - }; + use crate::auction::test_support::{demand_table, plan_config}; + use crate::auction::types::{AuctionResponse, Bid, BidRenderer, BidStatus}; use crate::openrtb::{Eid, Uid}; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; use http::Method; use serde_json::json; - use std::collections::{BTreeMap, BTreeSet, HashSet}; - use std::str::FromStr as _; + use std::collections::{BTreeSet, HashSet}; fn make_request() -> Request { Request::builder() @@ -582,26 +626,200 @@ mod tests { create_test_settings() } - fn single_prebid_plan() -> AuctionPlan { - AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 900, - providers: BTreeMap::from([( - ProviderId::from_str("pbs-primary").expect("should parse provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "prebid-server".to_string(), - endpoint: "https://pbs.example.test/openrtb".to_string(), - timeout_ms: None, - routing: RoutingMode::Explicit, - notifications: NotificationConfig::default(), - profile_config: json!({}), - }, - )]), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, - }) - .expect("should compile plan") + fn single_stored_plan() -> AuctionPlan { + let mut config = plan_config(vec![( + "stored_primary", + demand_table("auction.fixture", "https://stored.example.test/openrtb"), + )]); + config.timeout_ms = 900; + AuctionPlan::compile(config).expect("should compile plan") + } + + /// A device module that resolves fixed attributes, standing in for a + /// vendor that reaches a service. + #[derive(Debug)] + struct StubDeviceModule { + attributes: Option, + } + + #[async_trait::async_trait(?Send)] + impl crate::ec::device::DeviceModule for StubDeviceModule { + fn id(&self) -> &'static str { + "stub" + } + + async fn detect( + &self, + _call: crate::module_context::ModuleCall<'_>, + ) -> crate::ec::device::DeviceSignals { + crate::ec::device::DeviceSignals::unknown() + } + + async fn advertising_attributes( + &self, + _call: crate::module_context::ModuleCall<'_>, + ) -> Option { + self.attributes.clone() + } + } + + fn empty_request() -> http::Request { + http::Request::builder() + .uri("https://www.example.com/") + .body(EdgeBody::empty()) + .expect("should build the request") + } + + fn stub_attributes() -> crate::ec::device::DeviceAttributes { + crate::ec::device::DeviceAttributes { + device_type: Some(4), + make: Some("ExampleCorp".to_owned()), + model: Some("EX-1".to_owned()), + os: Some("Android".to_owned()), + os_version: Some("15.0".to_owned()), + screen_width: Some(1080), + screen_height: Some(2400), + } + } + + fn request_with_device(device: Option) -> AuctionRequest { + let mut request = make_auction_request(); + request.device = device; + request + } + + #[tokio::test] + async fn the_selected_module_fills_the_device_object() { + let services = crate::platform::test_support::noop_services_with_device_module( + std::sync::Arc::new(StubDeviceModule { + attributes: Some(stub_attributes()), + }), + ); + let mut request = request_with_device(Some(DeviceInfo { + user_agent: Some("Mozilla/5.0".to_owned()), + ip: None, + geo: None, + attributes: None, + })); + + attach_device_attributes( + &mut request, + &empty_request(), + &crate::test_support::tests::create_test_settings(), + &services, + ) + .await; + + assert_eq!( + request + .device + .as_ref() + .and_then(|device| device.attributes.as_ref()), + Some(&stub_attributes()), + "a resolved device must reach the request the bidder is sent" + ); + } + + #[tokio::test] + async fn a_module_that_resolves_an_empty_device_adds_no_device_object() { + let services = crate::platform::test_support::noop_services_with_device_module( + std::sync::Arc::new(StubDeviceModule { + attributes: Some(crate::ec::device::DeviceAttributes::default()), + }), + ); + let mut request = request_with_device(None); + + attach_device_attributes( + &mut request, + &empty_request(), + &crate::test_support::tests::create_test_settings(), + &services, + ) + .await; + + assert!( + request.device.is_none(), + "an empty device object says nothing a bidder cannot read from its absence" + ); + } + + #[tokio::test] + async fn no_device_module_leaves_the_request_untouched() { + let services = crate::platform::test_support::noop_services(); + let mut request = request_with_device(Some(DeviceInfo { + user_agent: Some("Mozilla/5.0".to_owned()), + ip: None, + geo: None, + attributes: None, + })); + + attach_device_attributes( + &mut request, + &empty_request(), + &crate::test_support::tests::create_test_settings(), + &services, + ) + .await; + + assert!( + request + .device + .as_ref() + .is_some_and(|device| device.attributes.is_none()), + "the default deployment supplies no device module and must send no attributes" + ); + } + + #[tokio::test] + async fn a_module_that_resolves_nothing_adds_no_device_object() { + let services = crate::platform::test_support::noop_services_with_device_module( + std::sync::Arc::new(StubDeviceModule { attributes: None }), + ); + let mut request = request_with_device(None); + + attach_device_attributes( + &mut request, + &empty_request(), + &crate::test_support::tests::create_test_settings(), + &services, + ) + .await; + + assert!( + request.device.is_none(), + "an unresolved device must not become an empty device object, which a bidder \ + reads as a claim that the fields are absent rather than unknown" + ); + } + + #[tokio::test] + async fn resolved_attributes_create_the_device_object_when_there_was_none() { + let services = crate::platform::test_support::noop_services_with_device_module( + std::sync::Arc::new(StubDeviceModule { + attributes: Some(stub_attributes()), + }), + ); + // The publisher path leaves the device object out entirely when the + // request carried no User-Agent, so this is a real state and not a + // contrived one. + let mut request = request_with_device(None); + + attach_device_attributes( + &mut request, + &empty_request(), + &crate::test_support::tests::create_test_settings(), + &services, + ) + .await; + + assert_eq!( + request + .device + .as_ref() + .and_then(|device| device.attributes.as_ref()), + Some(&stub_attributes()), + "attributes that were resolved must not be dropped for want of a device object" + ); } fn make_auction_request() -> AuctionRequest { @@ -636,7 +854,7 @@ mod tests { fn make_empty_result() -> OrchestrationResult { OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 10, metadata: HashMap::new(), @@ -685,7 +903,7 @@ mod tests { response_time_ms: 42, metadata: HashMap::new(), }], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), total_time_ms: 50, metadata: HashMap::new(), @@ -757,7 +975,7 @@ mod tests { "bids":[{"bidder":"trustedServer","params":{"bidderParams":{},"storedRequest":intent}}]}] })).expect("should deserialize wire request"); let request = convert_body_to_auction_request(&body, &make_settings()); - let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + let routed = route_auction(request, &make_request(), &single_stored_plan(), None); assert_eq!(routed.inputs().len(), expected_inputs); assert_eq!(routed.diagnostics().malformed_envelope_count(), malformed); } @@ -779,7 +997,7 @@ mod tests { eids: None, }; let request = convert_body_to_auction_request(&body, &make_settings()); - let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + let routed = route_auction(request, &make_request(), &single_stored_plan(), None); assert_eq!(routed.inputs().len(), 1); assert!( @@ -1503,7 +1721,7 @@ mod tests { response_time_ms: 42, metadata: HashMap::from([("debug".to_string(), debug.clone())]), }], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), total_time_ms: 50, metadata: HashMap::new(), @@ -1536,24 +1754,25 @@ mod tests { rejected.creative = Some("".to_string()); let unpriced = make_bid("unpriced", "invalid", None); let ordinary = make_bid("ordinary", "appnexus", Some(2.75)); - let mut renderer = make_bid("renderer", "aps", Some(2.5)); + let mut renderer = make_bid("renderer", "example", Some(2.5)); renderer.creative = Some("".to_string()); renderer.bid_id = Some("upstream-renderer-bid".to_string()); renderer.creative_id = None; - renderer.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "upstream-renderer-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + renderer.renderer = Some( + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "upstream-renderer-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor"), + ); let result = OrchestrationResult { provider_responses: vec![], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([ (missing.slot_id.clone(), missing), (whitespace.slot_id.clone(), whitespace), @@ -1637,18 +1856,19 @@ mod tests { fn convert_to_openrtb_response_prefers_creative_when_both_render_sources_exist() { let settings = make_settings(); let auction_request = make_auction_request(); - let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + let mut bid = make_bid("div-gpt-top", "example", Some(2.75)); + bid.renderer = Some( + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "fictional-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor"), + ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) @@ -1670,29 +1890,30 @@ mod tests { } #[test] - fn convert_to_openrtb_response_emits_typed_aps_renderer_without_adm() { + fn convert_to_openrtb_response_emits_a_typed_renderer_without_adm() { let settings = make_settings(); let auction_request = make_auction_request(); - let mut bid = make_bid("div-gpt-top", "aps", Some(2.75)); + let mut bid = make_bid("div-gpt-top", "example", Some(2.75)); bid.creative = None; bid.bid_id = Some("fictional-bid".to_string()); bid.ad_id = Some("fictional-ad".to_string()); bid.creative_id = Some("fictional-creative".to_string()); - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "fictional-bid".to_string(), - creative_id: Some("fictional-creative".to_string()), - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::new( + "example", + json!({ + "version": 1, + "bidId": "fictional-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor"), + ); let result = make_result(bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) - .expect("should convert APS renderer bid"); + .expect("should convert a typed renderer bid"); let json = response_json(response); let bid = json["seatbid"][0]["bid"][0] .as_object() @@ -1703,12 +1924,12 @@ mod tests { "should omit adm for renderer bids" ); assert_eq!(bid["id"], json!("fictional-bid")); - assert_eq!(json["seatbid"][0]["seat"], json!("aps")); + assert_eq!(json["seatbid"][0]["seat"], json!("example")); assert_eq!(bid["adid"], json!("fictional-ad")); assert_eq!(bid["crid"], json!("fictional-creative")); assert_eq!( bid["ext"]["trusted_server"]["renderer"]["type"], - json!("aps") + json!("example") ); assert_eq!( bid["ext"]["trusted_server"]["renderer"]["bidId"], @@ -1744,7 +1965,7 @@ mod tests { let auction_request = make_auction_request(); let result = OrchestrationResult { provider_responses: vec![], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 50, metadata: HashMap::new(), @@ -1781,7 +2002,7 @@ mod tests { response_time_ms: 42, metadata: HashMap::new(), }], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([ (top_bid.slot_id.clone(), top_bid), (sidebar_bid.slot_id.clone(), sidebar_bid), @@ -1868,9 +2089,12 @@ mod tests { } #[test] - fn convert_to_openrtb_response_uses_parallel_mediation_when_mediator_configured() { + fn convert_to_openrtb_response_uses_parallel_adserver_when_adserver_configured() { let mut settings = make_settings(); - settings.auction.mediator = Some("adserver_mock".to_string()); + settings.adserver = crate::provider_table::ProviderChoice::new( + Some("mock".to_string()), + std::collections::BTreeMap::new(), + ); let auction_request = make_auction_request(); let result = make_result(make_bid("div-gpt-top", "appnexus", Some(2.75))); @@ -1880,8 +2104,8 @@ mod tests { assert_eq!( json["ext"]["orchestrator"]["strategy"], - json!("parallel_mediation"), - "should use mediation strategy when mediator is configured" + json!("parallel_adserver"), + "should use ad server decision strategy when adserver is configured" ); } @@ -2028,8 +2252,11 @@ mod convert_tests { // (empty) applies — verify a key is NOT present rather than IS. // To test the allow-list, inject a key via a custom settings string. let settings_str = format!( - "{}\n[auction]\nallowed_context_keys = [\"permutive_segments\"]\n", - crate_test_settings_str() + "{}\n", + crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nallowed_context_keys = [\"permutive_segments\"]\n" + ) ); let settings = Settings::from_toml(&settings_str).expect("should parse"); let services = noop_services(); diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 4323039284..9cde111aed 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -1,38 +1,42 @@ -//! Auction orchestration module for managing multi-provider bidding. +//! Auction orchestration across the demand sources and the ad server a +//! deployment selects. //! -//! This module provides an extensible framework for running auctions across -//! multiple providers (Prebid, Amazon APS, Google GAM, etc.) with support for -//! parallel execution and mediation strategies. -//! -//! Note: Individual auction providers are located in the `integrations` module -//! (e.g., `crate::integrations::aps`, `crate::integrations::prebid`). +//! `[demand] modules` selects the demand sources and `[ad-server] module` +//! the optional ad server. Their implementations are registered by +//! integrations through [`demand`], so this module names no vendor. use error_stack::Report; use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; use crate::settings::Settings; use std::sync::Arc; pub mod config; pub mod context; +pub mod demand; pub mod endpoints; pub mod formats; -pub(crate) mod openrtb; +pub mod openrtb; pub mod orchestrator; pub mod plan; -pub(crate) mod profile; pub mod provider; pub(crate) mod routing; pub mod telemetry; -#[cfg(test)] -pub(crate) mod test_support; +#[cfg(any(test, feature = "test-utils"))] +#[allow( + clippy::must_use_candidate, + clippy::missing_panics_doc, + reason = "test helpers, offered to other crates' tests under the test-utils feature" +)] +pub mod test_support; pub mod types; pub use config::AuctionConfig; pub use context::{ContextQueryParams, ContextValue, build_url_with_context_params}; pub use orchestrator::AuctionOrchestrator; pub use plan::{ - AuctionPlan, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, + AdServerPlan, AuctionPlan, BidderId, BidderRouteConfig, NotificationConfig, ProviderId, RoutingMode, }; pub use provider::AuctionProvider; @@ -45,66 +49,86 @@ pub use types::{ AdFormat, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; -/// Compile the canonical target-independent auction plan for [`Settings`]. +/// Compile the canonical target-independent auction plan for [`Settings`] with +/// the implementations the built-in integrations register. /// -/// This is the single settings-to-plan boundary used by deploy validation, -/// adapter startup, and operator tooling. Global request signing remains owned -/// by [`Settings`] and is copied into compiler input only at this boundary. +/// This is the settings-to-plan boundary used by deploy validation, adapter +/// startup and operator tooling. Global request signing remains owned by +/// [`Settings`] and is copied into compiler input only at this boundary. /// /// # Errors /// -/// Returns an error when auction provider, bidder route, signing, or mediator +/// Returns an error when the `[demand]`, `[ad-server]`, bidder route or signing /// configuration is invalid. pub fn compile_auction_plan( settings: &Settings, ) -> Result> { + compile_auction_plan_with(settings, &[]) +} + +/// Compile the auction plan with the implementations the built-in integrations +/// register followed by those the `extra` builders register, the builders an +/// adapter or a vendor crate supplies. +/// +/// # Errors +/// +/// Returns an error when the `[demand]`, `[ad-server]`, bidder route or signing +/// configuration is invalid, including a selected name whose implementation no +/// builder registers. +pub fn compile_auction_plan_with( + settings: &Settings, + extra: &[IntegrationBuilder], +) -> Result> { + let builders = crate::integrations::all_builders(extra).collect::>(); AuctionPlan::compile(plan::AuctionPlanConfig { timeout_ms: settings.auction.timeout_ms, - providers: settings.auction.providers.clone(), + demand: settings.demand.clone(), + adserver: settings.adserver.clone(), bidders: settings.auction.bidders.clone(), - mediator: settings.auction.mediator.clone(), request_signing: settings.request_signing.clone(), + demand_implementations: builders + .iter() + .filter_map(IntegrationBuilder::demand) + .collect(), + adserver_implementations: builders + .iter() + .filter_map(IntegrationBuilder::adserver) + .collect(), + }) + .map(|plan| { + plan.with_enabled(settings.auction.enabled) + .with_auction_token(builders.iter().any(|builder| { + builder.reads_auction_token() + && builder + .module_name() + .is_some_and(|name| settings.selects_module(name)) + })) }) - .map(|plan| plan.with_enabled(settings.auction.enabled)) } /// Build a new auction orchestrator from one shared compiled plan. /// -/// This constructor registers all auction providers discovered from the provided settings. -/// Callers can reuse the returned [`AuctionOrchestrator`] across requests. -/// -/// # Arguments -/// * `plan` - Shared immutable compiled plan -/// * `settings` - Application settings used only for the separately registered mediator +/// The demand sources come from the plan, and so does the ad server, which is +/// built here from its implementation and settings. Callers can reuse the +/// returned [`AuctionOrchestrator`] across requests. /// /// # Errors /// -/// Returns an error when an enabled auction provider has invalid configuration. +/// Returns an error when the selected ad server cannot be built from its +/// settings. pub fn build_orchestrator_with_plan( plan: Arc, - settings: &Settings, ) -> Result> { log::info!("Building plan-backed auction orchestrator"); - let mediator = if let Some(expected_id) = plan.mediator() { - let provider = crate::integrations::adserver_mock::register_providers(settings)? - .into_iter() - .find(|provider| provider.provider_name() == expected_id) - .ok_or_else(|| { - Report::new(TrustedServerError::Configuration { - message: format!( - "auction mediator `{expected_id}` must reference a separately registered enabled integration with the exact same ID" - ), - }) - })?; - Some(provider) - } else { - None - }; - let orchestrator = AuctionOrchestrator::from_plan(plan, mediator); + let adserver = plan + .adserver() + .map(|adserver| (adserver.implementation.build)(adserver.id.as_str(), &adserver.settings)) + .transpose()?; + let orchestrator = AuctionOrchestrator::from_plan(plan, adserver); log::info!( - "Auction orchestrator built with {} bidder providers", + "Auction orchestrator built with {} demand sources", orchestrator.provider_count() ); @@ -115,27 +139,97 @@ pub fn build_orchestrator_with_plan( /// /// # Errors /// -/// Returns an error when plan compilation or mediator construction fails. +/// Returns an error when plan compilation or ad server construction fails. #[cfg(test)] pub fn build_orchestrator( settings: &Settings, ) -> Result> { let plan = Arc::new(compile_auction_plan(settings)?); - build_orchestrator_with_plan(plan, settings) + build_orchestrator_with_plan(plan) +} + +#[cfg(test)] +mod auction_token_tests { + use super::*; + use crate::integrations::registry_test_support::{ + probe_registration, request_fixture, validate_nothing, + }; + use crate::test_support::tests::create_test_settings; + + #[test] + fn the_plan_publishes_a_token_only_where_a_selected_module_reads_one() { + let mut settings = create_test_settings(); + assert!( + !compile_auction_plan(&settings) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish no token where no module is selected" + ); + + settings.select_module("testing", request_fixture::MODULE); + assert!( + compile_auction_plan(&settings) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish a token once a module that reads one is selected" + ); + } + + #[test] + fn a_selected_module_that_reads_no_token_does_not_turn_one_on() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let reads_nothing = + [ + IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe"), + ]; + assert!( + !compile_auction_plan_with(&settings, &reads_nothing) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish no token for a selected module that reads none" + ); + + let reads_one = [reads_nothing[0].with_auction_token()]; + assert!( + compile_auction_plan_with(&settings, &reads_one) + .expect("should compile the plan") + .publishes_auction_token(), + "should publish a token for a module a vendor crate supplies that reads one" + ); + } } #[cfg(test)] mod plan_sharing_tests { use super::*; + use crate::auction::test_support::demand_named; use crate::integrations::IntegrationRegistry; + use crate::provider_table::ProviderChoice; use crate::test_support::tests::create_test_settings; + use serde_json::{Map, json}; + use std::collections::BTreeMap; + + /// An `[ad-server]` table selecting one name with the settings given. + fn adserver(name: &str, settings: Map) -> ProviderChoice { + ProviderChoice::new( + Some(name.to_string()), + BTreeMap::from([(name.to_string(), settings)]), + ) + } #[test] fn orchestrator_and_registry_share_the_compiled_plan_allocation() { let settings = create_test_settings(); let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); - let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings) - .expect("should build orchestrator"); + let orchestrator = + build_orchestrator_with_plan(Arc::clone(&plan)).expect("should build orchestrator"); let registry = IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) .expect("should build integration registry"); @@ -144,54 +238,99 @@ mod plan_sharing_tests { } #[test] - fn configured_mediator_requires_enabled_exact_registration() { - for mediator_config in [None, Some(serde_json::json!({"enabled": false}))] { - let mut settings = create_test_settings(); - settings.auction.mediator = Some("adserver_mock".to_string()); - if let Some(config) = mediator_config { - settings - .integrations - .insert_config("adserver_mock", &config) - .expect("should insert mediator config"); - } else { - settings.integrations.remove("adserver_mock"); - } - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); - - let error = match build_orchestrator_with_plan(plan, &settings) { - Ok(_) => panic!("should require enabled mediator registration"), - Err(error) => error, - }; - assert!(error.to_string().contains("adserver_mock")); - } + fn an_ad_server_this_build_does_not_have_fails_the_plan() { + let mut settings = create_test_settings(); + settings.adserver = + ProviderChoice::new(Some("fictional_adserver".to_string()), BTreeMap::new()); + let error = compile_auction_plan(&settings) + .expect_err("should refuse an ad server no builder registers"); + assert!( + error.to_string().contains("fictional_adserver"), + "should name the ad server: {error:?}" + ); + } + + /// The ad server implementation a crate outside core supplies. + static EXTERNAL_ADSERVER: crate::auction::demand::AdServerImplementation = + crate::auction::demand::AdServerImplementation { + id: "ad-server.example", + build: build_external_adserver, + }; + + fn build_external_adserver( + name: &str, + _settings: &Map, + ) -> Result, Report> { + Ok(Arc::new( + crate::auction::test_support::adserver_fixture::FixtureAdServer::new(name, 500), + )) } + /// An ad server a builder outside core supplies resolves in the plan when + /// that builder is passed, and is a name no builder registers when it is + /// not. Both halves are asserted, so the test fails if either stops + /// holding. #[test] - fn configured_mediator_builds_when_exact_registration_is_enabled() { + fn an_ad_server_an_external_builder_supplies_compiles_only_when_it_is_passed() { let mut settings = create_test_settings(); - settings.auction.mediator = Some("adserver_mock".to_string()); - settings - .integrations - .insert_config( - "adserver_mock", - &serde_json::json!({ - "enabled": true, - "endpoint": "https://mediator.example/mediate" - }), - ) - .expect("should insert mediator config"); + settings.adserver = adserver("example", Map::new()); + let extra = [crate::integrations::IntegrationBuilder::implementations( + "example-adserver", + "example-crate", + ) + .with_adserver(&EXTERNAL_ADSERVER)]; + + let error = compile_auction_plan(&settings) + .expect_err("built-ins alone should not know this ad server"); + assert!( + error.to_string().contains("example"), + "should name the ad server: {error:?}" + ); + + let plan = compile_auction_plan_with(&settings, &extra) + .expect("the external builder's ad server should compile"); + assert_eq!( + plan.adserver().map(|adserver| adserver.implementation.id), + Some("ad-server.example"), + "the plan should carry the external ad server" + ); + } + + #[test] + fn the_selected_ad_server_is_built_from_its_own_table() { + let mut settings = create_test_settings(); + settings.adserver = adserver( + "fixture", + Map::from_iter([( + "endpoint".to_string(), + json!("https://adserver.example/mediate"), + )]), + ); let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); - build_orchestrator_with_plan(plan, &settings) - .expect("should build with enabled exact mediator registration"); + build_orchestrator_with_plan(plan).expect("should build the selected ad server"); + } + + #[test] + fn an_ad_server_table_with_no_endpoint_fails_the_plan() { + let mut settings = create_test_settings(); + settings.adserver = adserver("fixture", Map::new()); + let error = compile_auction_plan(&settings) + .expect_err("should refuse an ad server with no endpoint"); + assert!( + format!("{error:?}").contains("endpoint"), + "should say an endpoint is needed: {error:?}" + ); } #[test] fn cloudflare_and_spin_reject_multi_provider_plans_before_runtime_construction() { let mut settings = create_test_settings(); settings.auction.enabled = true; - settings.auction.providers = - AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]); + settings.demand = demand_named( + crate::auction::test_support::plain_fixture::MODULE, + &["provider_a", "provider_b"], + ); let plan = compile_auction_plan(&settings).expect("should compile target-independent plan"); for target in [ @@ -208,26 +347,4 @@ mod plan_sharing_tests { ); } } - - #[test] - fn aps_profile_registers_renderer_without_browser_aps_config() { - let mut settings = create_test_settings(); - settings.auction.providers = std::collections::BTreeMap::from([( - "aps-main".parse().expect("should parse APS provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({"account_id":"example-account"}), - }, - )]); - let plan = Arc::new(compile_auction_plan(&settings).expect("should compile APS plan")); - let registry = IntegrationRegistry::with_plan(&settings, plan) - .expect("should build APS renderer registry"); - - assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); - } } diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs index 74553c9bce..fdea467979 100644 --- a/crates/trusted-server-core/src/auction/openrtb.rs +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -1,41 +1,51 @@ -//! Shared `OpenRTB` 2.6 request/response support for config-first providers. +//! The shared `OpenRTB` 2.6 request and response driver. //! -//! Profiles receive only routed, privacy-approved facts and never the raw -//! downstream request or unrestricted runtime services. +//! Every demand source's request is built here from routed, privacy-approved +//! facts, never from the raw downstream request or unrestricted runtime +//! services. A demand implementation decides only what +//! [`crate::auction::demand`] exposes, and this driver builds the rest the same +//! way for all of them. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; +use http::StatusCode; use serde_json::{Map, Value, json}; -use url::Url; -use super::plan::{NotificationPolicy, ProviderPlan}; -use super::profile::{ - ApsProfilePlan, CompiledOpenRtbProfile, PrebidProfilePlan, StandardProfilePlan, -}; -use super::routing::{ - PrebidTransportHeaders, ProviderAuctionInput, ProviderSlotInput, RoutedAuction, +use super::demand::{ + CompiledDemand, DemandFieldPolicy, DemandResponse, ImpressionExtension, RegsPolicy, + RequestExtensions, }; +use super::plan::{NotificationPolicy, ProviderPlan}; +use super::routing::{ProviderAuctionInput, ProviderSlotInput, RoutedAuction, TransportHeaders}; use super::types::{AdFormat, AdSlot, AuctionResponse, Bid}; -use crate::consent::ConsentSource; use crate::error::TrustedServerError; use crate::openrtb::{ Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, ToExt as _, TrustedServerExt, User, UserExt, to_openrtb_i32, }; +use crate::platform::PlatformResponse; use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; const DEFAULT_CURRENCY: &str = "USD"; -const APS_SDK_SOURCE: &str = "prebid"; -const APS_SDK_VERSION: &str = "2.2.0"; -const MAX_CONSERVATIVE_LANGUAGE_BYTES: usize = 8; +/// The request extension the driver writes and no implementation may claim. +const TRUSTED_SERVER_EXT_KEY: &str = "trusted_server"; /// Fixed reasons why an upstream bid failed response admission. +/// +/// Public, with the dimension helpers below, so an auction implementation in +/// a crate of its own admits a bid by the rules core's own response reader +/// applies. #[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) enum BidRejectionReason { +pub enum BidRejectionReason { + /// The bid is malformed. InvalidBid, + /// The bid names an impression the request did not carry. UnrequestedImpression, + /// The bid's dimensions match no format the impression asked for. DimensionMismatch, + /// The bid's dimensions match more than one format the impression asked + /// for. AmbiguousDimensions, } @@ -52,21 +62,21 @@ impl BidRejectionReason { /// Bounded aggregate diagnostics for rejected upstream bids. #[derive(Debug, Default)] -pub(crate) struct ResponseAdmissionDiagnostics { +pub struct ResponseAdmissionDiagnostics { rejected_bid_count: u32, reason_counts: BTreeMap, } impl ResponseAdmissionDiagnostics { /// Record one rejected bid without retaining upstream payload data. - pub(crate) fn record(&mut self, reason: BidRejectionReason) { + pub fn record(&mut self, reason: BidRejectionReason) { self.rejected_bid_count = self.rejected_bid_count.saturating_add(1); let count = self.reason_counts.entry(reason).or_default(); *count = count.saturating_add(1); } /// Attach fixed-cardinality rejection counts to a provider response. - pub(crate) fn attach_to(self, response: &mut AuctionResponse) { + pub fn attach_to(self, response: &mut AuctionResponse) { if self.rejected_bid_count == 0 { return; } @@ -107,7 +117,7 @@ impl DimensionMatch { } /// Precomputed dimension resolutions for one requested impression. -pub(crate) struct SlotBidDimensions { +pub struct SlotBidDimensions { exact: BTreeSet<(u32, u32)>, by_width: BTreeMap, by_height: BTreeMap, @@ -150,17 +160,19 @@ impl SlotBidDimensions { } /// Precomputed requested banner dimensions keyed by impression ID. -pub(crate) type BidDimensionIndex = BTreeMap; +pub type BidDimensionIndex = BTreeMap; /// Build the requested-dimension index once for one provider response. -pub(crate) fn build_bid_dimension_index(input: &ProviderAuctionInput) -> BidDimensionIndex { +#[must_use] +pub fn build_bid_dimension_index(input: &ProviderAuctionInput) -> BidDimensionIndex { build_bid_dimension_index_from_slots(input.slots().iter().map(ProviderSlotInput::slot)) } /// Build the requested-dimension index from plain [`AdSlot`]s. /// /// The first slot wins when several share an ID. -pub(crate) fn build_bid_dimension_index_from_slots<'a>( +#[must_use] +pub fn build_bid_dimension_index_from_slots<'a>( slots: impl IntoIterator, ) -> BidDimensionIndex { let mut index = BidDimensionIndex::new(); @@ -173,7 +185,12 @@ pub(crate) fn build_bid_dimension_index_from_slots<'a>( } /// Parse an optional positive `OpenRTB` bid dimension. -pub(crate) fn parse_optional_bid_dimension( +/// +/// # Errors +/// +/// Returns [`BidRejectionReason::InvalidBid`] when the value is present and is +/// not a positive whole number that fits. +pub fn parse_optional_bid_dimension( value: &Value, key: &str, ) -> Result, BidRejectionReason> { @@ -201,7 +218,12 @@ pub(crate) fn parse_optional_bid_dimension( } /// Validate explicit dimensions or infer them from one matching banner format. -pub(crate) fn resolve_bid_dimensions( +/// +/// # Errors +/// +/// Returns the reason the bid is refused: the impression was not requested, +/// or the dimensions match none or more than one of its formats. +pub fn resolve_bid_dimensions( dimensions_by_slot: &BidDimensionIndex, slot_id: &str, width: Option, @@ -253,12 +275,13 @@ pub(crate) struct RequestFinalization<'a> { pub(crate) signing_params: SigningParams, } -/// Build one provider request from its immutable routed input. +/// Build one demand source's request from its immutable routed input. /// /// # Errors /// -/// Returns an auction error when static/profile extensions cannot be merged or -/// the supplied signing input does not bind the already-fixed request ID. +/// Returns an auction error when an implementation's own extensions cannot be +/// built or the supplied signing input does not bind the already-fixed request +/// ID. pub(crate) fn build_request( input: &ProviderAuctionInput, routed: &RoutedAuction, @@ -266,101 +289,90 @@ pub(crate) fn build_request( effective_timeout_ms: u32, finalization: &RequestFinalization<'_>, ) -> Result> { - let policy = ProfilePolicy::from(&provider.profile); - let mut request = build_common_request(input, routed, policy, effective_timeout_ms); + let demand = provider.demand.as_ref(); + let policy = demand.field_policy(); + let (mut request, slot_indices) = + build_common_request(input, routed, demand, policy, effective_timeout_ms); if request.imp.is_empty() { return Ok(OpenRtbBuildOutcome::NoImpressions); } - policy.augment_request(&mut request, input, routed)?; + let omitted = { + // The implementation sees the extension objects and the slot each + // impression came from, and nothing else of the request. + let OpenRtbRequest { ext, imp, .. } = &mut request; + let impressions = imp + .iter_mut() + .zip(&slot_indices) + .map(|(imp, &index)| ImpressionExtension { + slot: &input.slots()[index], + ext: &mut imp.ext, + omitted: false, + }) + .collect(); + let mut extensions = RequestExtensions { + request: ext, + impressions, + }; + demand.augment_request(&mut extensions, input)?; + extensions + .impressions + .iter() + .map(|impression| impression.omitted) + .collect::>() + }; + // An impression the implementation left without demand goes, and so does + // a request that has none left, so a demand source is never asked for + // nothing. + if omitted.iter().any(|&omitted| omitted) { + let mut omitted = omitted.into_iter(); + request.imp.retain(|_| !omitted.next().unwrap_or(false)); + } if request.imp.is_empty() { return Ok(OpenRtbBuildOutcome::NoImpressions); } + if request + .ext + .as_ref() + .is_some_and(|ext| ext.contains_key(TRUSTED_SERVER_EXT_KEY)) + { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} set ext.{TRUSTED_SERVER_EXT_KEY}, which only the driver writes", + provider.id + ), + })); + } finalize_request(&mut request, policy, finalization)?; Ok(OpenRtbBuildOutcome::Ready(request)) } -#[derive(Clone, Copy)] -enum ProfilePolicy<'a> { - Standard(&'a StandardProfilePlan), - Prebid(&'a PrebidProfilePlan), - Aps(&'a ApsProfilePlan), -} - -impl<'a> From<&'a CompiledOpenRtbProfile> for ProfilePolicy<'a> { - fn from(profile: &'a CompiledOpenRtbProfile) -> Self { - match profile { - CompiledOpenRtbProfile::Standard(plan) => Self::Standard(plan), - CompiledOpenRtbProfile::PrebidServer(plan) => Self::Prebid(plan), - CompiledOpenRtbProfile::Aps(plan) => Self::Aps(plan), - } - } -} - -impl ProfilePolicy<'_> { - fn augment_request( - self, - request: &mut OpenRtbRequest, - input: &ProviderAuctionInput, - routed: &RoutedAuction, - ) -> Result<(), Report> { - match self { - Self::Standard(plan) => apply_standard(request, plan), - Self::Prebid(plan) => apply_prebid(request, input, routed, plan), - Self::Aps(plan) => apply_aps(request, plan), - } - } - - fn keeps_pbs_identity_when_unsigned(self) -> bool { - matches!(self, Self::Prebid(_)) - } -} - +/// Builds every standard field of the request. The second value holds, for +/// each impression built, the index of the routed slot it came from. fn build_common_request( input: &ProviderAuctionInput, routed: &RoutedAuction, - policy: ProfilePolicy<'_>, + demand: &dyn CompiledDemand, + policy: DemandFieldPolicy, effective_timeout_ms: u32, -) -> OpenRtbRequest { +) -> (OpenRtbRequest, Vec) { let common = input.common_request(); - let imps = input + let (slot_indices, imps): (Vec, Vec) = input .slots() .iter() - .filter_map(|slot| build_imp(slot, policy)) - .collect(); - let site_domain = match policy { - ProfilePolicy::Aps(plan) => plan - .inventory_domain - .clone() - .unwrap_or_else(|| common.publisher.domain.clone()), - _ => common.publisher.domain.clone(), - }; - let page = match policy { - ProfilePolicy::Aps(plan) => { - aps_inventory_page(plan, common.publisher.page_url.as_deref(), &site_domain) - } - ProfilePolicy::Prebid(plan) => common.publisher.page_url.as_deref().map(|page| { - plan.debug_query_params.as_deref().map_or_else( - || page.to_string(), - |query| append_query_fragment(page, query), - ) - }), - ProfilePolicy::Standard(_) => common.publisher.page_url.clone(), - }; - let consent = common.user.consent.as_ref(); - let body_consent = match policy { - ProfilePolicy::Prebid(plan) => consent.filter(|value| { - plan.consent_forwarding.includes_body_consent() - || !matches!(value.source, ConsentSource::Cookie) - }), - _ => consent, - }; + .enumerate() + .filter_map(|(index, slot)| build_imp(slot, policy).map(|imp| (index, imp))) + .unzip(); + let site_domain = demand.site_domain(&common.publisher.domain); + let page = demand.site_page(common.publisher.page_url.as_deref(), &site_domain); + let body_consent = demand.body_consent(common.user.consent.as_ref()); let raw_tc = body_consent.and_then(|value| value.raw_tc_string.clone()); let user = Some(User { id: common.user.id.clone(), consent: raw_tc.clone(), ext: UserExt { consent: raw_tc, - consented_providers_settings: matches!(policy, ProfilePolicy::Prebid(_)) + consented_providers_settings: policy + .additional_consent .then(|| { body_consent .and_then(|value| value.raw_ac_string.clone()) @@ -374,7 +386,7 @@ fn build_common_request( .to_ext(), ..Default::default() }); - let language = normalized_language(routed.prebid_transport_headers(), policy); + let language = normalized_language(routed.transport_headers(), policy); let device = common .device .as_ref() @@ -385,14 +397,26 @@ fn build_common_request( country: Some(geo.country.clone()), region: geo.region.clone(), city: Some(geo.city.clone()), - lat: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.latitude), - lon: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.longitude), + lat: policy.precise_geo.then_some(geo.latitude), + lon: policy.precise_geo.then_some(geo.longitude), metro: (geo.metro_code > 0).then(|| geo.metro_code.to_string()), r#type: Some(2), ..Default::default() }), dnt: routed.dnt(), language: language.clone(), + // Attributes the selected device module resolved, each absent + // when nothing resolved it. + devicetype: device.attributes.as_ref().and_then(|a| a.device_type), + make: device.attributes.as_ref().and_then(|a| a.make.clone()), + model: device.attributes.as_ref().and_then(|a| a.model.clone()), + os: device.attributes.as_ref().and_then(|a| a.os.clone()), + osv: device + .attributes + .as_ref() + .and_then(|a| a.os_version.clone()), + w: device.attributes.as_ref().and_then(|a| a.screen_width), + h: device.attributes.as_ref().and_then(|a| a.screen_height), ..Default::default() }) .or_else(|| { @@ -403,14 +427,15 @@ fn build_common_request( }) }); - OpenRtbRequest { + let request = OpenRtbRequest { id: Some(common.id.clone()), imp: imps, site: Some(Site { domain: Some(site_domain.clone()), page, - r#ref: matches!(policy, ProfilePolicy::Prebid(_)) - .then(|| header_string(routed.prebid_transport_headers().referer())) + r#ref: policy + .site_ref + .then(|| header_string(routed.transport_headers().referer())) .flatten(), publisher: Some(Publisher { domain: Some(site_domain), @@ -420,11 +445,8 @@ fn build_common_request( }), user, device, - regs: build_regs(body_consent, policy), - test: match policy { - ProfilePolicy::Prebid(plan) => plan.test_mode.then_some(true), - _ => None, - }, + regs: build_regs(body_consent, policy.regs), + test: policy.test.then_some(true), tmax: to_openrtb_i32( effective_timeout_ms, "tmax", @@ -432,10 +454,11 @@ fn build_common_request( ), cur: vec![DEFAULT_CURRENCY.to_string()], ..Default::default() - } + }; + (request, slot_indices) } -fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option { +fn build_imp(slot: &ProviderSlotInput, policy: DemandFieldPolicy) -> Option { let formats = slot .slot() .formats @@ -451,17 +474,17 @@ fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option .collect::>(); let first_width = formats.first()?.w; let first_height = formats.first()?.h; - let aps_banner = matches!(policy, ProfilePolicy::Aps(_)); + let primary_size = policy.primary_banner_size; Some(Imp { id: Some(slot.slot().id.clone()), banner: Some(Banner { format: formats, - w: aps_banner.then_some(first_width).flatten(), - h: aps_banner.then_some(first_height).flatten(), - topframe: aps_banner.then_some(false), + w: primary_size.then_some(first_width).flatten(), + h: primary_size.then_some(first_height).flatten(), + topframe: primary_size.then_some(false), ..Default::default() }), - tagid: matches!(policy, ProfilePolicy::Prebid(_)).then(|| slot.slot().id.clone()), + tagid: policy.imp_tagid.then(|| slot.slot().id.clone()), bidfloor: slot.slot().floor_price, bidfloorcur: slot .slot() @@ -472,99 +495,9 @@ fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option }) } -fn apply_standard( - request: &mut OpenRtbRequest, - plan: &StandardProfilePlan, -) -> Result<(), Report> { - request.ext = nonempty_map(plan.request_ext.as_object().clone()); - for imp in &mut request.imp { - imp.ext = nonempty_map(plan.imp_ext.as_object().clone()); - } - Ok(()) -} - -fn apply_prebid( - request: &mut OpenRtbRequest, - input: &ProviderAuctionInput, - _routed: &RoutedAuction, - plan: &PrebidProfilePlan, -) -> Result<(), Report> { - // Routing admits only slots with positive banner dimensions that fit in i32, - // so build_imp produces one impression per routed slot, in the same order. - debug_assert_eq!( - request.imp.len(), - input.slots().len(), - "should keep one impression per routed slot" - ); - // Filter paired impressions and slots together so later demand keeps its slot ID. - request.imp = std::mem::take(&mut request.imp) - .into_iter() - .zip(input.slots()) - .filter_map(|(mut imp, slot)| { - let bidder = slot - .bidder_params() - .iter() - .filter_map(|(bidder, params)| { - let mut params = params.clone(); - plan.override_engine.apply_routed( - bidder.as_str(), - slot.prebid_zone(), - &mut params, - ); - params - .as_object() - .is_some_and(|params| !params.is_empty()) - .then(|| (bidder.as_str().to_string(), params)) - }) - .collect::>(); - let mut prebid = Map::new(); - if !bidder.is_empty() { - prebid.insert("bidder".to_string(), Value::Object(bidder)); - } else if slot.allows_stored_fallback() { - prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); - } - if prebid.is_empty() { - return None; - } - imp.ext = Some(Map::from_iter([( - "prebid".to_string(), - Value::Object(prebid), - )])); - Some(imp) - }) - .collect(); - let mut prebid_request = Map::new(); - if plan.debug { - prebid_request.insert("debug".to_string(), Value::Bool(true)); - prebid_request.insert("returnallbidstatus".to_string(), Value::Bool(true)); - } - request.ext = Some(Map::from_iter([( - "prebid".to_string(), - Value::Object(prebid_request), - )])); - Ok(()) -} - -fn apply_aps( - request: &mut OpenRtbRequest, - plan: &ApsProfilePlan, -) -> Result<(), Report> { - request.ext = Some(Map::from_iter([ - ( - "account".to_string(), - Value::String(plan.account_id.clone()), - ), - ( - "sdk".to_string(), - json!({"source": APS_SDK_SOURCE, "version": APS_SDK_VERSION}), - ), - ])); - Ok(()) -} - fn finalize_request( request: &mut OpenRtbRequest, - policy: ProfilePolicy<'_>, + policy: DemandFieldPolicy, finalization: &RequestFinalization<'_>, ) -> Result<(), Report> { let request_id = request.id.as_deref().ok_or_else(|| { @@ -587,7 +520,7 @@ fn finalize_request( request_scheme: Some(finalization.signing_params.request_scheme.clone()), ts: Some(finalization.signing_params.timestamp), }) - } else if policy.keeps_pbs_identity_when_unsigned() { + } else if policy.unsigned_request_identity { Some(TrustedServerExt { version: None, signature: None, @@ -606,19 +539,19 @@ fn finalize_request( message: format!("Failed to serialize Trusted Server extension: {error}"), }) })?; - ext.insert("trusted_server".to_string(), serialized); + ext.insert(TRUSTED_SERVER_EXT_KEY.to_string(), serialized); } Ok(()) } fn build_regs( consent: Option<&crate::consent::ConsentContext>, - policy: ProfilePolicy<'_>, + policy: RegsPolicy, ) -> Option { let consent = consent?; - if matches!(policy, ProfilePolicy::Aps(_)) { - // Preserve APS exactly: any admitted context produces regs and GDPR is - // derived only from the applicability bit, without jurisdiction rules. + if policy == RegsPolicy::ApplicabilityBit { + // Any admitted context produces regs and GDPR comes from the + // applicability bit alone, without jurisdiction rules. let ext = RegsExt { gdpr: Some(u8::from(consent.gdpr_applies)), us_privacy: consent.raw_us_privacy.clone(), @@ -639,8 +572,8 @@ fn build_regs( }); } - // Standard deliberately shares PBS's conservative consent baseline. Keep - // the legacy PBS empty-context and jurisdiction behavior byte-for-byte. + // The jurisdiction policy sends no regs for an empty context and reads + // GDPR from the applicability bit or from a GDPR jurisdiction. let has_data = consent.gdpr_applies || consent.raw_us_privacy.is_some() || consent.raw_gpp_string.is_some() @@ -684,21 +617,16 @@ fn build_regs( }) } -fn normalized_language( - headers: &PrebidTransportHeaders, - policy: ProfilePolicy<'_>, -) -> Option { +fn normalized_language(headers: &TransportHeaders, policy: DemandFieldPolicy) -> Option { let value = header_string(headers.accept_language()) .and_then(|value| value.split(',').next().map(str::to_string)) .and_then(|value| value.split(';').next().map(str::to_string)) .and_then(|value| value.split('-').next().map(str::to_string)) .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty())?; - match policy { - ProfilePolicy::Prebid(_) => Some(value), - ProfilePolicy::Aps(_) | ProfilePolicy::Standard(_) => { - (value.len() <= MAX_CONSERVATIVE_LANGUAGE_BYTES).then_some(value) - } + match policy.language_max_bytes { + None => Some(value), + Some(max_bytes) => (value.len() <= max_bytes).then_some(value), } } @@ -708,54 +636,8 @@ fn header_string(value: Option<&http::HeaderValue>) -> Option { .map(str::to_string) } -fn aps_inventory_page( - plan: &ApsProfilePlan, - publisher_page: Option<&str>, - domain: &str, -) -> Option { - let fallback = publisher_page - .and_then(valid_aps_page_url) - .unwrap_or_else(|| format!("https://{domain}")); - let Some(origin) = plan.inventory_page_origin.as_deref() else { - return Some(fallback); - }; - let (Ok(mut canonical), Ok(current)) = (Url::parse(origin), Url::parse(&fallback)) else { - return Some(fallback); - }; - canonical.set_path(current.path()); - canonical.set_query(current.query()); - canonical.set_fragment(None); - Some(canonical.to_string()) -} - -fn append_query_fragment(url: &str, query: &str) -> String { - if query.is_empty() || url.contains(query) { - return url.to_string(); - } - let separator = if url.contains('?') { '&' } else { '?' }; - format!("{url}{separator}{query}") -} - -fn valid_aps_page_url(value: &str) -> Option { - const MAX_APS_PAGE_URL_BYTES: usize = 8192; - - if value.len() > MAX_APS_PAGE_URL_BYTES { - return None; - } - let parsed = Url::parse(value).ok()?; - (matches!(parsed.scheme(), "http" | "https") - && parsed.host_str().is_some() - && parsed.username().is_empty() - && parsed.password().is_none()) - .then(|| parsed.to_string()) -} - -fn nonempty_map(value: Map) -> Option> { - (!value.is_empty()).then_some(value) -} - /// Suppress notification URLs using exact returned-seat identity. -pub(crate) fn apply_notification_policy(bids: &mut [Bid], policy: &NotificationPolicy) { +pub fn apply_notification_policy(bids: &mut [Bid], policy: &NotificationPolicy) { for bid in bids { let suppress = policy.suppress_all || bid @@ -897,21 +779,24 @@ fn extract_standard_bid( }) } -/// Count bidder parameter objects a profile did not consume. +/// Count bidder parameter objects a demand source did not consume. #[must_use] pub(crate) fn unused_bidder_params_count( - profile: &CompiledOpenRtbProfile, + demand: &dyn CompiledDemand, input: &ProviderAuctionInput, ) -> u32 { - if profile.is_prebid_server() { + if demand.field_policy().consumes_bidder_params { return 0; } ignored_bidder_params_count(input) } -/// Count routed bidder params for a profile known to ignore them. +/// Count routed bidder params for a demand source known to ignore them. +/// +/// Public so an implementation that takes no bidder parameters reports the +/// ones it was routed the way core's own reader does. #[must_use] -pub(crate) fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { +pub fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { saturating_bidder_param_counts(input.slots().iter().map(|slot| slot.bidder_params().len())) } @@ -921,20 +806,76 @@ fn saturating_bidder_param_counts(counts: impl IntoIterator) -> u3 }) } +/// The most a demand source's response body may hold. +const MAX_RESPONSE_BYTES: usize = 1024 * 1024; + +/// Reads an ordinary `OpenRTB` response. A `204` is a no-bid, any other +/// non-success status or unreadable JSON is an error response, and a body over +/// the size limit is an `Err`. +/// +/// Public so a demand implementation in a crate of its own, whose exchange +/// answers the ordinary way, has its response read as core reads one. +/// +/// # Errors +/// +/// Returns an error when the response body cannot be read. +pub async fn parse_openrtb_response( + context: DemandResponse<'_>, + response: PlatformResponse, +) -> Result> { + let provider_id = context.provider_id; + let response_time_ms = context.response_time_ms; + let response = response.response; + let status = response.status(); + if status == StatusCode::NO_CONTENT { + return Ok(AuctionResponse::no_bid(provider_id, response_time_ms)); + } + if !status.is_success() { + if status.is_redirection() { + log::warn!( + "Provider '{provider_id}' returned a redirect; generic OpenRTB redirects are refused" + ); + } + return Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16()))); + } + + let body = response + .into_body() + .into_bytes_bounded(MAX_RESPONSE_BYTES) + .await + .change_context(TrustedServerError::Auction { + message: format!("Provider {provider_id} response body failed"), + })?; + let value: Value = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(error) => { + log::warn!("Provider '{provider_id}' response JSON was invalid: {error}"); + return Ok(AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response"))); + } + }; + + Ok(extract_standard_response( + provider_id, + context.input, + &value, + response_time_ms, + )) +} + #[cfg(test)] mod routing_metadata_tests { use std::collections::BTreeMap; use std::str::FromStr as _; - use serde_json::json; - use super::{saturating_bidder_param_counts, unused_bidder_params_count}; - use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, - ProviderConfig, ProviderId, RoutingMode, - }; + use crate::auction::plan::{AuctionPlan, BidderId, BidderRouteConfig, ProviderId}; use crate::auction::routing::route_auction; - use crate::auction::test_support::canonical_parity_auction_request; + use crate::auction::test_support::{ + canonical_parity_auction_request, demand_table, plan_config, + }; #[test] fn unused_bidder_param_count_saturates_across_slots_and_large_values() { @@ -947,49 +888,32 @@ mod routing_metadata_tests { } #[test] - fn unused_bidder_param_count_is_profile_aware() { - for (profile, profile_config, expected) in [ - ("prebid-server", json!({}), 0), - ("standard", json!({}), 1), - ("aps", json!({"account_id":"example-account"}), 1), - ] { + fn unused_bidder_param_count_follows_the_implementation() { + for (implementation, expected) in [("auction.fixture", 0), ("auction.plain-fixture", 1)] { + let endpoint = "https://provider.example/openrtb"; let provider_id = - ProviderId::from_str("fictional-provider").expect("should parse provider ID"); - let plan = AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 1_000, - providers: BTreeMap::from([( - provider_id.clone(), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: if profile == "aps" { - "https://aps.example/e/pb/bid".to_string() - } else { - "https://provider.example/openrtb".to_string() - }, - timeout_ms: None, - routing: RoutingMode::Explicit, - notifications: NotificationConfig::default(), - profile_config, - }, - )]), - bidders: BTreeMap::from([( - BidderId::from_str("exampleBidder").expect("should parse bidder ID"), - BidderRouteConfig { - provider: provider_id, - }, - )]), - mediator: None, - request_signing: None, - }) - .expect("should compile profile plan"); + ProviderId::from_str("fictional_provider").expect("should parse provider ID"); + let mut config = plan_config(vec![( + "fictional_provider", + demand_table(implementation, endpoint), + )]); + config.bidders = BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + module: provider_id, + }, + )]); + let plan = AuctionPlan::compile(config).expect("should compile the plan"); let inbound = http::Request::new(edgezero_core::body::Body::empty()); let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); assert_eq!( - unused_bidder_params_count(&plan.providers()[0].profile, &routed.inputs()[0]), + unused_bidder_params_count( + plan.providers()[0].demand.as_ref(), + &routed.inputs()[0] + ), expected, - "{profile} should report only bidder params it ignores" + "{implementation} should report only bidder params it ignores" ); } } diff --git a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs index d322102e62..2f2fa20f84 100644 --- a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs +++ b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs @@ -66,7 +66,7 @@ pub(super) async fn execute_standard_fixture( return Ok( AuctionResponse::no_bid(provider.id.as_str(), 0).with_metadata( "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), ), ); } @@ -76,7 +76,7 @@ pub(super) async fn execute_standard_fixture( .with_metadata("http_status", json!(status.as_u16())) .with_metadata( "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), )); } let body = response @@ -93,7 +93,7 @@ pub(super) async fn execute_standard_fixture( .with_metadata("error_type", json!("parse_response")) .with_metadata( "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), )); } }; @@ -101,7 +101,7 @@ pub(super) async fn execute_standard_fixture( apply_notification_policy(&mut parsed.bids, &provider.notifications); parsed.metadata.insert( "routing".to_string(), - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(provider.demand.as_ref(), input)}), ); Ok(parsed) } diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs index f47efa475e..af47ffad9c 100644 --- a/crates/trusted-server-core/src/auction/openrtb/tests.rs +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -9,69 +9,62 @@ use serde_json::{Value, json}; use super::test_executor::execute_standard_fixture; use super::*; +use crate::auction::demand::DemandResponse; use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, - ProviderConfig, ProviderId, RoutingMode, + AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, ProviderId, }; use crate::auction::provider::{GenericOpenRtbProvider, ProviderRequestOutcome}; use crate::auction::routing::route_auction; -use crate::auction::test_support::canonical_parity_auction_request; +use crate::auction::test_support::{canonical_parity_auction_request, demand_table, plan_config}; use crate::auction::types::{AdFormat, AdSlot, BidStatus, MediaType}; +use crate::consent::ConsentContext; use crate::consent::jurisdiction::Jurisdiction; -use crate::consent::{ConsentContext, ConsentSource}; -use crate::platform::PlatformHttpClient; use crate::platform::test_support::{ HashMapConfigStore, HashMapSecretStore, NoopHttpClient, StubBackend, StubHttpClient, build_services_with_backend_and_http_client, build_services_with_config_secret_and_http_client, }; +use crate::platform::{PlatformHttpClient, PlatformResponse}; use crate::request_signing::RequestSigner; -fn config(profile: &str, profile_config: Value) -> AuctionPlanConfig { +fn config(implementation: &str, settings: Value) -> AuctionPlanConfig { config_with_endpoint( - profile, - profile_config, + implementation, + settings, "https://exchange.example.test/openrtb", ) } -fn config_with_endpoint(profile: &str, profile_config: Value, endpoint: &str) -> AuctionPlanConfig { - let provider_id = ProviderId::from_str("fictional-provider").expect("should parse provider ID"); - let prebid_server = profile == "prebid-server"; - AuctionPlanConfig { - timeout_ms: 321, - providers: BTreeMap::from([( - provider_id.clone(), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: endpoint.to_string(), - timeout_ms: Some(321), - routing: if prebid_server { - RoutingMode::Explicit - } else { - RoutingMode::AllEligible - }, - notifications: NotificationConfig::default(), - profile_config, +fn config_with_endpoint( + implementation: &str, + settings: Value, + endpoint: &str, +) -> AuctionPlanConfig { + let provider_id = ProviderId::from_str("fictional_provider").expect("should parse provider ID"); + // The stand-in forbids all-eligible routing, so a bidder is routed to it. + let routed_by_bidder = implementation == "auction.fixture"; + let mut table = demand_table(implementation, endpoint); + table.insert("timeout_ms".to_string(), json!(321)); + if !routed_by_bidder { + table.insert("routing".to_string(), json!("all_eligible")); + } + if let Value::Object(settings) = settings { + table.extend(settings); + } + let mut config = plan_config(vec![("fictional_provider", table)]); + config.timeout_ms = 321; + if routed_by_bidder { + config.bidders = BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + module: provider_id, }, - )]), - bidders: if prebid_server { - BTreeMap::from([( - BidderId::from_str("exampleBidder").expect("should parse bidder ID"), - BidderRouteConfig { - provider: provider_id, - }, - )]) - } else { - BTreeMap::new() - }, - mediator: None, - request_signing: None, + )]); } + config } -fn routed(profile: &str, profile_config: Value) -> (AuctionPlan, RoutedAuction) { - let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); +fn routed(implementation: &str, settings: Value) -> (AuctionPlan, RoutedAuction) { + let plan = AuctionPlan::compile(config(implementation, settings)).expect("should compile plan"); let inbound = Request::builder() .uri("https://publisher.example/auction") .header( @@ -87,12 +80,12 @@ fn routed(profile: &str, profile_config: Value) -> (AuctionPlan, RoutedAuction) } fn routed_with_request( - profile: &str, - profile_config: Value, + implementation: &str, + settings: Value, request: crate::auction::types::AuctionRequest, accept_language: Option<&str>, ) -> (AuctionPlan, RoutedAuction) { - let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); + let plan = AuctionPlan::compile(config(implementation, settings)).expect("should compile plan"); let mut builder = Request::builder().uri("https://publisher.example/auction"); if let Some(language) = accept_language { builder = builder.header(header::ACCEPT_LANGUAGE, language); @@ -105,12 +98,12 @@ fn routed_with_request( } fn build_with_request( - profile: &str, - profile_config: Value, + implementation: &str, + settings: Value, request: crate::auction::types::AuctionRequest, accept_language: Option<&str>, ) -> OpenRtbRequest { - let (plan, routed) = routed_with_request(profile, profile_config, request, accept_language); + let (plan, routed) = routed_with_request(implementation, settings, request, accept_language); match build_request( &routed.inputs()[0], &routed, @@ -137,8 +130,8 @@ fn finalization<'a>(signer: Option<&'a RequestSigner>) -> RequestFinalization<'a } } -fn build(profile: &str, profile_config: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { - let (plan, routed) = routed(profile, profile_config); +fn build(implementation: &str, settings: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { + let (plan, routed) = routed(implementation, settings); match build_request( &routed.inputs()[0], &routed, @@ -172,7 +165,7 @@ fn deterministic_signer() -> RequestSigner { } #[test] -fn consent_matrix_preserves_pbs_standard_and_aps_policies() { +fn consent_matrix_holds_under_each_implementation() { let cases = [ ("empty", ConsentContext::default()), ( @@ -212,26 +205,19 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { ), ]; for (name, consent) in cases { - for profile in ["standard", "prebid-server", "aps"] { + for implementation in ["auction.plain-fixture", "auction.fixture"] { let mut canonical = canonical_parity_auction_request(); canonical.user.consent = Some(consent.clone()); - let config = if profile == "aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; - let value = serde_json::to_value(build_with_request(profile, config, canonical, None)) - .expect("should serialize request"); + let value = serde_json::to_value(build_with_request( + implementation, + json!({}), + canonical, + None, + )) + .expect("should serialize request"); let regs = value.get("regs"); - if profile == "aps" { - let regs = regs.expect("APS should preserve empty admitted context"); - assert_eq!( - regs["gdpr"], - json!(u8::from(consent.gdpr_applies)), - "{name}" - ); - } else if name == "empty" { - assert!(regs.is_none(), "{profile} should omit empty regs"); + if name == "empty" { + assert!(regs.is_none(), "{implementation} should omit empty regs"); } else { let regs = regs.expect("should emit actionable regs"); let expected_gdpr = match consent.jurisdiction { @@ -242,7 +228,7 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { assert_eq!( regs.get("gdpr"), expected_gdpr.map(|value| json!(u8::from(value))).as_ref(), - "{name} {profile}" + "{name} {implementation}" ); } let serialized = value.to_string(); @@ -263,53 +249,16 @@ fn consent_matrix_preserves_pbs_standard_and_aps_policies() { } } -#[test] -fn pbs_body_consent_respects_source_and_forwarding_mode() { - for (mode, source, expected) in [ - ("cookies_only", ConsentSource::Cookie, false), - ("cookies_only", ConsentSource::PolicyDefault, true), - ("openrtb_only", ConsentSource::Cookie, true), - ("both", ConsentSource::Cookie, true), - ] { - let mut canonical = canonical_parity_auction_request(); - canonical - .user - .consent - .as_mut() - .expect("should have consent context") - .source = source; - let value = serde_json::to_value(build_with_request( - "prebid-server", - json!({"consent_forwarding": mode}), - canonical, - None, - )) - .expect("should serialize request"); - assert_eq!( - value["user"].get("consent").is_some(), - expected, - "{mode:?} {source:?}" - ); - assert_eq!(value.get("regs").is_some(), expected, "{mode:?} {source:?}"); - } -} - #[test] fn language_limits_are_profile_specific() { let language = "abcdefghijk"; - for (profile, expected) in [ - ("prebid-server", Some(language)), - ("aps", None), - ("standard", None), + for (implementation, expected) in [ + ("auction.fixture", Some(language)), + ("auction.plain-fixture", None), ] { - let config = if profile == "aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; let request = build_with_request( - profile, - config, + implementation, + json!({}), canonical_parity_auction_request(), Some(language), ); @@ -318,15 +267,10 @@ fn language_limits_are_profile_specific() { expected ); } - for profile in ["prebid-server", "aps", "standard"] { - let config = if profile == "aps" { - json!({"account_id": "example-account-id"}) - } else { - json!({}) - }; + for implementation in ["auction.fixture", "auction.plain-fixture"] { let request = build_with_request( - profile, - config, + implementation, + json!({}), canonical_parity_auction_request(), Some("en-US,en;q=0.9"), ); @@ -338,272 +282,9 @@ fn language_limits_are_profile_specific() { } #[test] -fn pbs_debug_query_fragment_preserves_exact_legacy_configured_semantics() { - for (page, fragment, expected) in [ - ( - "https://publisher.example/article", - "pbjs_debug=true", - "https://publisher.example/article?pbjs_debug=true", - ), - ( - "https://publisher.example/article?existing=1", - "pbjs_debug=true", - "https://publisher.example/article?existing=1&pbjs_debug=true", - ), - ( - "https://publisher.example/article", - "?pbjs_debug=true", - "https://publisher.example/article??pbjs_debug=true", - ), - ( - "https://publisher.example/article?pbjs_debug=true", - "pbjs_debug=true", - "https://publisher.example/article?pbjs_debug=true", - ), - ( - "https://publisher.example/article", - "", - "https://publisher.example/article", - ), - ] { - let mut request = canonical_parity_auction_request(); - request.publisher.page_url = Some(page.to_string()); - let built = build_with_request( - "prebid-server", - json!({"debug_query_params": fragment}), - request, - None, - ); - assert_eq!( - built.site.and_then(|site| site.page).as_deref(), - Some(expected), - "should preserve exact legacy query fragment semantics" - ); - } -} - -#[test] -fn pbs_routed_overrides_are_ordered_and_stored_request_is_trusted_fallback() { - let mut raw = config( - "prebid-server", - json!({ - "debug": true, - "test_mode": true, - "bid_param_overrides": {"exampleBidder": {"generic": 1, "shared": "generic"}}, - "bid_param_zone_overrides": {"exampleBidder": {"zone-a": {"zone": 2, "shared": "zone"}}}, - "bid_param_override_rules": [ - {"when":{"bidder":"exampleBidder"},"set":{"ordered":1,"shared":"rule-one"}}, - {"when":{"bidder":"exampleBidder","zone":"zone-a"},"set":{"ordered":2,"shared":"rule-two"}} - ] - }), - ); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS override plan"); - let mut request = canonical_parity_auction_request(); - request.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"zone":"zone-a","bidderParams":{"exampleBidder":{"original":true,"shared":"original"}}}), - )]); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(request, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), - }; - let value = serde_json::to_value(built).expect("should serialize request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"generic":1,"ordered":2,"original":true,"shared":"rule-two","zone":2}) - ); - assert_eq!(value["ext"]["prebid"]["debug"], true); - assert_eq!(value["ext"]["prebid"]["returnallbidstatus"], true); - assert_eq!(value["test"], 1); - - let mut empty_overridden = canonical_parity_auction_request(); - empty_overridden.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"zone":"zone-a","bidderParams":{"exampleBidder":{}}}), - )]); - let routed = route_auction(empty_overridden, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request after populating empty params") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain overridden impression"), - }; - let value = serde_json::to_value(built).expect("should serialize overridden request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"generic":1,"ordered":2,"shared":"rule-two","zone":2}), - "should allow profile overrides to populate empty browser params" - ); - - let mut stored = canonical_parity_auction_request(); - stored.slots[0].bidders.clear(); - let routed = route_auction(stored, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build stored request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), - }; - let value = serde_json::to_value(built).expect("should serialize stored request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], - "fictional-slot" - ); -} - -#[test] -fn pbs_pairs_each_impression_with_its_routed_slot_params() { - let mut raw = config("prebid-server", json!({})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); - let mut request = canonical_parity_auction_request(); - request.slots[0].id = "first-slot".to_string(); - request.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"bidderParams":{"exampleBidder":{"placement":"first"}}}), - )]); - let mut second_slot = request.slots[0].clone(); - second_slot.id = "second-slot".to_string(); - second_slot.bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"bidderParams":{"exampleBidder":{"placement":"second"}}}), - )]); - request.slots.push(second_slot); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(request, &inbound, &plan, None); - - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impressions"), - }; - let value = serde_json::to_value(built).expect("should serialize request"); - - assert_eq!(value["imp"][0]["id"], "first-slot"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"placement":"first"}) - ); - assert_eq!(value["imp"][1]["id"], "second-slot"); - assert_eq!( - value["imp"][1]["ext"]["prebid"]["bidder"]["exampleBidder"], - json!({"placement":"second"}) - ); -} - -#[test] -fn pbs_stored_intent_is_applied_after_overrides_with_inline_first() { - for intent in [None, Some(false), Some(true)] { - for inline in [false, true] { - for fill_override in [false, true] { - let profile = if fill_override { - json!({"bid_param_overrides":{"exampleBidder":{"filled":1}}}) - } else { - json!({}) - }; - let plan = AuctionPlan::compile(config("prebid-server", profile)) - .expect("should compile plan"); - let mut request = canonical_parity_auction_request(); - let mut envelope = json!({"bidderParams":{"exampleBidder":if inline { json!({"original":1}) } else { json!({}) }}}); - if let Some(intent) = intent { - envelope["storedRequest"] = json!(intent); - } - request.slots[0].bidders = HashMap::from([("trustedServer".to_string(), envelope)]); - let inbound = Request::new(EdgeBody::empty()); - let routed = route_auction(request, &inbound, &plan, None); - let result = build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build request"); - if !inline && !fill_override && intent == Some(false) { - assert!(matches!(result, OpenRtbBuildOutcome::NoImpressions)); - continue; - } - let OpenRtbBuildOutcome::Ready(request) = result else { - panic!("should retain demand") - }; - let wire = serde_json::to_value(request).expect("should serialize request"); - let prebid = &wire["imp"][0]["ext"]["prebid"]; - if inline || fill_override { - assert!(prebid.get("storedrequest").is_none()); - assert_eq!( - prebid["bidder"]["exampleBidder"].get("original"), - inline.then_some(&json!(1)) - ); - assert_eq!( - prebid["bidder"]["exampleBidder"].get("filled"), - fill_override.then_some(&json!(1)) - ); - } else { - assert_eq!(prebid["storedrequest"]["id"], "fictional-slot"); - } - } - } - } -} - -#[test] -fn pbs_filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { +fn filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { let plan = - AuctionPlan::compile(config("prebid-server", json!({}))).expect("should compile plan"); + AuctionPlan::compile(config("auction.fixture", json!({}))).expect("should compile plan"); let mut request = canonical_parity_auction_request(); let template = request.slots[0].clone(); request.slots = [ @@ -667,30 +348,23 @@ fn pbs_filtering_keeps_slot_pairs_and_drops_demandless_trusted_routes() { ); assert_eq!(wire["imp"][0]["id"], "keep-inline"); assert_eq!( - wire["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + wire["imp"][0]["ext"]["fixture"]["bidder"]["exampleBidder"], json!({"id":2}) ); assert_eq!(wire["imp"][1]["id"], "keep-stored"); - assert_eq!( - wire["imp"][1]["ext"]["prebid"]["storedrequest"]["id"], - "keep-stored" - ); + assert_eq!(wire["imp"][1]["ext"]["fixture"]["stored"], "keep-stored"); } #[test] -fn pbs_disabled_empty_candidate_does_not_become_stored_demand() { - let mut raw = config("prebid-server", json!({})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; +fn a_disabled_empty_candidate_does_not_become_stored_demand() { + let mut raw = config("auction.fixture", json!({"routing": "explicit"})); raw.bidders.insert( crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + module: ProviderId::from_str("fictional_provider").expect("should parse provider"), }, ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let plan = AuctionPlan::compile(raw).expect("should compile the plan"); let mut request = canonical_parity_auction_request(); request.slots[0].bidders = HashMap::from([( "trustedServer".to_string(), @@ -719,183 +393,33 @@ fn pbs_disabled_empty_candidate_does_not_become_stored_demand() { )); } -#[test] -fn pbs_empty_params_without_matching_override_fall_back_to_stored_request() { - let mut raw = config("prebid-server", json!({})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); - let mut request = canonical_parity_auction_request(); - request.slots[0].bidders = HashMap::from([( - "trustedServer".to_string(), - json!({"bidderParams":{"exampleBidder":{}}}), - )]); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(request, &inbound, &plan, None); - let built = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build stored request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain stored impression"), - }; - let value = serde_json::to_value(built).expect("should serialize stored request"); - assert_eq!( - value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], - "fictional-slot" - ); - assert!(value["imp"][0]["ext"]["prebid"].get("bidder").is_none()); -} - -#[test] -fn pbs_driver_exact_golden_preserves_profile_policy() { - let mut raw = config("prebid-server", json!({"consent_forwarding": "both"})); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; - raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), - }, - ); - let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); - let mut common = canonical_parity_auction_request(); - common.slots[0].bidders = HashMap::from([( - "exampleBidder".to_string(), - json!({"placement": "fictional-placement"}), - )]); - let inbound = Request::builder() - .uri("https://publisher.example/auction") - .header( - header::REFERER, - "https://referrer.example/story?fictional=1", - ) - .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") - .header("dnt", "1") - .body(EdgeBody::empty()) - .expect("should build inbound request"); - let routed = route_auction(common, &inbound, &plan, None); - let request = match build_request( - &routed.inputs()[0], - &routed, - &plan.providers()[0], - 321, - &finalization(None), - ) - .expect("should build PBS request") - { - OpenRtbBuildOutcome::Ready(request) => request, - OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), - }; - assert_eq!( - serde_json::to_string(&request).expect("should serialize PBS driver request"), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"request_host":"publisher.example","request_scheme":"https"}}}"#, - "should preserve PBS parity differences" - ); -} - -#[test] -fn aps_inventory_identity_and_page_fallback_preserve_legacy_policy() { - let mut request = canonical_parity_auction_request(); - request.publisher.domain = "deployment.example".to_string(); - request.publisher.page_url = - Some("https://deployment.example/news/story?edition=fictional#section".to_string()); - let built = build_with_request( - "aps", - json!({ - "account_id": "example-account-id", - "inventory_domain": "publisher.example", - "inventory_page_origin": "https://www.publisher.example" - }), - request, - None, - ); - let site = built.site.expect("should include APS site"); - assert_eq!(site.domain.as_deref(), Some("publisher.example")); - assert_eq!( - site.page.as_deref(), - Some("https://www.publisher.example/news/story?edition=fictional") - ); - assert_eq!( - site.publisher - .and_then(|publisher| publisher.domain) - .as_deref(), - Some("publisher.example") - ); - - for unsafe_page in [ - "https://user:password@publisher.example/private", - "data:text/html,fictional", - ] { - let mut request = canonical_parity_auction_request(); - request.publisher.page_url = Some(unsafe_page.to_string()); - let built = build_with_request( - "aps", - json!({"account_id":"example-account-id"}), - request, - None, - ); - assert_eq!( - built.site.and_then(|site| site.page).as_deref(), - Some("https://publisher.example"), - "unsafe page should fall back to publisher domain" - ); - } -} - -#[test] -fn aps_driver_exact_golden_preserves_profile_policy() { - let request = build("aps", json!({"account_id": "example-account-id"}), None); - assert_eq!( - serde_json::to_string(&request).expect("should serialize APS driver request"), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, - "should preserve APS parity differences" - ); -} - #[test] fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { let signer = deterministic_signer(); - for (profile, config) in [ - ("standard", json!({"request_ext": {"fictional": true}})), - ("prebid-server", json!({})), - ("aps", json!({"account_id": "example-account-id"})), + for (implementation, config) in [ + ( + "auction.plain-fixture", + json!({"request_ext": {"fictional": true}}), + ), + ("auction.fixture", json!({})), ] { - let unsigned = serde_json::to_value(build(profile, config.clone(), None)) + let unsigned = serde_json::to_value(build(implementation, config.clone(), None)) .expect("should serialize unsigned request"); let unsigned_ts = unsigned["ext"].get("trusted_server"); - if profile == "prebid-server" { + if implementation == "auction.fixture" { assert_eq!( unsigned_ts, Some(&json!({"request_host": "publisher.example", "request_scheme": "https"})), - "should retain only PBS host and scheme when unsigned" + "should retain only the host and scheme when unsigned" ); } else { assert!( unsigned_ts.is_none(), - "should omit unsigned non-PBS extension" + "should omit the extension when unsigned" ); } - let signed = serde_json::to_value(build(profile, config, Some(&signer))) + let signed = serde_json::to_value(build(implementation, config, Some(&signer))) .expect("should serialize signed request"); let extension = &signed["ext"]["trusted_server"]; assert_eq!(extension["version"], "1.1", "should set signing version"); @@ -921,35 +445,23 @@ fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { #[test] fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { let signer = deterministic_signer(); - let cases = [ - ( - "standard", - json!({"request_ext": {"fictional": true}}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), - ( - "prebid-server", - json!({}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), - ( - "aps", - json!({"account_id": "example-account-id"}), - r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, - ), - ]; - for (profile, config, expected) in cases { + let cases = [( + "auction.plain-fixture", + json!({"request_ext": {"fictional": true}}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + )]; + for (implementation, config, expected) in cases { assert_eq!( - serde_json::to_string(&build(profile, config, Some(&signer))) + serde_json::to_string(&build(implementation, config, Some(&signer))) .expect("should serialize signed request"), expected, - "{profile} signed wire fixture should stay exact" + "{implementation} signed wire fixture should stay exact" ); } assert_eq!( serde_json::to_string(&build( - "standard", + "auction.plain-fixture", json!({"request_ext": {"fictional": true}}), None, )) @@ -959,28 +471,9 @@ fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { ); } -#[test] -fn standard_static_extensions_have_no_invented_bidder_param_location() { - let request = build( - "standard", - json!({ - "request_ext": {"fictional_request": {"enabled": true}}, - "imp_ext": {"fictional_imp": "value"} - }), - None, - ); - let value = serde_json::to_value(request).expect("should serialize request"); - assert_eq!(value["ext"]["fictional_request"]["enabled"], true); - assert_eq!(value["imp"][0]["ext"]["fictional_imp"], "value"); - assert!( - !value.to_string().contains("exampleBidder"), - "standard profile must not invent bidder params placement" - ); -} - #[test] fn defensive_no_impression_outcome_does_not_build_transportable_request() { - let (plan, mut routed) = routed("standard", json!({})); + let (plan, mut routed) = routed("auction.plain-fixture", json!({})); let mut common = routed.inputs()[0].common_request().clone(); common.slots = vec![AdSlot { id: "video-only".to_string(), @@ -1008,17 +501,13 @@ fn standard_fixture_with_formats( formats: Vec, ) -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { let mut raw = config( - "standard", + "auction.plain-fixture", json!({"request_ext": {"fixture": true}, "imp_ext": {"slot_fixture": true}}), ); - raw.providers - .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) - .expect("should find provider") - .routing = RoutingMode::Explicit; raw.bidders.insert( - crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderId::from_str("exampleBidder").expect("should parse bidder"), BidderRouteConfig { - provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + module: ProviderId::from_str("fictional_provider").expect("should parse provider"), }, ); let plan = AuctionPlan::compile(raw).expect("should compile standard fixture plan"); @@ -1056,7 +545,7 @@ fn standard_fixture() -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { fn standard_response_extraction_isolates_malformed_siblings_and_ignores_response_id() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({ "id": "informational-mismatch", @@ -1088,7 +577,7 @@ fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malfo value["cur"] = currency; } let response = - extract_standard_response("fictional-provider", &routed.inputs()[0], &value, 0); + extract_standard_response("fictional_provider", &routed.inputs()[0], &value, 0); assert_eq!( response.status, BidStatus::Success, @@ -1099,14 +588,14 @@ fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malfo let mut eur = bid.clone(); eur["cur"] = json!("EUR"); - let response = extract_standard_response("fictional-provider", &routed.inputs()[0], &eur, 0); + let response = extract_standard_response("fictional_provider", &routed.inputs()[0], &eur, 0); assert_eq!(response.status, BidStatus::NoBid); assert_eq!(response.metadata["unsupported_currency"], "EUR"); let mut malformed = bid; malformed["cur"] = json!(["USD"]); let response = - extract_standard_response("fictional-provider", &routed.inputs()[0], &malformed, 0); + extract_standard_response("fictional_provider", &routed.inputs()[0], &malformed, 0); assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "parse_response"); } @@ -1115,7 +604,7 @@ fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malfo fn standard_response_rejects_unknown_impressions_and_dimensions_but_keeps_siblings() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"seat": "seat", "bid": [ {"id":"good","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250}, @@ -1145,7 +634,7 @@ fn standard_response_rejects_unknown_impressions_and_dimensions_but_keeps_siblin fn standard_response_infers_only_unambiguous_missing_dimensions() { let (_plan, routed, _request) = standard_fixture(); let inferred = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"inferred","impid":"fictional-slot","price":1.0,"adm":"ok"} @@ -1170,7 +659,7 @@ fn standard_response_infers_only_unambiguous_missing_dimensions() { ]; let (_plan, routed, _request) = standard_fixture_with_formats(formats); let ambiguous = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"ambiguous","impid":"fictional-slot","price":1.0,"adm":"ok"} @@ -1205,7 +694,7 @@ fn standard_response_infers_dimensions_from_partial_unique_matches() { let (_plan, routed, _request) = standard_fixture_with_formats(formats); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"width-only","impid":"fictional-slot","price":1.0,"adm":"width","w":300}, @@ -1237,7 +726,7 @@ fn standard_response_accepts_null_and_integral_float_dimensions() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"null","impid":"fictional-slot","price":1.0,"adm":"null","w":null,"h":null}, @@ -1287,7 +776,7 @@ fn standard_response_rejects_partial_dimension_mismatch_and_ambiguity() { let (_plan, routed, _request) = standard_fixture_with_formats(formats); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"ambiguous","impid":"fictional-slot","price":1.0,"adm":"ambiguous","w":300}, @@ -1316,7 +805,7 @@ fn standard_response_rejects_invalid_numeric_dimension_shapes() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [{"bid": [ {"id":"fractional","impid":"fictional-slot","price":1.0,"adm":"bad","w":300.5}, @@ -1347,7 +836,7 @@ fn standard_response_rejects_invalid_numeric_dimension_shapes() { fn notification_suppression_matrix_uses_only_exact_valid_returned_seat() { let (_plan, routed, _request) = standard_fixture(); let response = extract_standard_response( - "fictional-provider", + "fictional_provider", &routed.inputs()[0], &json!({"seatbid": [ {"seat": "exact", "bid": [{"id":"exact","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, @@ -1485,26 +974,29 @@ fn fictional_standard_executor_covers_bid_no_bid_malformed_unused_and_redirect() ); let spec = provider.backend_spec(); assert_eq!(spec.host, "exchange.example.test"); - assert_eq!(spec.discriminator.as_deref(), Some("fictional-provider")); + assert_eq!(spec.discriminator.as_deref(), Some("fictional_provider")); }); } #[test] -fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_paths() { +fn a_completed_endpoint_reaches_generic_execution_and_a_custom_path_is_kept() { futures::executor::block_on(async { for (configured_endpoint, expected_endpoint) in [ ( - "https://pbs.example", - "https://pbs.example/openrtb2/auction", + "https://exchange.example", + "https://exchange.example/fixture/auction", + ), + ( + "https://exchange.example/bid", + "https://exchange.example/bid", ), - ("https://pbs.example/bid", "https://pbs.example/bid"), ] { let plan = AuctionPlan::compile(config_with_endpoint( - "prebid-server", + "auction.fixture", json!({}), configured_endpoint, )) - .expect("should compile Prebid Server endpoint"); + .expect("should compile the endpoint"); let inbound = Request::builder() .uri("https://publisher.example/auction") .body(EdgeBody::empty()) @@ -1531,26 +1023,24 @@ fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_ &mut reserved_backend_names, ) .await - .expect("should launch one Prebid Server request"); + .expect("should launch one request"); let ProviderRequestOutcome::Pending { request, .. } = outcome else { - panic!("should launch a pending Prebid Server request"); + panic!("should launch a pending request"); }; let selected = services .http_client() .select(vec![request]) .await - .expect("should select one Prebid Server response"); - let response = selected - .ready - .expect("should receive the Prebid Server response"); + .expect("should select one response"); + let response = selected.ready.expect("should receive the response"); assert_eq!(response.response.status(), http::StatusCode::NO_CONTENT); assert_eq!(client.recorded_backend_names(), vec!["stub-backend"]); assert_eq!(client.recorded_request_methods(), vec!["POST"]); assert_eq!(client.recorded_request_uris(), vec![expected_endpoint]); - assert_eq!(provider_plan.backend_spec().host, "pbs.example"); + assert_eq!(provider_plan.backend_spec().host, "exchange.example"); let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) - .expect("should parse recorded Prebid Server body"); + .expect("should parse the recorded body"); assert!( !body .as_object() @@ -1565,7 +1055,173 @@ fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_ fn malformed_top_level_standard_response_is_error() { let (_plan, routed, _request) = standard_fixture(); let response = - extract_standard_response("fictional-provider", &routed.inputs()[0], &json!([]), 0); + extract_standard_response("fictional_provider", &routed.inputs()[0], &json!([]), 0); assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "parse_response"); } + +/// A demand implementation that tries to write the extension the driver owns. +struct ForgingDemand; + +#[async_trait::async_trait(?Send)] +impl CompiledDemand for ForgingDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy::default() + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = Some(serde_json::Map::from_iter([( + "trusted_server".to_string(), + json!({"signature": "forged"}), + )])); + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + _response: PlatformResponse, + ) -> Result> { + Ok(AuctionResponse::error( + context.provider_id, + context.response_time_ms, + )) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[test] +fn the_driver_refuses_an_implementation_that_claims_the_trusted_server_extension() { + let (plan, routed) = routed("auction.plain-fixture", json!({})); + let mut provider = plan.providers()[0].clone(); + provider.demand = Arc::new(ForgingDemand); + + let error = match build_request( + &routed.inputs()[0], + &routed, + &provider, + 321, + &finalization(None), + ) { + Ok(_) => panic!("should refuse a forged trusted_server extension"), + Err(error) => error, + }; + + assert!( + error.to_string().contains("trusted_server"), + "should name the extension the driver owns: {error:?}" + ); +} + +#[test] +fn an_implementation_sees_each_impression_beside_the_slot_it_came_from() { + let (plan, routed) = routed("auction.fixture", json!({})); + let request = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build the request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should keep the routed impression"), + }; + + let slots = routed.inputs()[0].slots(); + assert_eq!(request.imp.len(), slots.len()); + for (imp, slot) in request.imp.iter().zip(slots) { + assert_eq!(imp.id.as_deref(), Some(slot.slot().id.as_str())); + assert!( + imp.ext + .as_ref() + .and_then(|ext| ext.get("fixture")) + .is_some(), + "each impression should carry the extension built from its own slot" + ); + } +} + +fn request_with_device_attributes() -> crate::auction::types::AuctionRequest { + let mut request = canonical_parity_auction_request(); + request + .device + .as_mut() + .expect("the parity request carries a device") + .attributes = Some(crate::ec::device::DeviceAttributes { + device_type: Some(4), + make: Some("ExampleCorp".to_owned()), + model: Some("EX-1".to_owned()), + os: Some("Android".to_owned()), + os_version: Some("15.0".to_owned()), + screen_width: Some(1080), + screen_height: Some(2400), + }); + request +} + +/// The device object is the driver's, so attributes a device module +/// resolved reach every implementation without any of them mapping a field. +#[test] +fn the_driver_carries_resolved_device_attributes_to_every_implementation() { + for implementation in ["auction.fixture", "auction.plain-fixture"] { + let request = build_with_request( + implementation, + json!({}), + request_with_device_attributes(), + None, + ); + let device = request + .device + .as_ref() + .expect("should carry a device object"); + assert_eq!( + device.devicetype, + Some(4), + "{implementation}: a bidder prices on the device type" + ); + assert_eq!( + device.make.as_deref(), + Some("ExampleCorp"), + "{implementation}" + ); + assert_eq!(device.model.as_deref(), Some("EX-1"), "{implementation}"); + assert_eq!(device.os.as_deref(), Some("Android"), "{implementation}"); + assert_eq!(device.osv.as_deref(), Some("15.0"), "{implementation}"); + assert_eq!( + device.w, + Some(1080), + "{implementation}: screen width decides which creative fits" + ); + assert_eq!(device.h, Some(2400), "{implementation}"); + } +} + +#[test] +fn the_driver_omits_device_attributes_nothing_resolved() { + let request = build_with_request( + "auction.fixture", + json!({}), + canonical_parity_auction_request(), + None, + ); + let device = request + .device + .as_ref() + .expect("should carry a device object"); + assert_eq!( + device.devicetype, None, + "an absent field is what a bidder reads as unknown, and an invented default would \ + misprice the inventory" + ); + assert_eq!(device.make, None); + assert_eq!(device.w, None); +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 204201e605..77a0266749 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -40,7 +40,7 @@ pub struct DispatchedAuction { timeout_ms: u32, floor_prices: HashMap, provider_request_context: Box>, - /// Carried so the mediator call in collect can pass it as the auction request. + /// Carried so the ad server call in collect can pass it as the auction request. request: AuctionRequest, planned_unused_bidder_params: HashMap, planned_unroutable_bidder_count: u32, @@ -148,9 +148,9 @@ pub(crate) const ERROR_TYPE_TRANSPORT: &str = "transport"; pub(crate) const ERROR_TYPE_TIMEOUT: &str = "timeout"; /// A non-2xx HTTP status from an upstream SSP (e.g. a PBS 4xx/5xx). Distinct /// from [`ERROR_TYPE_TRANSPORT`] (a connection-level failure) so telemetry can -/// bucket it separately. `pub(crate)` so producers such as the prebid provider -/// tag errors with the exact value the telemetry layer recognises. -pub(crate) const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; +/// bucket it separately. Public so a producer in a crate of its own tags +/// errors with the exact value the telemetry layer recognises. +pub const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; /// Every server-owned `error_type` classification. /// @@ -309,7 +309,7 @@ pub struct AuctionOrchestrator { plan_backed: bool, plan: Arc, planned_providers: Vec>, - mediator: Option>, + adserver: Option>, #[cfg(test)] config: AuctionConfig, #[cfg(test)] @@ -321,7 +321,7 @@ pub struct AuctionOrchestrator { pub(crate) struct AuctionOrchestratorHarness { plan: Arc, providers: Vec>, - mediator: Option>, + adserver: Option>, } struct PlannedLaunchState { @@ -338,7 +338,7 @@ struct PlannedLaunchState { impl AuctionOrchestratorHarness { pub(crate) fn new( plan: impl Into>, - mediator: Option>, + adserver: Option>, ) -> Self { let plan = plan.into(); let providers = plan @@ -351,7 +351,7 @@ impl AuctionOrchestratorHarness { Self { plan, providers, - mediator, + adserver, } } @@ -359,8 +359,8 @@ impl AuctionOrchestratorHarness { self.providers.len() } - pub(crate) fn mediator(&self) -> Option<&Arc> { - self.mediator.as_ref() + pub(crate) fn adserver(&self) -> Option<&Arc> { + self.adserver.as_ref() } /// Route and execute config-first bidder providers in deterministic order. @@ -423,11 +423,11 @@ impl AuctionOrchestratorHarness { ( input.provider_id().as_str().to_string(), unused_bidder_params_count( - &self - .plan + self.plan .provider(input.provider_id()) .expect("should find routed provider in compiled plan") - .profile, + .demand + .as_ref(), input, ), ) @@ -642,33 +642,33 @@ impl AuctionOrchestratorHarness { .collect::>(); let helper = AuctionOrchestrator::new(AuctionConfig::default()); let local_winners = || helper.select_winning_bids(&responses, &floor_prices); - let (mediator_response, winning_bids) = if let Some(mediator) = &self.mediator { + let (adserver_response, winning_bids) = if let Some(adserver) = &self.adserver { let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); + let logical_budget_ms = remaining_ms.min(adserver.timeout_ms()); if logical_budget_ms == 0 { log::warn!( - "Auction deadline exhausted before planned mediator; using local ranking" + "Auction deadline exhausted before planned adserver; using local ranking" ); (None, local_winners()) } else { let transport_timeout_ms = context .services .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(logical_budget_ms, adserver.timeout_ms()); if transport_timeout_ms == 0 { log::warn!( - "Planned mediator transport budget canonicalized to zero; using local ranking" + "Planned adserver transport budget canonicalized to zero; using local ranking" ); let winning_bids = local_winners(); return Ok(OrchestrationResult { provider_responses: responses, - mediator_response: None, + adserver_response: None, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(routed.diagnostics().unroutable_bidder_count()), }); } - let mediator_context = AuctionContext { + let adserver_context = AuctionContext { settings: context.settings, request: context.request, timeout_ms: logical_budget_ms, @@ -676,9 +676,9 @@ impl AuctionOrchestratorHarness { provider_responses: Some(&responses), services: context.services, }; - let mediator_start = Instant::now(); - let mediated = match mediator - .request_bids(original_request, &mediator_context) + let adserver_start = Instant::now(); + let decided = match adserver + .request_bids(original_request, &adserver_context) .await { Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), @@ -687,30 +687,30 @@ impl AuctionOrchestratorHarness { parse_state, }) => match context.services.http_client().wait(pending).await { Ok(platform_response) => { - let response_time_ms = mediator_start.elapsed().as_millis() as u64; + let response_time_ms = adserver_start.elapsed().as_millis() as u64; if AuctionDeadlinePolicy::for_runtime(context.services) .rejects_late_completion(auction_start, context.timeout_ms) { log::warn!( - "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", - mediator.provider_name(), + "Planned adserver '{}' completed after the hard auction deadline; using local ranking ({}ms)", + adserver.provider_name(), response_time_ms ); None } else { - mediator + adserver .parse_response_with_context_and_state( platform_response, response_time_ms, original_request, - &mediator_context, + &adserver_context, parse_state.as_deref(), ) .await .map_err(|error| { log::warn!( - "Planned mediator '{}' parse failed: {:?}", - mediator.provider_name(), + "Planned adserver '{}' parse failed: {:?}", + adserver.provider_name(), error ); }) @@ -719,8 +719,8 @@ impl AuctionOrchestratorHarness { } Err(error) => { log::warn!( - "Planned mediator '{}' request failed: {:?}", - mediator.provider_name(), + "Planned adserver '{}' request failed: {:?}", + adserver.provider_name(), error ); None @@ -728,21 +728,21 @@ impl AuctionOrchestratorHarness { }, Err(error) => { log::warn!( - "Planned mediator '{}' failed to launch: {:?}", - mediator.provider_name(), + "Planned adserver '{}' failed to launch: {:?}", + adserver.provider_name(), error ); None } }; - if let Some(mediated) = mediated { - let winners = mediated + if let Some(decided) = decided { + let winners = decided .bids .iter() .filter_map(|bid| { if bid.price.is_none() { log::warn!( - "Planned mediator returned a bid without a decoded price" + "Planned adserver returned a bid without a decoded price" ); None } else { @@ -751,7 +751,7 @@ impl AuctionOrchestratorHarness { }) .collect(); ( - Some(mediated), + Some(decided), helper.apply_floor_prices(winners, &floor_prices), ) } else { @@ -769,7 +769,7 @@ impl AuctionOrchestratorHarness { ); Ok(OrchestrationResult { provider_responses: responses, - mediator_response, + adserver_response, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(unroutable_bidder_count), @@ -785,10 +785,7 @@ impl AuctionOrchestrator { let plan = Arc::new( AuctionPlan::compile(super::plan::AuctionPlanConfig { timeout_ms: config.timeout_ms, - providers: std::collections::BTreeMap::new(), - bidders: std::collections::BTreeMap::new(), - mediator: None, - request_signing: None, + ..super::plan::AuctionPlanConfig::default() }) .expect("should compile empty legacy test plan") .with_enabled(config.enabled), @@ -799,14 +796,14 @@ impl AuctionOrchestrator { config, plan, planned_providers: Vec::new(), - mediator: None, + adserver: None, providers: HashMap::new(), } } /// Create the live orchestrator from one shared compiled auction plan. #[must_use] - pub fn from_plan(plan: Arc, mediator: Option>) -> Self { + pub fn from_plan(plan: Arc, adserver: Option>) -> Self { let planned_providers = plan .providers() .iter() @@ -819,7 +816,7 @@ impl AuctionOrchestrator { plan_backed: true, plan, planned_providers, - mediator, + adserver, #[cfg(test)] config: AuctionConfig::default(), #[cfg(test)] @@ -833,6 +830,21 @@ impl AuctionOrchestrator { Arc::ptr_eq(&self.plan, plan) } + /// Return whether each auction publishes a token with its winning bids. + #[must_use] + pub fn publishes_auction_token(&self) -> bool { + self.plan.publishes_auction_token() + } + + /// The same orchestrator in a deployment where a module reads the + /// auction token. + #[cfg(test)] + #[must_use] + pub(crate) fn publishing_auction_token(mut self) -> Self { + self.plan = Arc::new((*self.plan).clone().with_auction_token(true)); + self + } + /// Register an auction provider in the legacy parity harness. #[cfg(test)] pub(crate) fn register_provider(&mut self, provider: Arc) { @@ -886,7 +898,7 @@ impl AuctionOrchestrator { /// # Errors /// /// Returns an error if the auction execution fails due to provider errors or - /// mediation errors. + /// ad server decision errors. pub async fn run_auction( &self, request: &AuctionRequest, @@ -904,12 +916,12 @@ impl AuctionOrchestrator { #[cfg(test)] let start_time = Instant::now(); - // Auto-detect strategy based on mediator configuration. + // Auto-detect strategy based on ad server configuration. #[cfg(test)] - let (strategy_name, result) = if self.config.has_mediator() { + let (strategy_name, result) = if self.config.has_adserver() { ( - "parallel_mediation", - self.run_parallel_mediation(request, context).await?, + "parallel_adserver", + self.run_parallel_adserver(request, context).await?, ) } else { ( @@ -920,7 +932,7 @@ impl AuctionOrchestrator { #[cfg(test)] log::info!( - "Running auction with strategy: {} (auto-detected from mediator config)", + "Running auction with strategy: {} (auto-detected from adserver config)", strategy_name ); @@ -931,128 +943,130 @@ impl AuctionOrchestrator { }) } - /// Run auction with parallel bidding + mediation. + /// Run auction with parallel bidding + ad server decision. #[cfg(test)] /// /// Flow: /// 1. Run all bidders in parallel /// 2. Collect bids from all bidders - /// 3. Send combined bids to mediator for final decision - async fn run_parallel_mediation( + /// 3. Send combined bids to ad server for final decision + async fn run_parallel_adserver( &self, request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { - let mediation_start = Instant::now(); + let adserver_start = Instant::now(); let provider_responses = self.run_providers_parallel(request, context).await?; let floor_prices = self.floor_prices_by_slot(request); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - let mediator = self.get_provider(mediator_name)?; + let (adserver_response, winning_bids) = if let Some(adserver_name) = + &self.config.adserver_name + { + let adserver = self.get_provider(adserver_name)?; log::info!( - "Sending {} provider responses to mediator: {}", + "Sending {} provider responses to adserver: {}", provider_responses.len(), - mediator.provider_name() + adserver.provider_name() ); - // Give the mediator only the remaining time from the auction + // Give the ad server only the remaining time from the auction // deadline, not the full timeout — the bidding phase already // consumed part of it. Canonicalize the transport timeout so the // backend name remains stable across equivalent budget values. - let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); - let mediator_timeout = context + let remaining_ms = remaining_budget_ms(adserver_start, context.timeout_ms); + let adserver_timeout = context .services .backend() - .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(remaining_ms, adserver.timeout_ms()); - if mediator_timeout == 0 { - log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); + if adserver_timeout == 0 { + log::warn!("Auction timeout exhausted during bidding phase; skipping adserver"); let winning = self.select_winning_bids(&provider_responses, &floor_prices); return Ok(OrchestrationResult { provider_responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: 0, metadata: HashMap::new(), }); } - let mediator_context = AuctionContext { + let adserver_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: mediator_timeout, - transport_timeout_ms: mediator_timeout, + timeout_ms: adserver_timeout, + transport_timeout_ms: adserver_timeout, provider_responses: Some(&provider_responses), services: context.services, }; let start_time = Instant::now(); - let mediator_resp = match mediator - .request_bids(request, &mediator_context) + let adserver_resp = match adserver + .request_bids(request, &adserver_context) .await .change_context(TrustedServerError::Auction { - message: format!("Mediator {} failed to launch", mediator.provider_name()), + message: format!("AdServer {} failed to launch", adserver.provider_name()), })? { ProviderRequestOutcome::Immediate(response) => response, ProviderRequestOutcome::Pending { request: pending, parse_state, } => { - let platform_resp = mediator_context + let platform_resp = adserver_context .services .http_client() .wait(pending) .await .change_context(TrustedServerError::Auction { message: format!( - "Mediator {} request failed", - mediator.provider_name() + "AdServer {} request failed", + adserver.provider_name() ), })?; let response_time_ms = start_time.elapsed().as_millis() as u64; if AuctionDeadlinePolicy::for_runtime(context.services) - .rejects_late_completion(mediation_start, context.timeout_ms) + .rejects_late_completion(adserver_start, context.timeout_ms) { log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", - mediator.provider_name(), + "AdServer '{}' completed after the hard auction deadline; using local ranking ({}ms)", + adserver.provider_name(), response_time_ms ); let winning = self.select_winning_bids(&provider_responses, &floor_prices); return Ok(OrchestrationResult { provider_responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: 0, metadata: HashMap::new(), }); } - mediator + adserver .parse_response_with_context_and_state( platform_resp, response_time_ms, request, - &mediator_context, + &adserver_context, parse_state.as_deref(), ) .await .change_context(TrustedServerError::Auction { - message: format!("Mediator {} parse failed", mediator.provider_name()), + message: format!("AdServer {} parse failed", adserver.provider_name()), })? } }; - // Extract only mediator bids with comparable numeric prices. - let winning = mediator_resp + // Extract only ad server bids with comparable numeric prices. + let winning = adserver_resp .bids .iter() .filter_map(|bid| { if bid.price.is_none() { log::warn!( - "Mediator '{}' returned bid for slot '{}' without a price - skipping", - mediator.provider_name(), + "AdServer '{}' returned bid for slot '{}' without a price - skipping", + adserver.provider_name(), bid.slot_id ); None @@ -1063,25 +1077,25 @@ impl AuctionOrchestrator { .collect(); ( - Some(mediator_resp), + Some(adserver_resp), self.apply_floor_prices(winning, &floor_prices), ) } else { - // No mediator - select best bid per slot from bidder responses + // No ad server - select best bid per slot from bidder responses let winning = self.select_winning_bids(&provider_responses, &floor_prices); (None, winning) }; Ok(OrchestrationResult { provider_responses, - mediator_response, + adserver_response, winning_bids, total_time_ms: 0, // Will be set by caller metadata: HashMap::new(), }) } - /// Run auction with only parallel bidding (no mediation). + /// Run auction with only parallel bidding (no ad server decision). #[cfg(test)] async fn run_parallel_only( &self, @@ -1094,7 +1108,7 @@ impl AuctionOrchestrator { Ok(OrchestrationResult { provider_responses, - mediator_response: None, + adserver_response: None, winning_bids, total_time_ms: 0, metadata: HashMap::new(), @@ -1113,9 +1127,9 @@ impl AuctionOrchestrator { ) -> Result, Report> { let provider_names = self .config - .providers - .keys() - .map(super::plan::ProviderId::as_str) + .provider_names + .iter() + .map(String::as_str) .collect::>(); if provider_names.is_empty() { @@ -1616,10 +1630,10 @@ impl AuctionOrchestrator { ( input.provider_id().as_str().to_string(), unused_bidder_params_count( - &plan - .provider(input.provider_id()) + plan.provider(input.provider_id()) .expect("should find routed provider in compiled plan") - .profile, + .demand + .as_ref(), input, ), ) @@ -1648,10 +1662,10 @@ impl AuctionOrchestrator { materialize_planned_response( provider_launch_failed_response(input.provider_id().as_str(), 0), unused_bidder_params_count( - &plan - .provider(input.provider_id()) + plan.provider(input.provider_id()) .expect("should find routed provider in compiled plan") - .profile, + .demand + .as_ref(), input, ), ) @@ -1859,9 +1873,9 @@ impl AuctionOrchestrator { #[cfg(test)] let provider_names = self .config - .providers - .keys() - .map(super::plan::ProviderId::as_str) + .provider_names + .iter() + .map(String::as_str) .collect::>(); #[cfg(test)] if provider_names.is_empty() { @@ -2085,7 +2099,7 @@ impl AuctionOrchestrator { /// Collect bid responses from a previously-dispatched auction. /// /// Runs the select-loop phase (equivalent to Phase 2 of - /// `run_providers_parallel`) and, if the orchestrator has a mediator + /// `run_providers_parallel`) and, if the orchestrator has an ad server /// configured, forwards collected bids to it. The overall auction deadline /// is enforced from `dispatched.auction_start`. /// @@ -2292,7 +2306,7 @@ impl AuctionOrchestrator { // collect phase the remaining handles may already be ready even if // wall-clock time elapsed while the origin was slow. Dropping them // here would discard SSP responses that already arrived. The - // mediator launch below still observes A_deadline via + // ad server launch below still observes A_deadline via // `remaining_budget_ms`. } @@ -2329,35 +2343,35 @@ impl AuctionOrchestrator { } #[cfg(not(test))] - let mediator = self.mediator.as_ref(); + let adserver = self.adserver.as_ref(); #[cfg(test)] - let mediator = self.mediator.as_ref().or_else(|| { + let adserver = self.adserver.as_ref().or_else(|| { self.config - .mediator + .adserver_name .as_ref() .and_then(|name| self.providers.get(name)) }); - let (mediator_response, winning_bids) = if let Some(mediator) = mediator { + let (adserver_response, winning_bids) = if let Some(adserver) = adserver { { - // Cap the mediator at whichever is tighter: its own configured + // Cap the ad server at whichever is tighter, being its own configured // timeout or the remaining auction budget (A_deadline). Backend // first-byte and between-bytes timeouts bound normal collection, but // they are transport timers rather than absolute wall-clock limits: // connection setup and byte-trickling can still consume more of the // auction budget. Recomputing the remaining budget here prevents the - // mediator from extending that bounded response hold. + // ad server from extending that bounded response hold. let remaining = remaining_budget_ms(auction_start, timeout_ms); - let logical_budget_ms = remaining.min(mediator.timeout_ms()); + let logical_budget_ms = remaining.min(adserver.timeout_ms()); if logical_budget_ms == 0 { log::warn!( - "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", - mediator.provider_name(), + "A_deadline exhausted before adserver '{}', returning {} SSP bids without an ad server decision", + adserver.provider_name(), responses.len(), ); let winning = self.select_winning_bids(&responses, &floor_prices); return OrchestrationResult { provider_responses: responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(planned_unroutable_bidder_count), @@ -2365,43 +2379,43 @@ impl AuctionOrchestrator { } let transport_timeout_ms = services .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + .canonicalize_transport_timeout_ms(logical_budget_ms, adserver.timeout_ms()); if transport_timeout_ms == 0 { log::warn!( - "Mediator '{}' transport budget canonicalized to zero — returning {} SSP bids without mediation", - mediator.provider_name(), + "AdServer '{}' transport budget canonicalized to zero, returning {} SSP bids without an ad server decision", + adserver.provider_name(), responses.len(), ); let winning = self.select_winning_bids(&responses, &floor_prices); return OrchestrationResult { provider_responses: responses, - mediator_response: None, + adserver_response: None, winning_bids: winning, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(planned_unroutable_bidder_count), }; } - let mediator_start = Instant::now(); - // This logs a configured mediator identifier and timeout values, not request data or secrets. + let adserver_start = Instant::now(); + // This logs a configured ad server identifier and timeout values, not request data or secrets. log::info!( - "Running mediator '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", - mediator.provider_name(), + "Running adserver '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", + adserver.provider_name(), logical_budget_ms, transport_timeout_ms, remaining, - mediator.timeout_ms(), + adserver.timeout_ms(), ); - // The mediator runs on the collect path. See the doc-comment on + // The ad server runs on the collect path. See the doc-comment on // `AuctionContext::request`: the real client request was already // consumed by `send_async` during dispatch, so we substitute a - // canonical placeholder URL. Any future mediator that needs real + // canonical placeholder URL. Any future ad server that needs real // client headers must snapshot them at dispatch time onto // `DispatchedAuction` rather than reading `context.request` here. let placeholder = http::Request::builder() .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) .body(edgezero_core::body::Body::empty()) .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); - let mediator_context = AuctionContext { + let adserver_context = AuctionContext { settings: context.settings, request: &placeholder, timeout_ms: logical_budget_ms, @@ -2409,8 +2423,8 @@ impl AuctionOrchestrator { provider_responses: Some(&responses), services: context.services, }; - let mediator_response = match mediator - .request_bids(&request, &mediator_context) + let adserver_response = match adserver + .request_bids(&request, &adserver_context) .await { Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), @@ -2420,27 +2434,27 @@ impl AuctionOrchestrator { }) => match services.http_client().wait(pending).await.change_context( TrustedServerError::Auction { message: format!( - "Mediator {} request failed", - mediator.provider_name() + "AdServer {} request failed", + adserver.provider_name() ), }, ) { Ok(platform_resp) => { - let response_time_ms = mediator_start.elapsed().as_millis() as u64; + let response_time_ms = adserver_start.elapsed().as_millis() as u64; if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { log::warn!( - "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", - mediator.provider_name(), + "AdServer '{}' completed after the hard auction deadline; using local ranking ({}ms)", + adserver.provider_name(), response_time_ms ); None } else { - match mediator + match adserver .parse_response_with_context_and_state( platform_resp, response_time_ms, &request, - &mediator_context, + &adserver_context, parse_state.as_deref(), ) .await @@ -2448,8 +2462,8 @@ impl AuctionOrchestrator { Ok(response) => Some(response), Err(error) => { log::warn!( - "Mediator '{}' parse failed: {:?}", - mediator.provider_name(), + "AdServer '{}' parse failed: {:?}", + adserver.provider_name(), error ); None @@ -2458,29 +2472,29 @@ impl AuctionOrchestrator { } } Err(error) => { - log::warn!("Mediator request failed: {:?}", error); + log::warn!("AdServer request failed: {:?}", error); None } }, Err(error) => { log::warn!( - "Mediator '{}' failed to dispatch: {:?}", - mediator.provider_name(), + "AdServer '{}' failed to dispatch: {:?}", + adserver.provider_name(), error ); None } }; - if let Some(mediator_response) = mediator_response { - let winning = mediator_response + if let Some(adserver_response) = adserver_response { + let winning = adserver_response .bids .iter() .filter_map(|bid| { if bid.price.is_none() { log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), + "AdServer '{}' returned bid for slot '{}' without decoded price - skipping", + adserver.provider_name(), bid.slot_id ); None @@ -2490,7 +2504,7 @@ impl AuctionOrchestrator { }) .collect(); let winning = self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_response), winning) + (Some(adserver_response), winning) } else { (None, self.select_winning_bids(&responses, &floor_prices)) } @@ -2501,7 +2515,7 @@ impl AuctionOrchestrator { OrchestrationResult { provider_responses: responses, - mediator_response, + adserver_response, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, metadata: routing_metadata(planned_unroutable_bidder_count), @@ -2520,8 +2534,8 @@ impl AuctionOrchestrator { pub struct OrchestrationResult { /// All responses from providers pub provider_responses: Vec, - /// Final response from mediator (if used) - pub mediator_response: Option, + /// Final response from ad server (if used) + pub adserver_response: Option, /// Winning bids per slot pub winning_bids: HashMap, /// Total orchestration time in milliseconds @@ -2534,7 +2548,7 @@ impl OrchestrationResult { fn no_bid() -> Self { Self { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 0, metadata: HashMap::new(), @@ -2566,30 +2580,25 @@ impl OrchestrationResult { #[cfg(test)] mod tests { - use std::str::FromStr as _; use std::time::Duration; - use base64::Engine as _; use web_time::Instant; use crate::auction::config::AuctionConfig; use crate::auction::orchestrator::DispatchAuctionOutcome; - use crate::auction::plan::{ - AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, - }; + use crate::auction::plan::{AuctionPlan, AuctionPlanConfig, NotificationConfig, RoutingMode}; use crate::auction::provider::{ AuctionProvider, GenericOpenRtbProvider, ProviderRequestOutcome, }; use crate::auction::routing::{RoutingDiagnostics, route_auction}; - use crate::auction::test_support::create_test_auction_context; + use crate::auction::test_support::{create_test_auction_context, demand_table, plan_config}; use crate::auction::types::{ - AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, - AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, + BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; - use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; use crate::platform::test_support::{ - StubHttpClient, build_services_with_backend_and_http_client, + NamingBackend, StubHttpClient, build_services_with_backend_and_http_client, build_services_with_http_client, noop_services, }; use crate::platform::{ @@ -2600,7 +2609,7 @@ mod tests { }; use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; - use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -2609,98 +2618,72 @@ mod tests { ERROR_TYPE_LAUNCH_FAILED, ERROR_TYPE_TIMEOUT, ERROR_TYPE_TRANSPORT, OrchestrationResult, }; - fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { - AuctionPlanConfig { - timeout_ms: 777, - providers: providers - .iter() - .map(|(id, routing)| { - ( - ProviderId::from_str(id).expect("should parse fictional provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: "https://example.test/openrtb".to_string(), - timeout_ms: Some(1_000), - routing: *routing, - notifications: Default::default(), - profile_config: serde_json::json!({}), - }, - ) - }) - .collect(), - bidders: BTreeMap::new(), - mediator: None, - request_signing: signing.then(|| crate::settings::RequestSigning { - enabled: true, - config_store_id: "fictional-config-store".to_string(), - secret_store_id: "fictional-secret-store".to_string(), - }), + /// One `[demand.]` table for a planned test source. + fn planned_table( + implementation: &str, + endpoint: &str, + routing: RoutingMode, + settings: &serde_json::Value, + notifications: &NotificationConfig, + ) -> serde_json::Map { + let mut entry = demand_table(implementation, endpoint); + entry.insert("timeout_ms".to_string(), serde_json::json!(1_000)); + if routing == RoutingMode::AllEligible { + entry.insert("routing".to_string(), serde_json::json!("all_eligible")); } - } - - fn planned_prebid_config( - providers: &[(&str, serde_json::Value, NotificationConfig)], - ) -> AuctionPlanConfig { - AuctionPlanConfig { - timeout_ms: 777, - providers: providers - .iter() - .map(|(id, profile_config, notifications)| { - ( - ProviderId::from_str(id).expect("should parse fictional provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "prebid-server".to_string(), - endpoint: format!("https://{id}.example.test/openrtb"), - timeout_ms: Some(1_000), - routing: RoutingMode::Explicit, - notifications: notifications.clone(), - profile_config: profile_config.clone(), - }, - ) - }) - .collect(), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, + entry.insert( + "notifications".to_string(), + serde_json::to_value(notifications).expect("should serialize notifications"), + ); + if let serde_json::Value::Object(settings) = settings { + entry.extend(settings.clone()); } + entry } - fn planned_aps_config() -> AuctionPlanConfig { - planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({"account_id": "example-account"}), - NotificationConfig::default(), - )]) + fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { + let tables = providers + .iter() + .map(|(id, routing)| { + ( + *id, + planned_table( + "auction.plain-fixture", + "https://example.test/openrtb", + *routing, + &serde_json::json!({}), + &NotificationConfig::default(), + ), + ) + }) + .collect::>(); + let mut config = plan_config(tables); + config.timeout_ms = 777; + config.request_signing = signing.then(|| crate::settings::RequestSigning::new(true)); + config } - fn planned_aps_instances_config( + fn planned_fixture_config( providers: &[(&str, serde_json::Value, NotificationConfig)], ) -> AuctionPlanConfig { - AuctionPlanConfig { - timeout_ms: 777, - providers: providers - .iter() - .map(|(id, profile_config, notifications)| { - ( - ProviderId::from_str(id).expect("should parse fictional provider ID"), - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), - timeout_ms: Some(1_000), - routing: RoutingMode::AllEligible, - notifications: notifications.clone(), - profile_config: profile_config.clone(), - }, - ) - }) - .collect(), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, - } + let tables = providers + .iter() + .map(|(id, settings, notifications)| { + ( + *id, + planned_table( + "auction.fixture", + &format!("https://{id}.example.test/openrtb"), + RoutingMode::Explicit, + settings, + notifications, + ), + ) + }) + .collect::>(); + let mut config = plan_config(tables); + config.timeout_ms = 777; + config } fn planned_request() -> AuctionRequest { @@ -2732,7 +2715,7 @@ mod tests { } } - fn planned_prebid_request() -> AuctionRequest { + fn planned_envelope_request() -> AuctionRequest { let mut request = planned_request(); request.slots[0] .bidders @@ -2744,7 +2727,7 @@ mod tests { async fn disabled_from_plan_is_a_no_work_kill_switch_for_sync_and_split_paths() { let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile plan") @@ -2815,14 +2798,14 @@ mod tests { async fn all_planned_launch_failures_error_direct_and_surface_split_failure() { let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("launch-fail", RoutingMode::AllEligible)], + &[("launch_fail", RoutingMode::AllEligible)], false, )) .expect("should compile launch-failure plan"), ); let orchestrator = AuctionOrchestrator::from_plan(plan, None); let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); + backend.fail_ensure_for("launch_fail"); let http = Arc::new(StubHttpClient::new()); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, @@ -2869,7 +2852,7 @@ mod tests { "should keep one launch-failure response" ); assert_eq!( - provider_responses[0].provider, "launch-fail", + provider_responses[0].provider, "launch_fail", "should attribute the response to the failed provider" ); assert_eq!( @@ -2884,7 +2867,7 @@ mod tests { let plan = Arc::new( AuctionPlan::compile(planned_config( &[ - ("launch-fail", RoutingMode::AllEligible), + ("launch_fail", RoutingMode::AllEligible), ("timeout", RoutingMode::AllEligible), ], false, @@ -2921,7 +2904,7 @@ mod tests { ); assert!( result.provider_responses.iter().any(|response| { - response.provider == "launch-fail" + response.provider == "launch_fail" && response.metadata["error_type"] == ERROR_TYPE_LAUNCH_FAILED }), "should retain the launch-failure outcome" @@ -2992,71 +2975,6 @@ mod tests { } } - struct NamingBackend { - policy: BackendNamingPolicy, - predicted: AtomicUsize, - ensured: AtomicUsize, - specs: Mutex>, - fail_ensure_for: Mutex>, - } - - impl NamingBackend { - fn new(policy: BackendNamingPolicy) -> Self { - Self { - policy, - predicted: AtomicUsize::new(0), - ensured: AtomicUsize::new(0), - specs: Mutex::new(Vec::new()), - fail_ensure_for: Mutex::new(HashSet::new()), - } - } - - fn fail_ensure_for(&self, provider_id: &str) { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .insert(provider_id.to_string()); - } - - fn name(&self, spec: &PlatformBackendSpec) -> Result> { - self.policy - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - } - - impl PlatformBackend for NamingBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - self.policy - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - self.predicted.fetch_add(1, Ordering::Relaxed); - self.name(spec) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.ensured.fetch_add(1, Ordering::Relaxed); - if spec.discriminator.as_deref().is_some_and(|provider_id| { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .contains(provider_id) - }) { - return Err(Report::new(PlatformError::Backend)); - } - self.specs - .lock() - .expect("should lock planned backend specs") - .push(spec.clone()); - self.name(spec) - } - } - struct ZeroCanonicalBackend { predicted: AtomicUsize, ensured: AtomicUsize, @@ -3452,19 +3370,19 @@ mod tests { } } - type RecordedMediatorBudgets = Arc>>; + type RecordedAdServerBudgets = Arc>>; - struct DeadlineRecordingMediator { + struct DeadlineRecordingAdServer { launches: Arc, - budgets: Option, + budgets: Option, } - struct PendingDeadlineMediator; + struct PendingDeadlineAdServer; #[async_trait::async_trait(?Send)] - impl AuctionProvider for PendingDeadlineMediator { + impl AuctionProvider for PendingDeadlineAdServer { fn provider_name(&self) -> &str { - "pending-deadline-mediator" + "pending_deadline_adserver" } async fn request_bids( @@ -3477,8 +3395,8 @@ mod tests { .method("POST") .uri("https://example.com/mediate") .body(edgezero_core::body::Body::empty()) - .expect("should build pending mediator request"), - "pending-mediator-backend", + .expect("should build pending adserver request"), + "pending-adserver-backend", ); context .services @@ -3486,7 +3404,7 @@ mod tests { .send_async(request) .await .change_context(TrustedServerError::Auction { - message: "pending mediator launch failed".to_string(), + message: "pending adserver launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -3498,7 +3416,7 @@ mod tests { ) -> Result> { Ok(AuctionResponse::success( self.provider_name(), - vec![auction_bid("mediated", 9.0)], + vec![auction_bid("adserver", 9.0)], response_time_ms, )) } @@ -3508,14 +3426,14 @@ mod tests { } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("pending-mediator-backend".to_string()) + Some("pending-adserver-backend".to_string()) } } #[async_trait::async_trait(?Send)] - impl AuctionProvider for DeadlineRecordingMediator { + impl AuctionProvider for DeadlineRecordingAdServer { fn provider_name(&self) -> &str { - "deadline-mediator" + "deadline_adserver" } async fn request_bids( @@ -3527,7 +3445,7 @@ mod tests { if let Some(budgets) = &self.budgets { budgets .lock() - .expect("should lock mediator budgets") + .expect("should lock adserver budgets") .push((context.timeout_ms, context.transport_timeout_ms)); } Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( @@ -3541,7 +3459,7 @@ mod tests { _response: PlatformResponse, _response_time_ms: u64, ) -> Result> { - panic!("immediate mediator response should not be parsed"); + panic!("immediate adserver response should not be parsed"); } fn timeout_ms(&self) -> u32 { @@ -3719,24 +3637,23 @@ mod tests { } } - /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring - /// `adserver_mock`), while its context-free parse does not. Lets a test prove - /// the synchronous mediation path calls `parse_response_with_context`. - struct CacheRestoringMediator; + /// Ad server whose context-aware parse restores `nurl`/`ad_id`, as one that + /// keeps them out of its request does, while its context-free parse does not. Lets a test prove + /// the synchronous ad server decision path calls `parse_response_with_context`. + struct CacheRestoringAdServer; fn auction_bid(bidder: &str, price: f64) -> Bid { - let renderer = (bidder == "aps").then(|| { - BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "aps-selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }) + let renderer = (bidder == "typed").then(|| { + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "typed-selected-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor") }); Bid { slot_id: "slot-1".to_string(), @@ -3752,7 +3669,7 @@ mod tests { height: 250, nurl: None, burl: None, - bid_id: (bidder == "aps").then(|| "aps-selected-bid".to_string()), + bid_id: (bidder == "typed").then(|| "typed-selected-bid".to_string()), ad_id: None, creative_id: None, renderer, @@ -3763,14 +3680,14 @@ mod tests { } } - fn mediated_bid(nurl: Option) -> Bid { + fn adserver_bid(nurl: Option) -> Bid { Bid { slot_id: "header-banner".to_string(), price: Some(2.5), currency: "USD".to_string(), creative: Some("
ad
".to_string()), adomain: None, - bidder: "mediator".to_string(), + bidder: "adserver".to_string(), returned_seat: None, width: 728, height: 90, @@ -3788,9 +3705,9 @@ mod tests { } #[async_trait::async_trait(?Send)] - impl AuctionProvider for CacheRestoringMediator { + impl AuctionProvider for CacheRestoringAdServer { fn provider_name(&self) -> &str { - "mediator" + "adserver" } async fn request_bids( @@ -3803,8 +3720,8 @@ mod tests { .method("POST") .uri("https://example.com/mediate") .body(edgezero_core::body::Body::empty()) - .expect("should build mediator request"), - "mediator-backend", + .expect("should build adserver request"), + "adserver-backend", ); context .services @@ -3812,7 +3729,7 @@ mod tests { .send_async(req) .await .change_context(TrustedServerError::Auction { - message: "mediator launch failed".to_string(), + message: "adserver launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -3824,8 +3741,8 @@ mod tests { ) -> Result> { // Context-free path: cannot restore SSP-only render/accounting fields. Ok(AuctionResponse::success( - "mediator", - vec![mediated_bid(None)], + "adserver", + vec![adserver_bid(None)], response_time_ms, )) } @@ -3839,8 +3756,8 @@ mod tests { ) -> Result> { // Context-aware path: restores nurl/ad_id from the collected SSP bids. Ok(AuctionResponse::success( - "mediator", - vec![mediated_bid(Some("https://nurl.example/win".to_string()))], + "adserver", + vec![adserver_bid(Some("https://nurl.example/win".to_string()))], response_time_ms, )) } @@ -3850,16 +3767,16 @@ mod tests { } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("mediator-backend".to_string()) + Some("adserver-backend".to_string()) } } - struct ImmediateMediator; + struct ImmediateAdServer; #[async_trait::async_trait(?Send)] - impl AuctionProvider for ImmediateMediator { + impl AuctionProvider for ImmediateAdServer { fn provider_name(&self) -> &str { - "immediate-mediator" + "immediate_adserver" } async fn request_bids( @@ -3869,7 +3786,7 @@ mod tests { ) -> Result> { Ok(ProviderRequestOutcome::Immediate(AuctionResponse::success( self.provider_name(), - vec![mediated_bid(Some( + vec![adserver_bid(Some( "https://nurl.example/immediate".to_string(), ))], 0, @@ -3881,7 +3798,7 @@ mod tests { _response: PlatformResponse, _response_time_ms: u64, ) -> Result> { - panic!("immediate mediator response should not be parsed"); + panic!("immediate adserver response should not be parsed"); } fn timeout_ms(&self) -> u32 { @@ -3890,22 +3807,22 @@ mod tests { } #[tokio::test] - async fn mediated_bid_preserves_restored_fields_through_run_auction() { - // run_parallel_mediation must parse the mediator response via + async fn adserver_bid_preserves_restored_fields_through_run_auction() { + // run_parallel_adserver must parse the ad server response via // parse_response_with_context so cache/nurl fields restored from SSP - // responses survive the synchronous mediation path (POST /auction, + // responses survive the synchronous ad server decision path (POST /auction, // /_ts/page-bids), matching the dispatched collect path. let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); // bidder send_async - stub.push_response(200, b"{}".to_vec()); // mediator send_async + stub.push_response(200, b"{}".to_vec()); // adserver send_async let services = build_services_with_http_client(stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. let services: &'static RuntimeServices = Box::leak(Box::new(services)); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("adserver".to_string()), timeout_ms: 2000, ..Default::default() }; @@ -3914,7 +3831,7 @@ mod tests { name: "bidder", backend: "bidder-backend", })); - orchestrator.register_provider(Arc::new(CacheRestoringMediator)); + orchestrator.register_provider(Arc::new(CacheRestoringAdServer)); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -3935,34 +3852,34 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("mediated auction should complete"); + .expect("ad server auction should complete"); let bid = result .winning_bids .get("header-banner") - .expect("mediator should produce a winning bid for the slot"); + .expect("adserver should produce a winning bid for the slot"); assert_eq!( bid.nurl.as_deref(), Some("https://nurl.example/win"), - "synchronous mediation must restore nurl via parse_response_with_context" + "the synchronous ad server must restore nurl via parse_response_with_context" ); assert_eq!( bid.ad_id.as_deref(), Some("creative-123"), - "mediated bid must keep its restored ad_id" + "decided bid must keep its restored ad_id" ); } #[tokio::test] - async fn immediate_mediator_completes_in_sync_and_split_paths() { + async fn immediate_adserver_completes_in_sync_and_split_paths() { for split in [false, true] { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("immediate-mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("immediate_adserver".to_string()), timeout_ms: 2000, ..Default::default() }; @@ -3971,7 +3888,7 @@ mod tests { name: "bidder", backend: "bidder-backend", })); - orchestrator.register_provider(Arc::new(ImmediateMediator)); + orchestrator.register_provider(Arc::new(ImmediateAdServer)); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -3997,15 +3914,15 @@ mod tests { orchestrator .run_auction(&request, &context) .await - .expect("auction with immediate mediator should complete") + .expect("auction with immediate adserver should complete") }; assert_eq!( result - .mediator_response + .adserver_response .as_ref() .map(|response| response.provider.as_str()), - Some("immediate-mediator") + Some("immediate_adserver") ); assert_eq!( result @@ -4029,17 +3946,17 @@ mod tests { let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one", "late-two"]), + provider_names: vec!["late_one".to_string(), "late_two".to_string()], timeout_ms: 10, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", + name: "late_one", backend: "late-one-backend", })); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-two", + name: "late_two", backend: "late-two-backend", })); let request = create_test_auction_request(); @@ -4076,9 +3993,9 @@ mod tests { for split in [false, true] { let result = collect_deadline_test_result(split, false).await; assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "late-one"); + assert_eq!(result.provider_responses[0].provider, "late_one"); assert_eq!(result.provider_responses[0].status, BidStatus::Success); - assert_eq!(result.provider_responses[1].provider, "late-two"); + assert_eq!(result.provider_responses[1].provider, "late_two"); assert_eq!(result.provider_responses[1].status, BidStatus::Success); assert!( result @@ -4088,7 +4005,7 @@ mod tests { "late response times should retain actual elapsed duration" ); assert_eq!( - result.winning_bids["slot-1"].bidder, "late-one", + result.winning_bids["slot-1"].bidder, "late_one", "a completed response remains eligible after the logical deadline" ); } @@ -4108,7 +4025,7 @@ mod tests { } } - async fn pending_mediator_deadline_test_result( + async fn pending_adserver_deadline_test_result( split: bool, enforceable_total_request_deadline: bool, ) -> OrchestrationResult { @@ -4121,8 +4038,8 @@ mod tests { let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["local"]), - mediator: Some("pending-deadline-mediator".to_string()), + provider_names: vec!["local".to_string()], + adserver_name: Some("pending_deadline_adserver".to_string()), timeout_ms: 20, ..Default::default() }; @@ -4131,7 +4048,7 @@ mod tests { name: "local", backend: "local-backend", })); - orchestrator.register_provider(Arc::new(PendingDeadlineMediator)); + orchestrator.register_provider(Arc::new(PendingDeadlineAdServer)); let request = create_test_auction_request(); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -4162,31 +4079,31 @@ mod tests { } #[tokio::test] - async fn pending_mediator_late_completion_policy_is_equivalent_in_sync_and_split_paths() { + async fn pending_adserver_late_completion_policy_is_equivalent_in_sync_and_split_paths() { for split in [false, true] { - let current = pending_mediator_deadline_test_result(split, false).await; - let current_mediator = current - .mediator_response + let current = pending_adserver_deadline_test_result(split, false).await; + let current_adserver = current + .adserver_response .as_ref() - .expect("current adapters should accept completed late mediator responses"); + .expect("current adapters should accept completed late adserver responses"); assert!( - current_mediator.response_time_ms >= 50, - "mediator timing should preserve actual elapsed duration" + current_adserver.response_time_ms >= 50, + "adserver timing should preserve actual elapsed duration" ); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + assert_eq!(current.winning_bids["slot-1"].bidder, "adserver"); - let hard = pending_mediator_deadline_test_result(split, true).await; - assert!(hard.mediator_response.is_none()); + let hard = pending_adserver_deadline_test_result(split, true).await; + assert!(hard.adserver_response.is_none()); assert_eq!(hard.winning_bids["slot-1"].bidder, "local"); assert!( hard.total_time_ms >= 50, - "discarding a late mediator must retain actual total elapsed time" + "discarding a late adserver must retain actual total elapsed time" ); } } #[tokio::test] - async fn split_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn split_deadline_skips_adserver_and_falls_back_to_provider_winner() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4194,17 +4111,17 @@ mod tests { let launches = Arc::new(AtomicUsize::new(0)); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one"]), - mediator: Some("deadline-mediator".to_string()), + provider_names: vec!["late_one".to_string()], + adserver_name: Some("deadline_adserver".to_string()), timeout_ms: 10, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", + name: "late_one", backend: "late-one-backend", })); - orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + orchestrator.register_provider(Arc::new(DeadlineRecordingAdServer { launches: Arc::clone(&launches), budgets: None, })); @@ -4229,12 +4146,12 @@ mod tests { .await; assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!(result.adserver_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late_one"); } #[tokio::test] - async fn synchronous_deadline_skips_mediator_and_falls_back_to_provider_winner() { + async fn synchronous_deadline_skips_adserver_and_falls_back_to_provider_winner() { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); stub.push_select_delay(Duration::from_millis(50)); @@ -4242,17 +4159,17 @@ mod tests { let launches = Arc::new(AtomicUsize::new(0)); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one"]), - mediator: Some("deadline-mediator".to_string()), + provider_names: vec!["late_one".to_string()], + adserver_name: Some("deadline_adserver".to_string()), timeout_ms: 10, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", + name: "late_one", backend: "late-one-backend", })); - orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + orchestrator.register_provider(Arc::new(DeadlineRecordingAdServer { launches: Arc::clone(&launches), budgets: None, })); @@ -4273,8 +4190,8 @@ mod tests { .expect("synchronous deadline test should complete"); assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!(result.mediator_response.is_none()); - assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + assert!(result.adserver_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late_one"); } fn create_test_auction_request() -> AuctionRequest { @@ -4412,7 +4329,7 @@ mod tests { async fn synchronous_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate"]), + provider_names: vec!["immediate".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4437,7 +4354,7 @@ mod tests { async fn split_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate"]), + provider_names: vec!["immediate".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4468,7 +4385,7 @@ mod tests { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate", "pending"]), + provider_names: vec!["immediate".to_string(), "pending".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4682,7 +4599,7 @@ mod tests { // Timeout paths still need a controllable delayed pending response: // - Deadline check in select() loop (drops remaining requests) - // - Mediator skip when remaining_ms == 0 (bidding exhausts budget) + // - Ad server skip when remaining_ms == 0 (bidding exhausts budget) // - Provider skip when effective_timeout == 0 (budget exhausted before launch) // - Provider context receives reduced timeout_ms per remaining budget // @@ -4696,12 +4613,13 @@ mod tests { enabled: true, sanitize_creatives: true, rewrite_creatives: true, - providers: AuctionConfig::legacy_provider_map(&[]), + provider_names: vec![], bidders: Default::default(), - mediator: None, + adserver_name: None, timeout_ms: 2000, creative_store: "creative_store".to_string(), allowed_context_keys: BTreeSet::from(["permutive_segments".to_string()]), + ..Default::default() }; let orchestrator = AuctionOrchestrator::new(config); @@ -4728,7 +4646,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["launch-failing"]), + provider_names: vec!["launch_failing".to_string()], timeout_ms: 2000, ..Default::default() }; @@ -4763,17 +4681,17 @@ mod tests { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "shared-backend", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "shared-backend", })); let stub = Arc::new(StubHttpClient::new()); @@ -4801,10 +4719,10 @@ mod tests { }; assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success + response.provider == "provider_a" && response.status == BidStatus::Success })); assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-b" && response.status == BidStatus::Error + response.provider == "provider_b" && response.status == BidStatus::Error })); } } @@ -4878,7 +4796,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), + provider_names: vec!["bidder".to_string()], timeout_ms: 2000, ..Default::default() }); @@ -4922,7 +4840,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), + provider_names: vec!["bidder".to_string()], timeout_ms: 2000, ..Default::default() }); @@ -4948,7 +4866,7 @@ mod tests { } #[test] - fn synchronous_mediation_applies_canonical_timeout_to_mediator() { + fn synchronous_adserver_applies_canonical_timeout_to_adserver() { futures::executor::block_on(async { let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); @@ -4965,8 +4883,8 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("adserver".to_string()), timeout_ms: 2000, ..Default::default() }); @@ -4975,8 +4893,8 @@ mod tests { backend: "bidder-backend", })); orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", + "adserver", + "adserver-backend", 2000, &predicted, &requested, @@ -4988,7 +4906,7 @@ mod tests { orchestrator .run_auction(&create_test_auction_request(), &context) .await - .expect("should complete mediated auction"); + .expect("should complete ad server auction"); assert!(predicted.lock().expect("should lock predicted").is_empty()); assert_eq!(*requested.lock().expect("should lock requested"), vec![500]); @@ -5011,12 +4929,12 @@ mod tests { ); let bidder_predicted = Arc::new(Mutex::new(Vec::new())); let bidder_requested = Arc::new(Mutex::new(Vec::new())); - let mediator_predicted = Arc::new(Mutex::new(Vec::new())); - let mediator_requested = Arc::new(Mutex::new(Vec::new())); + let adserver_predicted = Arc::new(Mutex::new(Vec::new())); + let adserver_requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), + provider_names: vec!["bidder".to_string()], + adserver_name: Some("adserver".to_string()), timeout_ms: 2000, ..Default::default() }); @@ -5028,11 +4946,11 @@ mod tests { &bidder_requested, ))); orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", + "adserver", + "adserver-backend", 2000, - &mediator_predicted, - &mediator_requested, + &adserver_predicted, + &adserver_requested, ))); let settings = create_test_settings(); let downstream = http::Request::new(edgezero_core::body::Body::empty()); @@ -5057,20 +4975,20 @@ mod tests { vec![500] ); assert!( - mediator_predicted + adserver_predicted .lock() .expect("should lock predicted") .is_empty() ); assert_eq!( - *mediator_requested.lock().expect("should lock requested"), + *adserver_requested.lock().expect("should lock requested"), vec![500] ); }); } #[test] - fn planned_collect_skips_mediator_with_zero_canonical_transport_budget() { + fn planned_collect_skips_adserver_with_zero_canonical_transport_budget() { futures::executor::block_on(async { let calls = Arc::new(Mutex::new(Vec::new())); let services = build_services_with_backend_and_http_client( @@ -5082,18 +5000,24 @@ mod tests { ); let launches = Arc::new(AtomicUsize::new(0)); let budgets = Arc::new(Mutex::new(Vec::new())); - let plan = AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 49, - providers: BTreeMap::new(), - bidders: BTreeMap::new(), - mediator: Some("adserver_mock".to_string()), - request_signing: None, - }) - .expect("should compile mediator-only plan") - .with_enabled(true); + let mut adserver_only = plan_config(Vec::new()); + adserver_only.timeout_ms = 49; + adserver_only.adserver = crate::provider_table::ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + serde_json::Map::from_iter([( + "endpoint".to_string(), + serde_json::json!("https://adserver.example/mediate"), + )]), + )]), + ); + let plan = AuctionPlan::compile(adserver_only) + .expect("should compile adserver-only plan") + .with_enabled(true); let orchestrator = AuctionOrchestrator::from_plan( Arc::new(plan), - Some(Arc::new(DeadlineRecordingMediator { + Some(Arc::new(DeadlineRecordingAdServer { launches: Arc::clone(&launches), budgets: Some(Arc::clone(&budgets)), })), @@ -5119,11 +5043,11 @@ mod tests { assert!( budgets .lock() - .expect("should lock mediator budgets") + .expect("should lock adserver budgets") .is_empty() ); assert_eq!(calls.lock().expect("should lock calls").len(), 1); - assert!(result.mediator_response.is_none()); + assert!(result.adserver_response.is_none()); }); } @@ -5135,12 +5059,12 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a"]), + provider_names: vec!["provider_a".to_string()], timeout_ms: 2000, ..Default::default() }); orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", + name: "provider_a", predicted: "predicted-backend", resolved: "resolved-backend", })); @@ -5159,7 +5083,7 @@ mod tests { .await; assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success + response.provider == "provider_a" && response.status == BidStatus::Success })); }); } @@ -5173,17 +5097,17 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, ..Default::default() }); orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", + name: "provider_a", predicted: "predicted-a", resolved: "shared-resolved", })); orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-b", + name: "provider_b", predicted: "predicted-b", resolved: "shared-resolved", })); @@ -5202,10 +5126,10 @@ mod tests { .await; assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success + response.provider == "provider_a" && response.status == BidStatus::Success })); assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-b" && response.status == BidStatus::Error + response.provider == "provider_b" && response.status == BidStatus::Error })); }); } @@ -5228,18 +5152,18 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "backend-b", })); @@ -5275,12 +5199,12 @@ mod tests { let provider_a = result .provider_responses .iter() - .find(|r| r.provider == "provider-a") + .find(|r| r.provider == "provider_a") .expect("should have provider-a response"); let provider_b = result .provider_responses .iter() - .find(|r| r.provider == "provider-b") + .find(|r| r.provider == "provider_b") .expect("should have provider-b response"); assert_eq!( @@ -5307,7 +5231,7 @@ mod tests { ); let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"), @@ -5357,7 +5281,7 @@ mod tests { ); let plan = Arc::new( AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"), @@ -5404,8 +5328,8 @@ mod tests { let plan = Arc::new( AuctionPlan::compile(planned_config( &[ - ("provider-b", RoutingMode::AllEligible), - ("provider-a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), ], false, )) @@ -5441,8 +5365,8 @@ mod tests { .iter() .map(|response| response.provider.as_str()) .collect::>(), - vec!["provider-a", "provider-b"], - "outer select errors should retain deterministic plan order" + vec!["provider_b", "provider_a"], + "outer select errors should keep the order the deployment selected" ); for response in &result.provider_responses { assert_eq!(response.status, BidStatus::Error); @@ -5462,14 +5386,14 @@ mod tests { let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a"]), + provider_names: vec!["provider_a".to_string()], timeout_ms: 750, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); let request = create_test_auction_request(); @@ -5542,18 +5466,18 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "backend-b", })); @@ -5608,18 +5532,18 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), + provider_names: vec!["provider_a".to_string(), "provider_b".to_string()], timeout_ms: 2000, - mediator: None, + adserver_name: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", + name: "provider_a", backend: "backend-a", })); orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", + name: "provider_b", backend: "backend-b", })); @@ -5673,13 +5597,13 @@ mod tests { Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); - let mut config = planned_config(&[("provider-a", RoutingMode::AllEligible)], false); + let mut config = planned_config(&[("provider_a", RoutingMode::AllEligible)], false); config.bidders.insert( "routed-bidder" .parse() .expect("should parse fictional bidder ID"), crate::auction::plan::BidderRouteConfig { - provider: "provider-a" + module: "provider_a" .parse() .expect("should parse fictional provider ID"), }, @@ -5725,7 +5649,7 @@ mod tests { assert_eq!(http.recorded_backend_names().len(), 1); assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); assert_eq!(result.provider_responses.len(), 1); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!(result.provider_responses[0].status, BidStatus::Success); assert_eq!( result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], @@ -5743,7 +5667,7 @@ mod tests { } #[tokio::test] - async fn planned_executor_invokes_immediate_mediator_and_applies_floor() { + async fn planned_executor_invokes_immediate_adserver_and_applies_floor() { let http = Arc::new(StubHttpClient::new()); http.push_response( 200, @@ -5761,11 +5685,11 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateMediator))); + let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateAdServer))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -5781,14 +5705,14 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned mediation"); + .expect("should execute the planned ad server"); assert_eq!( result - .mediator_response + .adserver_response .as_ref() .map(|response| response.provider.as_str()), - Some("immediate-mediator") + Some("immediate_adserver") ); assert_eq!( result.winning_bids["header-banner"].nurl.as_deref(), @@ -5796,11 +5720,11 @@ mod tests { ); assert!( !result.winning_bids.contains_key("fictional-slot"), - "mediator output owns final selection" + "adserver output owns final selection" ); } - async fn planned_pending_mediator_deadline_result( + async fn planned_pending_adserver_deadline_result( enforceable_total_request_deadline: bool, ) -> OrchestrationResult { let http = Arc::new(StubHttpClient::new()); @@ -5824,12 +5748,12 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineMediator))); + AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineAdServer))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -5845,21 +5769,21 @@ mod tests { orchestrator .run_auction(&request, &context) .await - .expect("should execute planned pending mediator") + .expect("should execute planned pending adserver") } #[tokio::test] - async fn planned_pending_mediator_applies_explicit_hard_deadline_policy() { - let current = planned_pending_mediator_deadline_result(false).await; - let current_mediator = current - .mediator_response + async fn planned_pending_adserver_applies_explicit_hard_deadline_policy() { + let current = planned_pending_adserver_deadline_result(false).await; + let current_adserver = current + .adserver_response .as_ref() - .expect("current adapters should accept completed late mediator responses"); - assert!(current_mediator.response_time_ms >= 50); - assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + .expect("current adapters should accept completed late adserver responses"); + assert!(current_adserver.response_time_ms >= 50); + assert_eq!(current.winning_bids["slot-1"].bidder, "adserver"); - let hard = planned_pending_mediator_deadline_result(true).await; - assert!(hard.mediator_response.is_none()); + let hard = planned_pending_adserver_deadline_result(true).await; + assert!(hard.adserver_response.is_none()); assert_eq!( hard.winning_bids["fictional-slot"].bid_id.as_deref(), Some("provider") @@ -5868,7 +5792,7 @@ mod tests { } #[tokio::test] - async fn planned_executor_mediator_transport_failure_falls_back_locally() { + async fn planned_executor_adserver_transport_failure_falls_back_locally() { let http = Arc::new(StubHttpClient::new()); http.push_response( 200, @@ -5889,12 +5813,12 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], + &[("provider_a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringMediator))); + AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringAdServer))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -5910,9 +5834,9 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should fall back from mediator transport failure"); + .expect("should fall back from adserver transport failure"); - assert!(result.mediator_response.is_none()); + assert!(result.adserver_response.is_none()); assert_eq!( result.winning_bids["fictional-slot"].bid_id.as_deref(), Some("provider") @@ -5920,10 +5844,11 @@ mod tests { } #[tokio::test] - async fn planned_prebid_stored_intent_filters_wire_demand_and_skips_empty_transports() { + async fn planned_stored_intent_filters_wire_demand_and_skips_empty_transports() { for inline_providers in 0..=2 { let http = Arc::new(StubHttpClient::new()); - // Providers launch in ID order: APS, then only PBS instances with usable demand. + // Providers launch in ID order: the source that takes every slot, then + // only the stored-request sources with usable demand. http.push_response(204, Vec::new()); for index in 0..inline_providers { http.push_response( @@ -5940,7 +5865,7 @@ mod tests { } ]}] })) - .expect("should serialize PBS response"), + .expect("should serialize the upstream response"), ); } let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); @@ -5948,24 +5873,45 @@ mod tests { Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); - let mut config = planned_prebid_config(&[ + let mut config = planned_fixture_config(&[ ( - "pbs-a", + "stored_a", serde_json::json!({}), NotificationConfig::default(), ), ( - "pbs-b", + "stored_b", serde_json::json!({}), NotificationConfig::default(), ), ]); - config.providers.extend(planned_aps_config().providers); - for (bidder, provider) in [("alpha", "pbs-a"), ("beta", "pbs-b")] { + let all_eligible = + planned_config(&[("all_eligible_source", RoutingMode::AllEligible)], false); + let mut tables = config.demand.tables().clone(); + tables.extend(all_eligible.demand.tables().clone()); + let mut selected = config + .demand + .selected() + .into_iter() + .map(str::to_owned) + .collect::>(); + selected.extend( + all_eligible + .demand + .selected() + .into_iter() + .map(str::to_owned), + ); + // Sources launch in the order they are selected, and this test reads + // the all-eligible source's request first, so the selection is in + // name order. + selected.sort(); + config.demand = crate::provider_table::ProviderList::new(selected, tables); + for (bidder, provider) in [("alpha", "stored_a"), ("beta", "stored_b")] { config.bidders.insert( bidder.parse().expect("should parse bidder"), crate::auction::plan::BidderRouteConfig { - provider: provider.parse().expect("should parse provider"), + module: provider.parse().expect("should parse provider"), }, ); } @@ -5973,7 +5919,7 @@ mod tests { let orchestrator = AuctionOrchestratorHarness::new(plan, None); let mut request = planned_request(); let template = request.slots[0].clone(); - // Both PBS instances must evaluate candidates but omit them after overrides. + // Both stored-request sources must evaluate candidates and omit them. request.slots[0].bidders.insert( "trustedServer".to_string(), serde_json::json!({ @@ -5981,7 +5927,7 @@ mod tests { }), ); request.slots.push(AdSlot { - id: "synthetic-no-pbs".to_string(), + id: "synthetic-no-demand".to_string(), bidders: HashMap::from([( "trustedServer".to_string(), serde_json::json!({"storedRequest":false,"bidderParams":{}}), @@ -6017,12 +5963,12 @@ mod tests { assert_eq!( bodies.len(), 1 + inline_providers, - "should never transport an empty PBS request" + "should never transport an empty request" ); - let aps: serde_json::Value = - serde_json::from_slice(&bodies[0]).expect("should parse APS wire request"); + let all_eligible_wire: serde_json::Value = serde_json::from_slice(&bodies[0]) + .expect("should parse the all-eligible source's wire request"); assert_eq!( - aps["imp"] + all_eligible_wire["imp"] .as_array() .expect("should have impressions") .len(), @@ -6034,12 +5980,12 @@ mod tests { "should reject a bid for an omitted impression" ); for index in inline_providers..2 { - let provider_id = if index == 0 { "pbs-a" } else { "pbs-b" }; + let provider_id = if index == 0 { "stored_a" } else { "stored_b" }; let response = result .provider_responses .iter() .find(|response| response.provider == provider_id) - .expect("should retain skipped PBS provider response"); + .expect("should retain the skipped source's response"); assert_eq!( response.metadata["routing"]["skipped_no_usable_demand"], true @@ -6047,7 +5993,7 @@ mod tests { } for index in 0..inline_providers { let wire: serde_json::Value = serde_json::from_slice(&bodies[index + 1]) - .expect("should parse PBS wire request"); + .expect("should parse the source's wire request"); assert_eq!( wire["imp"] .as_array() @@ -6056,11 +6002,11 @@ mod tests { 1 ); assert_eq!(wire["imp"][0]["id"], format!("inline-{index}")); - let prebid = &wire["imp"][0]["ext"]["prebid"]; - assert!(prebid.get("storedrequest").is_none()); + let fixture = &wire["imp"][0]["ext"]["fixture"]; + assert!(fixture.get("stored").is_none()); let bidder = if index == 0 { "alpha" } else { "beta" }; assert_eq!( - prebid["bidder"], + fixture["bidder"], serde_json::json!({bidder:{"placement":index}}) ); assert_eq!( @@ -6069,12 +6015,12 @@ mod tests { .as_deref(), Some(format!("bid-{index}").as_str()) ); - let provider_id = if index == 0 { "pbs-a" } else { "pbs-b" }; + let provider_id = if index == 0 { "stored_a" } else { "stored_b" }; let response = result .provider_responses .iter() .find(|response| response.provider == provider_id) - .expect("should retain PBS provider response"); + .expect("should retain the source's response"); assert_eq!( response.metadata["response_admission"]["rejected_bid_count"], 1 @@ -6088,59 +6034,119 @@ mod tests { } #[tokio::test] - async fn planned_prebid_instances_preserve_headers_metadata_suppression_and_identity() { + async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { + let standard_plan = AuctionPlan::compile(planned_config( + &[("standard", RoutingMode::AllEligible)], + false, + )) + .expect("should compile standard plan"); + let fixture_plan = AuctionPlan::compile(planned_fixture_config(&[( + "stored", + serde_json::json!({}), + NotificationConfig::default(), + )])) + .expect("should compile the stand-in plan"); + let cases = [ + (&standard_plan, 204, Vec::new(), BidStatus::NoBid), + (&standard_plan, 502, Vec::new(), BidStatus::Error), + (&standard_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &standard_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + (&fixture_plan, 204, b"{}".to_vec(), BidStatus::NoBid), + (&fixture_plan, 502, Vec::new(), BidStatus::Error), + (&fixture_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &fixture_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + ]; + + for (plan, status, body, expected) in cases { + let routed = route_auction( + planned_envelope_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build provider matrix response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should classify provider matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if expected == BidStatus::Error { + let expected_error_type = if (200..300).contains(&status) { + "parse_response" + } else { + "http_status" + }; + assert_eq!(parsed.metadata["error_type"], expected_error_type); + } + assert_eq!( + parsed.metadata["routing"], + serde_json::json!({"unused_bidder_params_count": 0}) + ); + let serialized = serde_json::to_string(&parsed.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!serialized.contains("fictional_provider")); + assert!(!serialized.contains("fictional-slot")); + } + } + + #[tokio::test] + async fn planned_standard_instances_have_distinct_backends_and_independent_results() { let http = Arc::new(StubHttpClient::new()); http.push_response( 200, serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "suppress-exact", "bid": [ - {"id":"good-a","impid":"fictional-slot","price":1.25,"adm":"
a
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill","ext":{"prebid":{"cache":{"bids":{"cacheId":"cache-a","url":"https://cache-a.example/cache/path"}}}}}, - {"id":"bad-a","price":2.0} - ]}], - "ext": {"responsetimemillis":{"suppress-exact":4},"errors":{"other":["fictional"]},"warnings":{"other":["warning"]},"debug":{"httpcalls":[]},"prebid":{"bidstatus":{"suppress-exact":[{"bidid":"good-a"}]}}} + "seatbid": [{"seat": "fictional-seat-a", "bid": [{ + "id": "bid-a", "impid": "fictional-slot", "price": 1.25, + "adm": "
a
", "w": 300, "h": 250 + }]}] })) - .expect("should serialize PBS response a"), + .expect("should serialize response a"), ); http.push_response( 200, serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "keep-seat", "bid": [{ - "id":"good-b","impid":"fictional-slot","price":2.5,"adm":"
b
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill" + "seatbid": [{"seat": "fictional-seat-b", "bid": [{ + "id": "bid-b", "impid": "fictional-slot", "price": 2.5, + "adm": "
b
", "w": 300, "h": 250 }]}] })) - .expect("should serialize PBS response b"), + .expect("should serialize response b"), ); let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); - let notifications = NotificationConfig { - suppress_all: false, - suppress_seats: vec!["suppress-exact".to_string()], - }; - let plan = AuctionPlan::compile(planned_prebid_config(&[ - ( - "pbs-a", - serde_json::json!({"debug":true,"test_mode":true,"consent_forwarding":"openrtb_only"}), - notifications, - ), - ("pbs-b", serde_json::json!({}), NotificationConfig::default()), - ])) - .expect("should compile planned PBS auction"); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_prebid_request(); + let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::builder() .uri("https://publisher.example/auction") - .header( - http::header::COOKIE, - "consent=keep; euconsent-v2=drop; other=value", - ) - .header(http::header::USER_AGENT, "Fictional Browser/7") - .header(http::header::REFERER, "https://referrer.example/story") - .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") - .header("x-forwarded-for", "203.0.113.250") .body(edgezero_core::body::Body::empty()) .expect("should build inbound request"); let context = AuctionContext { @@ -6155,830 +6161,75 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned PBS auction"); + .expect("should execute planned auction"); + assert_eq!(orchestrator.provider_count(), 2); + assert!(orchestrator.adserver().is_none()); assert_eq!(result.provider_responses.len(), 2); - let first = &result.provider_responses[0]; - assert_eq!(first.provider, "pbs-a"); - assert_eq!(first.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); assert_eq!( - first.bids[0].returned_seat.as_deref(), - Some("suppress-exact") + result.provider_responses[0].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-a") ); - assert_eq!(first.bids[0].bidder, "suppress-exact"); - assert!( - first.bids[0].nurl.is_none(), - "should suppress after normalization" + assert_eq!( + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 0 ); - assert!( - first.bids[0].burl.is_none(), - "should suppress billing notification" - ); - assert_eq!(first.bids[0].cache_id.as_deref(), Some("cache-a")); - assert_eq!(first.bids[0].cache_host.as_deref(), Some("cache-a.example")); - assert_eq!(first.bids[0].cache_path.as_deref(), Some("/cache/path")); - assert_eq!(first.metadata["responsetimemillis"]["suppress-exact"], 4); - assert!(first.metadata.contains_key("errors")); - assert!(first.metadata.contains_key("warnings")); - assert!(first.metadata.contains_key("debug")); - assert!(first.metadata.contains_key("bidstatus")); - let second = &result.provider_responses[1]; - assert_eq!(second.provider, "pbs-b"); - assert_eq!(second.bids[0].returned_seat.as_deref(), Some("keep-seat")); - assert!(second.bids[0].nurl.is_some()); - assert!(!second.metadata.contains_key("debug")); - assert!(!second.metadata.contains_key("bidstatus")); - - let headers = http.recorded_request_headers(); - assert_eq!(headers.len(), 2); - for request_headers in &headers { - assert!( - request_headers - .iter() - .any(|(name, value)| name == "user-agent" && value == "Fictional Browser/7") - ); - assert!(request_headers.iter().any( - |(name, value)| name == "referer" && value == "https://referrer.example/story" - )); - assert!( - request_headers - .iter() - .any(|(name, value)| name == "accept-language" && value == "en-US,en;q=0.9") - ); + assert_eq!(result.provider_responses[1].provider, "provider_b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!( + result.provider_responses[1].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-b") + ); + assert_eq!( + result.provider_responses[1].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b") + ); + let request_headers = http.recorded_request_headers(); + assert_eq!(request_headers.len(), 2); + for headers in request_headers { assert!( - request_headers + headers .iter() - .all(|(name, _)| name != "x-forwarded-for"), - "must ignore inbound XFF without attestation" + .any(|(name, value)| name == "accept" && value == "application/json"), + "standard planned transport should retain its JSON Accept header" ); + } + let backend_names = http.recorded_backend_names(); + assert_eq!(backend_names.len(), 2); + assert_ne!(backend_names[0], backend_names[1]); + let request_bodies = http.recorded_request_bodies(); + assert_eq!(request_bodies.len(), 2); + for body in request_bodies { + let value: serde_json::Value = + serde_json::from_slice(&body).expect("should parse planned request"); + let tmax = value["tmax"].as_u64().expect("should include logical tmax"); assert!( - request_headers.iter().all(|(name, _)| name != "accept"), - "planned PBS transport must not add Accept beyond legacy headers" + (750..=777).contains(&tmax), + "logical budget should remain near the auction budget, got {tmax}" ); + assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); } - let first_cookie = headers[0] - .iter() - .find(|(name, _)| name == "cookie") - .map(|(_, value)| value.as_str()); - assert_eq!(first_cookie, Some("consent=keep; other=value")); - let second_cookie = headers[1] - .iter() - .find(|(name, _)| name == "cookie") - .map(|(_, value)| value.as_str()); - assert_eq!( - second_cookie, - Some("consent=keep; euconsent-v2=drop; other=value") - ); - } - - #[tokio::test] - async fn planned_aps_mock_mediation_preserves_three_identities_and_renderer() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "upstream-seat", "bid": [{ - "id": "aps-bid", "impid": "fictional-slot", "price": 2.0, - "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"} - }]}] - })) - .expect("should serialize APS response"), - ); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "aps-instance", "bid": [{ - "id": "mediated-aps", "impid": "fictional-slot", "price": 2.0, - "adm": "ignored", "w": 300, "h": 250, "crid": "aps-creative" - }]}] - })) - .expect("should serialize mediator response"), - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = - AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let mediator = AdServerMockProvider::new(AdServerMockConfig { - enabled: true, - endpoint: "https://mediator.example/mediate".to_string(), - timeout_ms: 500, - ..AdServerMockConfig::default() - }); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(mediator))); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should mediate planned APS bid"); - - let provider_bid = &result.provider_responses[0].bids[0]; - assert_eq!(result.provider_responses[0].provider, "aps-instance"); - assert_eq!(provider_bid.returned_seat.as_deref(), Some("upstream-seat")); - assert_eq!(provider_bid.bidder, "aps"); - let winner = &result.winning_bids["fictional-slot"]; - assert_eq!(winner.returned_seat.as_deref(), Some("upstream-seat")); - assert_eq!(winner.bidder, "aps"); - assert!(winner.renderer.is_some()); - assert!(winner.creative.is_none()); - assert_eq!( - result - .mediator_response - .as_ref() - .map(|response| response.provider.as_str()), - Some("adserver_mock") - ); - } - - #[tokio::test] - async fn planned_aps_transport_omits_accept_header() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(400, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = - AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::builder() - .uri("https://publisher.example/auction") - .body(edgezero_core::body::Body::empty()) - .expect("should build inbound request"); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned APS auction"); - - let headers = http.recorded_request_headers(); - assert_eq!(headers.len(), 1); - assert!( - headers[0].iter().all(|(name, _)| name != "accept"), - "planned APS transport must not add Accept beyond legacy headers" - ); - } - - #[tokio::test] - async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( - 200, - serde_json::to_vec(&serde_json::json!({ - "cur": "USD", - "seatbid": [ - {"seat": "returned-seat", "bid": [ - {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, - "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", - "crid": "fictional-creative", "adomain": ["advertiser.example"], - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, - {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, - {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, - {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "crid": "x".repeat(1025), - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, - {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, - "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} - ]}, - {"seat": 7, "bid": "bad-shape"} - ] - })) - .expect("should serialize APS profile response"), - vec![ - ("content-type", "application/json"), - ("authorization", "fictional-secret"), - ], - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({"account_id": "example-account", "debug": true}), - NotificationConfig { - suppress_all: false, - suppress_seats: vec!["returned-seat".to_string()], - }, - )])) - .expect("should compile planned APS profile"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned APS profile"); - - let response = &result.provider_responses[0]; - assert_eq!(response.provider, "aps-instance"); - assert_eq!(response.status, BidStatus::Success); - assert_eq!( - response.bids.len(), - 1, - "should retain one bid per impression" - ); - let bid = &response.bids[0]; - assert_eq!(bid.bidder, "aps"); - assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); - assert_eq!( - bid.bid_id.as_deref(), - Some("a-high"), - "lexical ID should break equal-price tie" - ); - assert!(bid.creative.is_none()); - assert!( - bid.nurl.is_none() && bid.burl.is_none(), - "APS must discard notification URLs" - ); - let renderer = bid - .renderer - .as_ref() - .and_then(BidRenderer::as_aps) - .expect("should construct typed APS renderer"); - assert_eq!(renderer.account_id, "example-account"); - let decoded = base64::engine::general_purpose::STANDARD - .decode(&renderer.aax_response) - .expect("should decode minimized APS response"); - assert_eq!( - serde_json::from_slice::(&decoded) - .expect("should parse minimized APS response"), - serde_json::json!({"seatbid":[{"bid":[{ - "id":"a-high","price":2.0,"w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]}) - ); - assert_eq!(response.metadata["seatbid_count"], 2); - assert_eq!(response.metadata["accepted_bid_count"], 1); - assert_eq!(response.metadata["dropped_bid_count"], 12); - for reason in [ - "lost_to_higher_bid", - "script_rendering_disabled", - "unknown_impid", - "invalid_dimensions", - "invalid_price", - "unsupported_media_type", - "unsupported_tagtype", - "creative_id_too_large", - "missing_render_source", - "empty_seatbid_bids", - ] { - assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); - } - assert_eq!( - response.metadata["drop_reasons"]["invalid_creative_url"], 2, - "same-publisher and credentialed URLs should both be rejected" - ); - assert_eq!( - response.metadata["routing"]["unused_bidder_params_count"], - 0 - ); - let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; - assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); - assert_eq!( - debug["responseheaders"], - serde_json::json!({"content-type": ["application/json"]}), - "async stub should preserve allowlisted response headers" - ); - assert!( - debug["requestbody"] - .as_str() - .is_some_and(|body| body.contains("example-account")) - ); - assert!(debug["requestheaders"].get("authorization").is_none()); - assert!(debug["responseheaders"].get("authorization").is_none()); - } - - #[tokio::test] - async fn two_planned_aps_instances_correlate_independently() { - let http = Arc::new(StubHttpClient::new()); - for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ - "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]})) - .expect("should serialize APS instance response"), - ); - } - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_aps_instances_config(&[ - ( - "aps-a", - serde_json::json!({"account_id":"account-a"}), - NotificationConfig::default(), - ), - ( - "aps-b", - serde_json::json!({"account_id":"account-b"}), - NotificationConfig::default(), - ), - ])) - .expect("should compile two APS instances"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute two APS instances"); - - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "aps-a"); - assert_eq!( - result.provider_responses[0].bids[0].bid_id.as_deref(), - Some("bid-a") - ); - assert_eq!(result.provider_responses[1].provider, "aps-b"); - assert_eq!( - result.provider_responses[1].bids[0].bid_id.as_deref(), - Some("bid-b") - ); - assert_eq!(http.recorded_request_bodies().len(), 2); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("bid-b"), - "global ranking should remain orchestrator-owned" - ); - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_ne!(specs[0].discriminator, specs[1].discriminator); - } - - #[tokio::test] - async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { - let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - for (seat, expected) in [ - (serde_json::Value::Null, None), - (serde_json::json!(7), None), - (serde_json::json!(""), None), - (serde_json::json!("exact-seat"), Some("exact-seat")), - ] { - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::from( - serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ - "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, - "nurl":"https://notice.example/win","burl":"https://notice.example/bill", - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - }]}]})) - .expect("should serialize seat identity response"), - )) - .expect("should build seat identity response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should parse seat identity response"); - assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); - assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); - } - } - - #[tokio::test] - async fn planned_aps_response_status_shape_and_currency_matrix() { - let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let cases = [ - (204, Vec::new(), BidStatus::NoBid, None, None), - (400, Vec::new(), BidStatus::Error, None, Some("http_status")), - ( - 200, - b"not-json".to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - b"[]".to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - br#"{"contextual":true}"#.to_vec(), - BidStatus::Error, - Some("unexpected_response_shape"), - Some("parse_response"), - ), - ( - 200, - br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - Some("unsupported_currency"), - None, - ), - ]; - for (status, body, expected, reason, error_type) in cases { - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(edgezero_core::body::Body::from(body)) - .expect("should build APS matrix response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should classify APS matrix response"); - assert_eq!(parsed.status, expected, "status {status}"); - if let Some(reason) = reason { - assert_eq!( - parsed.metadata["drop_reasons"][reason], 1, - "status {status}" - ); - } - if let Some(error_type) = error_type { - assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); - } - } - } - - #[tokio::test] - async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { - let standard_plan = AuctionPlan::compile(planned_config( - &[("standard", RoutingMode::AllEligible)], - false, - )) - .expect("should compile standard plan"); - let prebid_plan = AuctionPlan::compile(planned_prebid_config(&[( - "pbs", - serde_json::json!({}), - NotificationConfig::default(), - )])) - .expect("should compile PBS plan"); - let cases = [ - (&standard_plan, 204, Vec::new(), BidStatus::NoBid), - (&standard_plan, 502, Vec::new(), BidStatus::Error), - (&standard_plan, 200, b"not-json".to_vec(), BidStatus::Error), - ( - &standard_plan, - 200, - br#"{"seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - ), - (&prebid_plan, 204, b"{}".to_vec(), BidStatus::NoBid), - (&prebid_plan, 502, Vec::new(), BidStatus::Error), - (&prebid_plan, 200, b"not-json".to_vec(), BidStatus::Error), - ( - &prebid_plan, - 200, - br#"{"seatbid":[]}"#.to_vec(), - BidStatus::NoBid, - ), - ]; - - for (plan, status, body, expected) in cases { - let routed = route_auction( - planned_prebid_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(edgezero_core::body::Body::from(body)) - .expect("should build provider matrix response"), - ); - let parsed = provider - .parse_response_with_state(response, 4, Some(state.as_ref())) - .await - .expect("should classify provider matrix response"); - assert_eq!(parsed.status, expected, "status {status}"); - if expected == BidStatus::Error { - let expected_error_type = if (200..300).contains(&status) { - "parse_response" - } else { - "http_status" - }; - assert_eq!(parsed.metadata["error_type"], expected_error_type); - } - assert_eq!( - parsed.metadata["routing"], - serde_json::json!({"unused_bidder_params_count": 0}) - ); - let serialized = serde_json::to_string(&parsed.metadata["routing"]) - .expect("should serialize routing metadata"); - assert!(!serialized.contains("fictional-provider")); - assert!(!serialized.contains("fictional-slot")); - } - } - - #[tokio::test] - async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({ - "account_id":"example-account-id", - "allow_script_creatives":true - }), - NotificationConfig::default(), - )])) - .expect("should compile script-enabled APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::from( - serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ - "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, - "w":300,"h":250,"crid":"fictional-creative", - "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} - },{ - "id":"script-bid","impid":"fictional-slot","price":1.0, - "w":300,"h":250, - "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} - }]}]})) - .expect("should serialize APS renderer fixture response"), - )) - .expect("should build APS renderer fixture response"), - ); - - let parsed = provider - .parse_response_with_state(response, 3, Some(state.as_ref())) - .await - .expect("should parse APS renderer fixture response"); - - assert_eq!(parsed.status, BidStatus::Success); - assert_eq!( - parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, - "enabled script creative should be eligible before reduction" - ); - let renderer = parsed.bids[0] - .renderer - .as_ref() - .and_then(BidRenderer::as_aps) - .expect("should construct APS renderer"); - let decoded = base64::engine::general_purpose::STANDARD - .decode(&renderer.aax_response) - .expect("should decode APS fixture envelope"); - let fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" - )) - .expect("should parse shared APS renderer fixture"); - assert_eq!( - serde_json::from_slice::(&decoded) - .expect("should parse decoded APS renderer"), - fixture - ); - } - - #[tokio::test] - async fn planned_aps_debug_response_headers_are_allowlisted() { - let plan = AuctionPlan::compile(planned_aps_instances_config(&[( - "aps-instance", - serde_json::json!({"account_id":"example-account","debug":true}), - NotificationConfig::default(), - )])) - .expect("should compile debug APS plan"); - let routed = route_auction( - planned_request(), - &http::Request::new(edgezero_core::body::Body::empty()), - &plan, - None, - ); - let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); - let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(200) - .header("content-type", "application/json") - .header("authorization", "fictional-secret") - .body(edgezero_core::body::Body::from("{}")) - .expect("should build debug APS response"), - ); - - let parsed = provider - .parse_response_with_state(response, 3, Some(state.as_ref())) - .await - .expect("should parse debug APS response"); - - let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; - assert_eq!( - headers, - &serde_json::json!({"content-type":["application/json"]}) - ); - assert!(headers.get("authorization").is_none()); - } - - #[tokio::test] - async fn planned_standard_instances_have_distinct_backends_and_independent_results() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "fictional-seat-a", "bid": [{ - "id": "bid-a", "impid": "fictional-slot", "price": 1.25, - "adm": "
a
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize response a"), - ); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "fictional-seat-b", "bid": [{ - "id": "bid-b", "impid": "fictional-slot", "price": 2.5, - "adm": "
b
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize response b"), - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::builder() - .uri("https://publisher.example/auction") - .body(edgezero_core::body::Body::empty()) - .expect("should build inbound request"); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned auction"); - - assert_eq!(orchestrator.provider_count(), 2); - assert!(orchestrator.mediator().is_none()); - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); - assert_eq!( - result.provider_responses[0].bids[0].bid_id.as_deref(), - Some("bid-a") - ); - assert_eq!( - result.provider_responses[0].bids[0] - .returned_seat - .as_deref(), - Some("fictional-seat-a") - ); - assert_eq!( - result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], - 0 - ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); - assert_eq!( - result.provider_responses[1].bids[0].bid_id.as_deref(), - Some("bid-b") - ); - assert_eq!( - result.provider_responses[1].bids[0] - .returned_seat - .as_deref(), - Some("fictional-seat-b") - ); - assert_eq!( - result.provider_responses[1].metadata["routing"]["unused_bidder_params_count"], - 0 - ); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("bid-b") - ); - let request_headers = http.recorded_request_headers(); - assert_eq!(request_headers.len(), 2); - for headers in request_headers { - assert!( - headers - .iter() - .any(|(name, value)| name == "accept" && value == "application/json"), - "standard planned transport should retain its JSON Accept header" - ); - } - let backend_names = http.recorded_backend_names(); - assert_eq!(backend_names.len(), 2); - assert_ne!(backend_names[0], backend_names[1]); - let request_bodies = http.recorded_request_bodies(); - assert_eq!(request_bodies.len(), 2); - for body in request_bodies { - let value: serde_json::Value = - serde_json::from_slice(&body).expect("should parse planned request"); - let tmax = value["tmax"].as_u64().expect("should include logical tmax"); - assert!( - (750..=777).contains(&tmax), - "logical budget should remain near the auction budget, got {tmax}" - ); - assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); - } - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); - assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); - assert_ne!(specs[0].discriminator, specs[1].discriminator); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); + assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); + assert_ne!(specs[0].discriminator, specs[1].discriminator); } #[tokio::test] @@ -7001,8 +6252,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7026,12 +6277,12 @@ mod tests { .expect("should isolate backend collision"); assert_eq!(http.recorded_backend_names().len(), 1); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!( result.provider_responses[0].bids[0].bid_id.as_deref(), Some("first-bid") ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].provider, "provider_b"); assert_eq!( result.provider_responses[1].metadata["error_type"], "launch_failed" @@ -7051,8 +6302,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7076,12 +6327,12 @@ mod tests { .expect("should isolate divergent pending backend"); assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!( result.provider_responses[0].metadata["error_type"], "launch_failed" ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].provider, "provider_b"); assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); } @@ -7098,8 +6349,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7123,12 +6374,12 @@ mod tests { .expect("should isolate missing pending backend"); assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].provider, "provider_a"); assert_eq!( result.provider_responses[0].metadata["error_type"], "launch_failed" ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].provider, "provider_b"); assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); } @@ -7136,8 +6387,8 @@ mod tests { async fn planned_same_profile_rejects_cross_provider_parse_state() { let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7164,28 +6415,28 @@ mod tests { .expect_err("should reject another provider's parse state"); assert!( - error.to_string().contains("owned by provider provider-a"), + error.to_string().contains("owned by provider provider_a"), "should identify cross-provider state ownership" ); } #[tokio::test] - async fn planned_prebid_rejects_cross_provider_parse_state() { - let plan = AuctionPlan::compile(planned_prebid_config(&[ + async fn planned_source_rejects_cross_provider_parse_state() { + let plan = AuctionPlan::compile(planned_fixture_config(&[ ( - "pbs-a", + "stored_a", serde_json::json!({}), NotificationConfig::default(), ), ( - "pbs-b", + "stored_b", serde_json::json!({}), NotificationConfig::default(), ), ])) - .expect("should compile planned PBS auction"); + .expect("should compile the planned auction"); let routed = route_auction( - planned_prebid_request(), + planned_envelope_request(), &http::Request::new(edgezero_core::body::Body::empty()), &plan, None, @@ -7197,17 +6448,17 @@ mod tests { edgezero_core::http::response_builder() .status(200) .body(edgezero_core::body::Body::from_bytes(b"{}".as_slice())) - .expect("should build PBS response"), + .expect("should build the upstream response"), ); let error = provider_b .parse_response_with_state(response, 1, Some(parse_state.as_ref())) .await - .expect_err("should reject another PBS provider's parse state"); + .expect_err("should reject another source's parse state"); assert!( - error.to_string().contains("owned by provider pbs-a"), - "should identify cross-provider PBS state ownership" + error.to_string().contains("owned by provider stored_a"), + "should identify cross-provider state ownership" ); } @@ -7274,18 +6525,27 @@ mod tests { Arc::clone(&http) as Arc<_>, ); let mut config = planned_config(&[("provider", RoutingMode::AllEligible)], false); - config.mediator = Some("adserver_mock".to_string()); + config.adserver = crate::provider_table::ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + serde_json::Map::from_iter([( + "endpoint".to_string(), + serde_json::json!("https://adserver.example/mediate"), + )]), + )]), + ); let plan = AuctionPlan::compile(config).expect("should compile planned auction"); - let mediator_predicted = Arc::new(Mutex::new(Vec::new())); - let mediator_requested = Arc::new(Mutex::new(Vec::new())); - let mediator = Arc::new(recording_provider( + let adserver_predicted = Arc::new(Mutex::new(Vec::new())); + let adserver_requested = Arc::new(Mutex::new(Vec::new())); + let adserver = Arc::new(recording_provider( "adserver_mock", - "mediator-backend", + "adserver-backend", 777, - &mediator_predicted, - &mediator_requested, + &adserver_predicted, + &adserver_requested, )); - let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(adserver)); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -7323,15 +6583,15 @@ mod tests { assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); assert!(http.recorded_backend_names().is_empty()); assert!( - mediator_predicted + adserver_predicted .lock() - .expect("should lock mediator predictions") + .expect("should lock adserver predictions") .is_empty() ); assert!( - mediator_requested + adserver_requested .lock() - .expect("should lock mediator requests") + .expect("should lock adserver requests") .is_empty() ); } @@ -7340,12 +6600,14 @@ mod tests { #[tokio::test] async fn planned_launch_transport_parse_failures_are_isolated_from_valid_winner_and_floor() { let http = Arc::new(StubHttpClient::new()); - // BTreeMap plan order is alphabetical: below-floor, parse-fail, - // transport-fail, valid-winner. Queue responses in that exact order. + // The plan runs its sources in the order the deployment selected them, + // so queue the responses in that same order: below_floor, parse_fail, + // transport_fail, valid_winner, with launch_fail never reaching + // transport. http.push_response( 200, serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "below-floor", "bid": [{ + "seatbid": [{"seat": "below_floor", "bid": [{ "id": "below", "impid": "fictional-slot", "price": 0.5, "adm": "
below
", "w": 300, "h": 250 }, { @@ -7371,18 +6633,18 @@ mod tests { http.push_select_success(); http.push_select_error(); let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); + backend.fail_ensure_for("launch_fail"); let services = build_services_with_backend_and_http_client( Arc::clone(&backend) as Arc<_>, Arc::clone(&http) as Arc<_>, ); let plan = AuctionPlan::compile(planned_config( &[ - ("launch-fail", RoutingMode::AllEligible), - ("transport-fail", RoutingMode::AllEligible), - ("parse-fail", RoutingMode::AllEligible), - ("below-floor", RoutingMode::AllEligible), - ("valid-winner", RoutingMode::AllEligible), + ("below_floor", RoutingMode::AllEligible), + ("launch_fail", RoutingMode::AllEligible), + ("parse_fail", RoutingMode::AllEligible), + ("transport_fail", RoutingMode::AllEligible), + ("valid_winner", RoutingMode::AllEligible), ], false, )) @@ -7415,7 +6677,7 @@ mod tests { .collect::>(); assert_eq!( by_provider - .get("launch-fail") + .get("launch_fail") .unwrap_or_else(|| panic!( "should include launch-fail response; got {:?}", by_provider.keys().collect::>() @@ -7423,7 +6685,7 @@ mod tests { .metadata["error_type"], "launch_failed" ); - let below_floor = by_provider.get("below-floor").unwrap_or_else(|| { + let below_floor = by_provider.get("below_floor").unwrap_or_else(|| { panic!( "should include below-floor response; got {:?}", by_provider.keys().collect::>() @@ -7434,10 +6696,10 @@ mod tests { assert_eq!(below_floor.bids[0].price, Some(0.5)); assert_eq!(below_floor.bids[1].bid_id.as_deref(), Some("below-only")); assert_eq!( - by_provider["transport-fail"].metadata["error_type"], + by_provider["transport_fail"].metadata["error_type"], "transport" ); - let parse_failure = by_provider.get("parse-fail").unwrap_or_else(|| { + let parse_failure = by_provider.get("parse_fail").unwrap_or_else(|| { panic!( "should include parse-fail response; got {:?}", by_provider.keys().collect::>() @@ -7455,7 +6717,7 @@ mod tests { "every materialized planned provider response should have routing count" ); } - assert_eq!(by_provider["valid-winner"].status, BidStatus::Success); + assert_eq!(by_provider["valid_winner"].status, BidStatus::Success); assert_eq!( result.winning_bids["fictional-slot"].bid_id.as_deref(), Some("winner") @@ -7467,62 +6729,57 @@ mod tests { } #[tokio::test] - async fn planned_routing_count_survives_standard_and_aps_bounded_body_failures() { - for (profile, provider_id) in [("standard", "standard"), ("aps", "aps-instance")] { - let http = Arc::new(StubHttpClient::new()); - http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let mut config = if profile == "standard" { - planned_config(&[(provider_id, RoutingMode::AllEligible)], false) - } else { - planned_aps_config() - }; - config.bidders.insert( - "example-bidder" + async fn planned_routing_count_survives_a_bounded_body_failure() { + let (profile, provider_id) = ("standard", "standard"); + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[(provider_id, RoutingMode::AllEligible)], false); + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + module: provider_id .parse() - .expect("should parse fictional bidder ID"), - crate::auction::plan::BidderRouteConfig { - provider: provider_id - .parse() - .expect("should parse fictional provider ID"), - }, - ); - let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let mut request = planned_request(); - request.slots[0].bidders.insert( - "example-bidder".to_string(), - serde_json::json!({"private": "value"}), - ); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should materialize bounded-body failure"); - let response = &result.provider_responses[0]; - assert_eq!(response.status, BidStatus::Error, "{profile}"); - assert_eq!( - response.metadata["routing"]["unused_bidder_params_count"], 1, - "{profile} bounded-body failure should retain the input-derived count" - ); - let routing = serde_json::to_string(&response.metadata["routing"]) - .expect("should serialize routing metadata"); - assert!(!routing.contains("example-bidder") && !routing.contains("private")); - } + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); } #[tokio::test] @@ -7620,8 +6877,8 @@ mod tests { .build(); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], true, )) @@ -7684,7 +6941,7 @@ mod tests { .parse() .expect("should parse fictional bidder ID"), crate::auction::plan::BidderRouteConfig { - provider: "signed" + module: "signed" .parse() .expect("should parse fictional provider ID"), }, @@ -7828,11 +7085,11 @@ mod tests { )) .expect("should compile signed plan"), ); - let mediator_launches = Arc::new(AtomicUsize::new(0)); + let adserver_launches = Arc::new(AtomicUsize::new(0)); let orchestrator = AuctionOrchestrator::from_plan( plan, - Some(Arc::new(DeadlineRecordingMediator { - launches: Arc::clone(&mediator_launches), + Some(Arc::new(DeadlineRecordingAdServer { + launches: Arc::clone(&adserver_launches), budgets: None, })), ); @@ -7868,9 +7125,9 @@ mod tests { assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); assert!(http.recorded_backend_names().is_empty()); assert_eq!( - mediator_launches.load(Ordering::Relaxed), + adserver_launches.load(Ordering::Relaxed), 0, - "zero budget must not invoke even an immediate mediator" + "zero budget must not invoke even an immediate adserver" ); } @@ -7885,8 +7142,8 @@ mod tests { ); let plan = AuctionPlan::compile(planned_config( &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), + ("provider_a", RoutingMode::AllEligible), + ("provider_b", RoutingMode::AllEligible), ], false, )) @@ -7935,26 +7192,28 @@ mod tests { } #[test] - fn decoded_aps_bid_competes_directly_by_cpm() { + fn a_bid_with_a_typed_renderer_competes_directly_by_cpm() { let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); let floor_prices = HashMap::new(); let response = |provider: &str, bid: Bid| AuctionResponse::success(provider, vec![bid], 1); - let aps_wins = orchestrator.select_winning_bids( + let typed_wins = orchestrator.select_winning_bids( &[ - response("aps", auction_bid("aps", 2.0)), + response("typed", auction_bid("typed", 2.0)), response("ordinary", auction_bid("ordinary", 1.0)), ], &floor_prices, ); - let winner = aps_wins.get("slot-1").expect("should select APS bid"); - assert_eq!(winner.bidder, "aps"); + let winner = typed_wins + .get("slot-1") + .expect("should select the typed renderer bid"); + assert_eq!(winner.bidder, "typed"); assert!(winner.renderer.is_some()); assert!(winner.creative.is_none()); let ordinary_wins = orchestrator.select_winning_bids( &[ - response("aps", auction_bid("aps", 2.0)), + response("typed", auction_bid("typed", 2.0)), response("ordinary", auction_bid("ordinary", 3.0)), ], &floor_prices, diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs index df36fc09b7..49fff990f0 100644 --- a/crates/trusted-server-core/src/auction/plan.rs +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -1,37 +1,57 @@ -//! Target-independent config-first auction plan compiler. +//! Target-independent auction plan compiler. +//! +//! The plan is compiled once at startup from `[demand]`, `[ad-server]` and +//! `[auction.bidders]`. Every selected name resolves to an implementation an +//! integration registered, so the compiler names no vendor, and request +//! handling reads only the compiled plan. use std::collections::{BTreeMap, BTreeSet}; use std::str::FromStr; +use std::sync::Arc; use std::time::Duration; use error_stack::{Report, ResultExt as _}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use url::Url; +use serde_json::{Map, Value}; +use url::{Host, Url}; -use super::profile::{CompiledOpenRtbProfile, ProfileTimeoutDefault, find_profile}; +use super::demand::{ + AdServerImplementation, CompiledDemand, DemandImplementation, DemandTimeoutDefault, +}; use crate::error::TrustedServerError; use crate::platform::{AuctionTargetId, PlatformBackendSpec}; +use crate::provider_table::{ProviderChoice, ProviderList}; use crate::settings::RequestSigning; const MAX_ID_BYTES: usize = 128; const MAX_SUPPRESS_SEATS: usize = 128; const MAX_SUPPRESS_SEAT_BYTES: usize = 128; -const MOCK_MEDIATOR_ID: &str = "adserver_mock"; const RESERVED_BROWSER_ENVELOPE_BIDDER_ID: &str = "trustedServer"; -/// Validated operator-defined provider identifier. +/// The keys of a `[demand.]` table that the common driver reads itself, +/// so an implementation receives the rest. +const ENDPOINT_KEY: &str = "endpoint"; +const TIMEOUT_KEY: &str = "timeout_ms"; +const ROUTING_KEY: &str = "routing"; +const NOTIFICATIONS_KEY: &str = "notifications"; + +/// A validated provider name, as written in `[demand] modules` or +/// `[ad-server] module`. #[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] pub struct ProviderId(String); impl ProviderId { - /// Borrow the validated identifier. + /// Borrow the validated name. #[must_use] pub fn as_str(&self) -> &str { &self.0 } #[cfg(test)] + #[allow( + dead_code, + reason = "kept for a legacy test path the provider types will retire" + )] pub(crate) fn unchecked_for_legacy_test(value: &str) -> Self { Self(value.to_string()) } @@ -47,10 +67,10 @@ impl FromStr for ProviderId { && value .as_bytes() .iter() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-'); + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_'); if !valid { return Err(configuration_error(format!( - "provider ID `{value}` must match ^[a-z][a-z0-9-]{{0,62}}$" + "provider name `{value}` must be snake_case, matching ^[a-z][a-z0-9_]{{0,62}}$" ))); } Ok(Self(value.to_string())) @@ -124,80 +144,55 @@ impl Serialize for BidderId { } } -/// Raw config-first provider declaration. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct ProviderConfig { - /// Protocol identifier. Version one accepts only `openrtb-2.6`. - pub protocol: String, - /// Registered profile identifier. - #[serde(default = "default_profile")] - pub profile: String, - /// Fixed provider endpoint. - pub endpoint: String, - /// Optional profile-default timeout override. - #[serde(default)] - pub timeout_ms: Option, - /// Slot routing mode. - #[serde(default)] - pub routing: RoutingMode, - /// Common `OpenRTB` notification policy. - #[serde(default)] - pub notifications: NotificationConfig, - /// Selected profile's typed configuration object. - #[serde(default = "empty_object")] - pub profile_config: Value, -} - -/// Raw central bidder route. +/// A bidder code's route to one selected demand source. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct BidderRouteConfig { - /// Referenced provider identifier. - pub provider: ProviderId, + /// The `[demand]` module the bidder is sent to. + pub module: ProviderId, } -/// Provider slot routing behavior. +/// Which slots a demand source receives. #[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum RoutingMode { - /// Route only centrally assigned or trusted demand. + /// Only slots carrying a bidder routed to the source, or demand the server + /// routed to it. #[default] Explicit, - /// Route every banner-compatible slot. + /// Every banner-compatible slot. AllEligible, } -/// Common normalized-notification suppression configuration. +/// Notification URL suppression for one demand source. #[derive(Debug, Clone, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct NotificationConfig { - /// Suppress notification URLs for every normalized bid. + /// Suppress notification URLs for every bid. #[serde(default)] pub suppress_all: bool, - /// Suppress notification URLs for exact returned-seat matches. + /// Suppress notification URLs for bids from these returned seats. #[serde(default)] pub suppress_seats: Vec, } -/// Raw internal input for target-independent plan compilation. -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] +/// Input for target-independent plan compilation. +#[derive(Debug, Clone, Default)] pub struct AuctionPlanConfig { /// Auction-wide logical timeout. pub timeout_ms: u32, - /// Operator-defined provider instances. - #[serde(default)] - pub providers: BTreeMap, - /// Client bidder-to-provider routes. - #[serde(default)] + /// The `[demand]` table. + pub demand: ProviderList, + /// The `[ad-server]` table. + pub adserver: ProviderChoice, + /// Client bidder routes from `[auction.bidders]`. pub bidders: BTreeMap, - /// Existing separately registered mock mediator. - #[serde(default)] - pub mediator: Option, - /// Existing global Trusted Server signing configuration. - #[serde(default)] + /// Trusted Server's request signing configuration. pub request_signing: Option, + /// The demand implementations the integration builders registered. + pub demand_implementations: Vec<&'static DemandImplementation>, + /// The ad server implementations the integration builders registered. + pub adserver_implementations: Vec<&'static AdServerImplementation>, } /// Canonical absolute provider endpoint. @@ -216,13 +211,6 @@ impl CanonicalProviderEndpoint { } } -/// Closed first-version protocol plan. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum ProtocolPlan { - /// `OpenRTB` version 2.6 subset. - OpenRtb26, -} - /// Immutable common notification policy. #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct NotificationPolicy { @@ -232,23 +220,58 @@ pub struct NotificationPolicy { pub suppress_seats: BTreeSet, } -/// Immutable compiled provider instance. -#[derive(Debug, Clone)] +/// One compiled demand source. +#[derive(Clone)] pub struct ProviderPlan { - /// Provider identity. + /// The configured name. pub id: ProviderId, + /// The implementation the name resolved to. + pub implementation: &'static DemandImplementation, /// Canonical endpoint. pub endpoint: CanonicalProviderEndpoint, - /// Resolved profile-default or explicit timeout. + /// The table's timeout, or the implementation's default. pub timeout_ms: u32, /// Slot routing mode. pub routing: RoutingMode, /// Common notification policy. pub notifications: NotificationPolicy, - /// Compiled protocol behavior. - pub protocol: ProtocolPlan, - /// Compiled typed profile behavior. - pub profile: CompiledOpenRtbProfile, + /// The compiled request and response behavior. + pub demand: Arc, +} + +impl core::fmt::Debug for ProviderPlan { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("ProviderPlan") + .field("id", &self.id) + .field("implementation", &self.implementation.id) + .field("endpoint", &self.endpoint) + .field("timeout_ms", &self.timeout_ms) + .field("routing", &self.routing) + .field("notifications", &self.notifications) + .finish_non_exhaustive() + } +} + +/// The compiled ad server selection. +#[derive(Clone)] +pub struct AdServerPlan { + /// The configured name. + pub id: ProviderId, + /// The implementation the name resolved to. + pub implementation: &'static AdServerImplementation, + /// The table's own settings, without `implementation`. + pub settings: Map, +} + +impl core::fmt::Debug for AdServerPlan { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("AdServerPlan") + .field("id", &self.id) + .field("implementation", &self.implementation.id) + .finish_non_exhaustive() + } } /// Immutable target-independent auction plan. @@ -259,7 +282,8 @@ pub struct AuctionPlan { providers: Vec, bidder_routes: BTreeMap, signing_enabled: bool, - mediator: Option, + adserver: Option, + publishes_auction_token: bool, } impl AuctionPlan { @@ -269,77 +293,118 @@ impl AuctionPlan { self.enabled } - pub(crate) fn with_enabled(mut self, enabled: bool) -> Self { + /// Return the plan with auction execution switched on or off, which is how + /// `[auction] enabled` is applied to a compiled plan. + #[must_use] + pub fn with_enabled(mut self, enabled: bool) -> Self { self.enabled = enabled; self } + /// Return whether each auction publishes a token with its winning bids, + /// which it does when a module that runs reads one. + #[must_use] + pub fn publishes_auction_token(&self) -> bool { + self.publishes_auction_token + } + + pub(crate) fn with_auction_token(mut self, publishes: bool) -> Self { + self.publishes_auction_token = publishes; + self + } + /// Compile a deterministic plan without adapter-specific validation. /// /// # Errors /// - /// Returns a configuration error for invalid identifiers, protocol/profile - /// declarations, endpoints, profile configuration, routes, notifications, - /// signing structure, or mediator selection. + /// Returns a configuration error for a table no selector names, a label + /// that is not `snake_case`, an implementation no builder registered, an + /// invalid endpoint, timeout, route or notification setting, settings an + /// implementation rejects, or invalid signing structure. pub fn compile(config: AuctionPlanConfig) -> Result> { if config.timeout_ms == 0 { return Err(configuration_error( "auction timeout_ms must be greater than zero", )); } - validate_mediator(config.mediator.as_deref())?; - let signing_enabled = compile_signing_enabled(config.request_signing.as_ref())?; - let mut providers = Vec::with_capacity(config.providers.len()); + config + .demand + .validate("demand") + .map_err(configuration_error)?; + config + .adserver + .validate("ad-server") + .map_err(configuration_error)?; + let signing_enabled = config + .request_signing + .as_ref() + .is_some_and(|signing| signing.enabled); + + let mut providers = Vec::new(); let mut provider_indices = BTreeMap::new(); - for (id, raw) in config.providers { - if raw.protocol != "openrtb-2.6" { + for name in config.demand.selected() { + let id = ProviderId::from_str(name)?; + let implementation = resolve_implementation( + "demand", + name, + config.demand.implementation_of(name), + &config.demand_implementations, + |implementation| implementation.id, + )?; + let mut settings = config.demand.settings_of(name); + let endpoint = take_setting::(&mut settings, "demand", name, ENDPOINT_KEY)? + .ok_or_else(|| configuration_error(format!("[demand.{name}] needs an endpoint")))?; + let timeout_ms = take_setting::(&mut settings, "demand", name, TIMEOUT_KEY)?; + let routing = take_setting::(&mut settings, "demand", name, ROUTING_KEY)? + .unwrap_or_default(); + let notifications = take_setting::( + &mut settings, + "demand", + name, + NOTIFICATIONS_KEY, + )? + .unwrap_or_default(); + + if routing == RoutingMode::AllEligible && !implementation.allows_all_eligible { return Err(configuration_error(format!( - "provider `{id}` uses unsupported protocol `{}`", - raw.protocol + "[demand.{name}] cannot use routing `all_eligible` with implementation `{}`; configure explicit bidder routes", + implementation.id ))); } - let registration = find_profile(&raw.profile).ok_or_else(|| { - configuration_error(format!( - "provider `{id}` uses unknown OpenRTB profile `{}`", - raw.profile - )) + let mut endpoint_url = check_endpoint("demand", name, &endpoint)?; + (implementation.canonicalize_endpoint)(&mut endpoint_url).map_err(|reason| { + configuration_error(format!("[demand.{name}] endpoint {reason}")) })?; - if registration.id == "prebid-server" && raw.routing == RoutingMode::AllEligible { - return Err(configuration_error(format!( - "provider `{id}` cannot use routing `all_eligible` with profile `prebid-server`; configure explicit bidder routes" - ))); - } - if !raw.profile_config.is_object() { - return Err(configuration_error(format!( - "provider `{id}` profile_config must be an object" - ))); - } - let endpoint = canonicalize_endpoint(&id, registration.id, &raw.endpoint)?; - let timeout_ms = raw - .timeout_ms - .unwrap_or(match registration.default_timeout { - ProfileTimeoutDefault::Auction => config.timeout_ms, - ProfileTimeoutDefault::Fixed(value) => value, - }); + let timeout_ms = timeout_ms.unwrap_or(match implementation.default_timeout { + DemandTimeoutDefault::Auction => config.timeout_ms, + DemandTimeoutDefault::Fixed(value) => value, + }); if timeout_ms == 0 { return Err(configuration_error(format!( - "provider `{id}` timeout_ms must be greater than zero" + "[demand.{name}] timeout_ms must be greater than zero" ))); } - let notifications = compile_notifications(&id, raw.notifications)?; - let profile = registration.compile(&raw.profile_config)?; - let index = providers.len(); - provider_indices.insert(id.clone(), index); + let notifications = compile_notifications(&id, notifications)?; + let demand = (implementation.compile)(&settings).change_context( + TrustedServerError::Configuration { + message: format!( + "[demand.{name}] settings are not valid for implementation `{}`", + implementation.id + ), + }, + )?; + provider_indices.insert(id.clone(), providers.len()); providers.push(ProviderPlan { id, - endpoint, + implementation, + endpoint: CanonicalProviderEndpoint(endpoint_url), timeout_ms, - routing: raw.routing, + routing, notifications, - protocol: ProtocolPlan::OpenRtb26, - profile, + demand, }); } + let mut bidder_routes = BTreeMap::new(); for (bidder, route) in config.bidders { if bidder.as_str() == RESERVED_BROWSER_ENVELOPE_BIDDER_ID { @@ -349,23 +414,59 @@ impl AuctionPlan { } let provider_index = provider_indices - .get(&route.provider) + .get(&route.module) .copied() .ok_or_else(|| { configuration_error(format!( - "bidder `{bidder}` references unknown provider `{}`", - route.provider + "[auction.bidders.{bidder}] sends the bidder to `{}`, which [demand] modules does not select", + route.module )) })?; bidder_routes.insert(bidder, provider_index); } + + let adserver = match config.adserver.selected().first().copied() { + None => None, + Some(name) => { + let id = ProviderId::from_str(name)?; + let implementation = resolve_implementation( + "ad-server", + name, + config.adserver.implementation_of(name), + &config.adserver_implementations, + |implementation| implementation.id, + )?; + let settings = config.adserver.settings_of(name); + if let Some(endpoint) = settings.get(ENDPOINT_KEY) { + let endpoint = endpoint.as_str().ok_or_else(|| { + configuration_error(format!("[ad-server.{name}] endpoint must be a URL")) + })?; + check_endpoint("ad-server", name, endpoint)?; + } + (implementation.build)(name, &settings).change_context( + TrustedServerError::Configuration { + message: format!( + "[ad-server.{name}] settings are not valid for implementation `{}`", + implementation.id + ), + }, + )?; + Some(AdServerPlan { + id, + implementation, + settings, + }) + } + }; + Ok(Self { enabled: true, timeout_ms: config.timeout_ms, providers, bidder_routes, signing_enabled, - mediator: config.mediator, + adserver, + publishes_auction_token: false, }) } } @@ -470,38 +571,39 @@ impl AuctionPlan { Ok(()) } - /// Borrow compiled providers in deterministic provider-ID order. + /// Borrow compiled demand sources in the order `[demand] modules` lists + /// them. #[must_use] pub fn providers(&self) -> &[ProviderPlan] { &self.providers } - /// Borrow a compiled provider by its validated identity. + /// Borrow a compiled demand source by its name. #[must_use] pub(crate) fn provider(&self, id: &ProviderId) -> Option<&ProviderPlan> { self.providers.iter().find(|provider| provider.id == *id) } - /// Return whether any compiled provider uses the named profile. + /// Return whether any compiled demand source uses the named implementation. /// - /// This narrow query allows capability activation to follow the validated - /// plan without exposing profile configuration. + /// This narrow query lets an implementation's integration activate its own + /// capabilities from the validated plan. #[must_use] - pub fn has_profile(&self, profile_id: &str) -> bool { + pub fn has_implementation(&self, implementation_id: &str) -> bool { self.providers .iter() - .any(|provider| provider.profile.id() == profile_id) + .any(|provider| provider.implementation.id == implementation_id) } - /// Borrow validated client-visible bidder route codes in deterministic order. + /// Borrow the validated bidder route codes in deterministic order. /// - /// This intentionally exposes route keys rather than provider identities or - /// profile configuration for the browser Prebid injection boundary. - pub(crate) fn browser_bidder_codes(&self) -> impl Iterator { + /// These are the route keys alone, with no source name and no + /// implementation setting, so a page module may show them to a browser. + pub fn browser_bidder_codes(&self) -> impl Iterator { self.bidder_routes.keys().map(BidderId::as_str) } - /// Resolve a bidder route to a compiled provider. + /// Resolve a bidder route to a compiled demand source. #[must_use] pub fn provider_for_bidder(&self, bidder: &BidderId) -> Option<&ProviderPlan> { self.bidder_routes @@ -515,95 +617,106 @@ impl AuctionPlan { self.signing_enabled } - /// Borrow the separately validated static mediator identifier. + /// Borrow the selected ad server, when `[ad-server] module` names one. #[must_use] - pub fn mediator(&self) -> Option<&str> { - self.mediator.as_deref() + pub fn adserver(&self) -> Option<&AdServerPlan> { + self.adserver.as_ref() } } -fn default_profile() -> String { - "standard".to_string() -} - -fn empty_object() -> Value { - Value::Object(serde_json::Map::new()) -} - fn configuration_error(message: impl Into) -> Report { Report::new(TrustedServerError::Configuration { message: message.into(), }) } -fn compile_signing_enabled( - request_signing: Option<&RequestSigning>, -) -> Result> { - let Some(request_signing) = request_signing else { - return Ok(false); - }; - if request_signing.enabled - && (request_signing.config_store_id.trim().is_empty() - || request_signing.secret_store_id.trim().is_empty()) - { - return Err(configuration_error( - "enabled request_signing requires nonblank config_store_id and secret_store_id", - )); - } - Ok(request_signing.enabled) +/// The registered implementation `written` names in `[section]`. +/// +/// An implementation is named by its module path, which a section named by +/// the implementation's type folder may shorten, so `mock` in `[ad-server]` +/// is `ad-server.mock`. The refusal lists what the build has, as the section +/// would write it. +fn resolve_implementation( + section: &str, + name: &str, + written: &str, + registered: &[&'static T], + id: fn(&T) -> &'static str, +) -> Result<&'static T, Report> { + let ids = registered + .iter() + .map(|implementation| id(implementation)) + .collect::>(); + crate::module_name::resolve(section, written, &ids) + .and_then(|resolved| { + registered + .iter() + .copied() + .find(|implementation| id(implementation) == resolved) + }) + .ok_or_else(|| { + let mut known = ids + .iter() + .map(|id| crate::module_name::short_form(section, id)) + .collect::>(); + known.sort_unstable(); + configuration_error(format!( + "[{section}] `{name}` uses implementation `{written}`, which this build does not have. The implementations it has are: {}", + known.join(", ") + )) + }) } -fn validate_mediator(mediator: Option<&str>) -> Result<(), Report> { - if mediator.is_some_and(|value| value != MOCK_MEDIATOR_ID) { - return Err(configuration_error(format!( - "auction mediator must be `{MOCK_MEDIATOR_ID}` when configured" - ))); - } - Ok(()) +/// Takes one common setting out of a table's settings, parsing it as `T`. +fn take_setting( + settings: &mut Map, + type_name: &str, + name: &str, + key: &str, +) -> Result, Report> +where + T: for<'de> Deserialize<'de>, +{ + settings + .remove(key) + .map(|value| { + T::deserialize(value).map_err(|error| { + configuration_error(format!("[{type_name}.{name}] {key} is not valid: {error}")) + }) + }) + .transpose() } -fn canonicalize_endpoint( - provider_id: &ProviderId, - profile_id: &str, +/// Checks an endpoint the common way. It must be HTTPS, or plain HTTP to a +/// loopback address, with a host and no credentials or fragment. +fn check_endpoint( + type_name: &str, + name: &str, value: &str, -) -> Result> { - let mut endpoint = Url::parse(value).map_err(|error| { +) -> Result> { + let endpoint = Url::parse(value).map_err(|error| { configuration_error(format!( - "provider `{provider_id}` endpoint must be an absolute HTTPS URL: {error}" + "[{type_name}.{name}] endpoint must be an absolute URL: {error}" )) })?; - if endpoint.scheme() != "https" - || endpoint.host_str().is_none() + let loopback = match endpoint.host() { + Some(Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + Some(Host::Ipv4(address)) => address.is_loopback(), + Some(Host::Ipv6(address)) => address.is_loopback(), + None => false, + }; + let scheme_allowed = endpoint.scheme() == "https" || (endpoint.scheme() == "http" && loopback); + if !scheme_allowed + || endpoint.host().is_none() || !endpoint.username().is_empty() || endpoint.password().is_some() || endpoint.fragment().is_some() { return Err(configuration_error(format!( - "provider `{provider_id}` endpoint must be absolute HTTPS with a host and no credentials or fragment" - ))); - } - if profile_id == "aps" - && endpoint - .path() - .trim_end_matches('/') - .ends_with("/e/dtb/bid") - { - return Err(configuration_error(format!( - "provider `{provider_id}` uses unsupported legacy APS endpoint `/e/dtb/bid`" + "[{type_name}.{name}] endpoint must be HTTPS, or HTTP to 127.0.0.1, ::1 or localhost, with a host and no credentials or fragment" ))); } - if profile_id == "prebid-server" { - normalize_prebid_server_endpoint(&mut endpoint); - } - Ok(CanonicalProviderEndpoint(endpoint)) -} - -fn normalize_prebid_server_endpoint(endpoint: &mut Url) { - match endpoint.path() { - "" | "/" => endpoint.set_path("/openrtb2/auction"), - "/openrtb2/auction/" => endpoint.set_path("/openrtb2/auction"), - _ => {} - } + Ok(endpoint) } fn compile_notifications( @@ -640,28 +753,63 @@ fn compile_notifications( #[cfg(test)] mod tests { use super::*; - use crate::auction::profile::CompiledOpenRtbProfile; - - fn provider(profile: &str) -> ProviderConfig { - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: "https://bid.example/openrtb2/auction".to_string(), - timeout_ms: None, - routing: RoutingMode::Explicit, - notifications: NotificationConfig::default(), - profile_config: empty_object(), - } + use crate::auction::test_support::plain_fixture::PlainDemand; + use crate::provider_table::IMPLEMENTATION_KEY; + use serde_json::json; + + /// The demand implementations the built-in builders register. + fn demand_implementations() -> Vec<&'static DemandImplementation> { + crate::integrations::all_builders(&[]) + .filter_map(|builder| builder.demand()) + .collect() + } + + /// The ad server implementations the built-in builders register. + fn adserver_implementations() -> Vec<&'static AdServerImplementation> { + crate::integrations::all_builders(&[]) + .filter_map(|builder| builder.adserver()) + .collect() + } + + /// One `[demand.]` table naming an implementation. + fn table(implementation: &str) -> Map { + Map::from_iter([ + (IMPLEMENTATION_KEY.to_string(), json!(implementation)), + ( + ENDPOINT_KEY.to_string(), + json!("https://bid.example/openrtb2/auction"), + ), + ]) } - fn config(providers: BTreeMap) -> AuctionPlanConfig { + /// A `[demand]` table selecting every name given, in order. + fn demand(tables: Vec<(&str, Map)>) -> ProviderList { + let selected = tables + .iter() + .map(|(name, _)| (*name).to_string()) + .collect::>(); + let tables = tables + .into_iter() + .map(|(name, table)| (name.to_string(), table)) + .collect::>(); + ProviderList::new(selected, tables) + } + + fn config(tables: Vec<(&str, Map)>) -> AuctionPlanConfig { AuctionPlanConfig { timeout_ms: 1500, - providers, + demand: demand(tables), + demand_implementations: demand_implementations(), + adserver_implementations: adserver_implementations(), ..AuctionPlanConfig::default() } } + /// A plan with one ordinary `OpenRTB` source called `one`. + fn one_source() -> AuctionPlanConfig { + config(vec![("one", table("auction.plain-fixture"))]) + } + fn id(value: &str) -> ProviderId { ProviderId::from_str(value).expect("should parse provider ID") } @@ -670,32 +818,13 @@ mod tests { BidderId::from_str(value).expect("should parse bidder ID") } - fn nested_object(levels: usize) -> Value { - let mut value = Value::String("leaf".to_string()); - for level in 0..levels { - value = Value::Object(serde_json::Map::from_iter([( - format!("level-{level}"), - value, - )])); - } - value - } - - fn nested_array(levels: usize) -> Value { - let mut value = Value::String("leaf".to_string()); - for _ in 0..levels { - value = Value::Array(vec![value]); - } - value - } - #[test] fn target_validation_accepts_fanout_and_rejects_unsupported_targets() { - let providers = BTreeMap::from([ - (id("provider-one"), provider("standard")), - (id("provider-two"), provider("standard")), - ]); - let plan = AuctionPlan::compile(config(providers)).expect("should compile plan"); + let plan = AuctionPlan::compile(config(vec![ + ("provider_one", table("auction.plain-fixture")), + ("provider_two", table("auction.plain-fixture")), + ])) + .expect("should compile plan"); assert!( plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) @@ -708,253 +837,171 @@ mod tests { "Axum should accept provider fanout" ); for target in [ - crate::platform::AuctionTargetId::Cloudflare, crate::platform::AuctionTargetId::Spin, + crate::platform::AuctionTargetId::Cloudflare, ] { - let error = plan - .validate_for_target(target) - .expect_err("should reject unsupported provider fanout"); assert!( - error.to_string().contains("fanout"), - "should explain fanout rejection: {error:?}" + plan.validate_for_target(target).is_err(), + "{target:?} runs one demand source at a time and should refuse fanout" ); } } #[test] - fn fastly_target_validation_canonicalizes_url_derived_ipv6_prediction() { - let mut ipv6_provider = provider("standard"); - ipv6_provider.endpoint = "https://[2001:db8::5]:8443/openrtb".to_string(); - ipv6_provider.timeout_ms = Some(750); - let plan = AuctionPlan::compile(config(BTreeMap::from([( - id("ipv6-provider"), - ipv6_provider, - )]))) - .expect("should compile IPv6 provider plan"); - - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("Fastly target validation should accept canonical IPv6 prediction"); - let bracketed_spec = plan.providers()[0].backend_spec(); - assert_eq!(bracketed_spec.host, "[2001:db8::5]"); - let mut bare_spec = bracketed_spec.clone(); - bare_spec.host = "2001:db8::5".to_string(); - let policy = crate::platform::BackendNamingPolicy::Fastly; - assert_eq!( - policy - .predict(&bracketed_spec) - .expect("should predict URL-derived bracketed IPv6 backend"), - policy - .predict(&bare_spec) - .expect("should predict runtime-normalized bare IPv6 backend"), - "startup target validation and Fastly runtime must hash the same backend spec" + fn a_table_no_selector_names_is_refused() { + let mut raw = one_source(); + raw.demand = ProviderList::new( + vec!["one".to_string()], + BTreeMap::from([ + ("one".to_string(), table("auction.plain-fixture")), + ("left_behind".to_string(), table("auction.plain-fixture")), + ]), ); - } - - #[test] - fn fastly_target_validation_reserves_dynamic_backends_outside_auction() { - let plan_with = |count: usize| { - let providers = (0..count) - .map(|index| { - let mut provider = provider("standard"); - provider.timeout_ms = Some(1000); - (id(&format!("provider-{index}")), provider) - }) - .collect(); - AuctionPlan::compile(config(providers)).expect("should compile provider plan") - }; - - plan_with(19) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("below-budget provider plan should validate"); - plan_with(20) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("at-budget provider plan should validate"); - let error = plan_with(21) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect_err("over-budget provider plan should fail"); - assert!( - error - .to_string() - .contains("exceeding its auction budget of 160") - ); - } - - #[test] - fn fastly_backend_quota_uses_auction_timeout_as_reachable_bucket_ceiling() { - let providers = (0..21) - .map(|index| { - let mut provider = provider("standard"); - provider.timeout_ms = Some(1000); - (id(&format!("provider-{index}")), provider) - }) - .collect(); - let mut bounded = config(providers); - bounded.timeout_ms = 100; - let plan = AuctionPlan::compile(bounded).expect("should compile bounded provider plan"); - + let error = AuctionPlan::compile(raw).expect_err("should refuse a table nothing selects"); + let message = error.to_string(); assert!( - plan.providers() - .iter() - .all(|provider| provider.timeout_ms == 1000), - "quota validation must not rewrite configured provider timeouts" + message.contains("left_behind") && message.contains("modules"), + "should name the table and the selector: {error:?}" ); - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("21 providers reach only the 50ms and 100ms Fastly buckets"); } #[test] - fn disabled_target_validation_skips_fanout_and_collision_checks() { - let providers = BTreeMap::from([ - (id("provider-one"), provider("standard")), - (id("provider-two"), provider("standard")), - ]); - let disabled = AuctionPlan::compile(config(providers)) - .expect("should compile plan") - .with_enabled(false); - - for target in [ - crate::platform::AuctionTargetId::Cloudflare, - crate::platform::AuctionTargetId::Spin, + fn an_unknown_implementation_names_the_ones_this_build_has() { + let mut absent = table("auction.plain-fixture"); + absent.insert(IMPLEMENTATION_KEY.to_string(), json!("fictional_exchange")); + let error = AuctionPlan::compile(config(vec![("one", absent)])) + .expect_err("should refuse an implementation this build does not have"); + let message = error.to_string(); + for expected in [ + "fictional_exchange", + "auction.plain-fixture", + "auction.fixture", ] { - disabled - .validate_for_target(target) - .expect("disabled dormant providers should skip target validation"); + assert!( + message.contains(expected), + "should name the unknown implementation and the known ones: {error:?}" + ); } - - let provider = disabled.providers[0].clone(); - let disabled_collision = AuctionPlan { - enabled: false, - timeout_ms: disabled.timeout_ms, - providers: vec![provider.clone(), provider], - bidder_routes: BTreeMap::new(), - signing_enabled: false, - mediator: None, - }; - disabled_collision - .validate_for_target(crate::platform::AuctionTargetId::Axum) - .expect("disabled dormant providers should skip collision validation"); } #[test] - fn target_validation_keeps_same_origin_timeout_profile_instances_distinct() { - let shared = ProviderConfig { - timeout_ms: Some(777), - ..provider("standard") - }; - let plan = AuctionPlan::compile(config(BTreeMap::from([ - (id("provider-one"), shared.clone()), - (id("provider-two"), shared), - ]))) - .expect("should compile same-origin provider instances"); - - for target in [ - crate::platform::AuctionTargetId::Fastly, - crate::platform::AuctionTargetId::Axum, - ] { - plan.validate_for_target(target) - .expect("provider ID discriminators should prevent predicted collisions"); - } + fn a_demand_source_names_its_implementation_in_full() { + // `[demand]` is no implementation's type folder, so a source's name + // never stands for its implementation. + let unnamed = Map::from_iter([( + ENDPOINT_KEY.to_string(), + json!("https://bid.example/openrtb2/auction"), + )]); + let error = AuctionPlan::compile(config(vec![("exchange", unnamed)])) + .expect_err("should refuse a source with no implementation line"); + let message = error.to_string(); + assert!( + message.contains("uses implementation `exchange`") + && message.contains("auction.plain-fixture"), + "should list the implementations by module path: {error:?}" + ); } #[test] - fn target_validation_rejects_predicted_name_collisions() { - let compiled = AuctionPlan::compile(config(BTreeMap::from([( - id("provider-a"), - provider("standard"), - )]))) - .expect("should compile plan"); - let provider = compiled.providers[0].clone(); - // The compiler prevents duplicate provider IDs. Construct the otherwise - // impossible duplicate internally to pin validation's defense-in-depth - // collision rejection independently of compiler invariants. - let collision_plan = AuctionPlan { - enabled: true, - timeout_ms: compiled.timeout_ms, - providers: vec![provider.clone(), provider], - bidder_routes: BTreeMap::new(), - signing_enabled: false, - mediator: None, - }; + fn an_ad_server_named_by_its_short_form_needs_no_implementation_line() { + let mut raw = one_source(); + raw.adserver = ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + Map::from_iter([( + ENDPOINT_KEY.to_string(), + json!("https://adserver.example/mediate"), + )]), + )]), + ); + let plan = AuctionPlan::compile(raw).expect("should resolve `fixture` within [ad-server]"); + assert_eq!( + plan.adserver().map(|adserver| adserver.implementation.id), + Some("ad-server.fixture") + ); + } - let error = collision_plan - .validate_for_target(crate::platform::AuctionTargetId::Axum) - .expect_err("should reject predicted backend collision"); - assert!(error.to_string().contains("same backend name")); + #[test] + fn a_demand_table_needs_an_endpoint() { + let mut without = table("auction.plain-fixture"); + without.remove(ENDPOINT_KEY); + let error = AuctionPlan::compile(config(vec![("one", without)])) + .expect_err("should refuse a source with no endpoint"); + assert!( + error.to_string().contains("endpoint"), + "should say an endpoint is needed: {error:?}" + ); } #[test] fn provider_id_enforces_exact_grammar_and_bounds() { - for valid in ["a", "pbs-primary", &format!("a{}", "0".repeat(62))] { - assert!(ProviderId::from_str(valid).is_ok(), "should accept {valid}"); + for value in ["a", "provider_one", "p0", &format!("a{}", "b".repeat(62))] { + assert!( + ProviderId::from_str(value).is_ok(), + "should accept `{value}`" + ); } - for invalid in [ + for value in [ "", - "A", + "Provider", + "provider-one", "1provider", - "provider_name", - "provider.name", - "provider/one", - &format!("a{}", "0".repeat(63)), + "_provider", + "provider.one", + &format!("a{}", "b".repeat(63)), ] { assert!( - ProviderId::from_str(invalid).is_err(), - "should reject {invalid}" + ProviderId::from_str(value).is_err(), + "should reject `{value}`" ); } } #[test] fn bidder_id_enforces_admission_bounds() { - assert!(BidderId::from_str("exampleBidder").is_ok()); - for invalid in ["", " bidder", "bidder\n", &"a".repeat(129)] { - let error = BidderId::from_str(invalid).expect_err("should reject invalid bidder ID"); - assert!( - error.to_string().contains(&format!("{invalid:?}")), - "should identify invalid bidder ID {invalid:?}: {error:?}" - ); - } + assert!(BidderId::from_str("example").is_ok()); + assert!(BidderId::from_str("Example").is_ok()); + assert!(BidderId::from_str("").is_err()); + assert!(BidderId::from_str(&"a".repeat(MAX_ID_BYTES + 1)).is_err()); } #[test] - fn compiler_rejects_all_eligible_for_prebid_server_only() { - let mut prebid = provider("prebid-server"); - prebid.routing = RoutingMode::AllEligible; - let error = AuctionPlan::compile(config(BTreeMap::from([(id("pbs-main"), prebid)]))) - .expect_err("should reject all_eligible Prebid Server routing"); + fn all_eligible_is_refused_for_an_implementation_that_forbids_it() { + let mut explicit_only = table("auction.fixture"); + explicit_only.insert(ROUTING_KEY.to_string(), json!("all_eligible")); + let error = AuctionPlan::compile(config(vec![("explicit_main", explicit_only)])) + .expect_err("should refuse all_eligible routing for the implementation"); let message = error.to_string(); - for expected in ["pbs-main", "all_eligible", "prebid-server"] { + for expected in ["explicit_main", "all_eligible", "auction.fixture"] { assert!( message.contains(expected), - "should identify provider, routing, and profile: {error:?}" + "should name the source, the routing and the implementation: {error:?}" ); } - let mut standard = provider("standard"); - standard.routing = RoutingMode::AllEligible; - AuctionPlan::compile(config(BTreeMap::from([(id("standard-main"), standard)]))) - .expect("should retain all_eligible for non-Prebid profiles"); + let mut plain = table("auction.plain-fixture"); + plain.insert(ROUTING_KEY.to_string(), json!("all_eligible")); + AuctionPlan::compile(config(vec![("plain_main", plain)])) + .expect("should keep all_eligible for an implementation that allows it"); } #[test] fn compiler_rejects_exact_reserved_browser_envelope_bidder_id() { - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + let mut raw = one_source(); raw.bidders.insert( bidder("trustedServer"), - BidderRouteConfig { - provider: id("one"), - }, + BidderRouteConfig { module: id("one") }, ); assert!( AuctionPlan::compile(raw).is_err(), "exact reserved bidder ID should be rejected" ); - let mut case_distinct = config(BTreeMap::from([(id("one"), provider("standard"))])); + let mut case_distinct = one_source(); case_distinct.bidders.insert( bidder("TrustedServer"), - BidderRouteConfig { - provider: id("one"), - }, + BidderRouteConfig { module: id("one") }, ); assert!( AuctionPlan::compile(case_distinct).is_ok(), @@ -963,26 +1010,30 @@ mod tests { } #[test] - fn compiler_orders_providers_and_routes_deterministically() { - let mut providers = BTreeMap::new(); - providers.insert(id("z-provider"), provider("standard")); - providers.insert(id("a-provider"), provider("standard")); - let mut raw = config(providers); + fn sources_keep_the_order_they_were_selected_in_and_routes_are_deterministic() { + let mut raw = config(vec![ + ("z_provider", table("auction.plain-fixture")), + ("a_provider", table("auction.plain-fixture")), + ]); raw.bidders.insert( bidder("z-bidder"), BidderRouteConfig { - provider: id("z-provider"), + module: id("z_provider"), }, ); raw.bidders.insert( bidder("a-bidder"), BidderRouteConfig { - provider: id("a-provider"), + module: id("a_provider"), }, ); let plan = AuctionPlan::compile(raw).expect("should compile deterministic plan"); - assert_eq!(plan.providers()[0].id.as_str(), "a-provider"); - assert_eq!(plan.providers()[1].id.as_str(), "z-provider"); + assert_eq!( + plan.providers()[0].id.as_str(), + "z_provider", + "the selector's order is the deployment's order" + ); + assert_eq!(plan.providers()[1].id.as_str(), "a_provider"); assert_eq!( plan.browser_bidder_codes().collect::>(), vec!["a-bidder", "z-bidder"], @@ -991,113 +1042,95 @@ mod tests { assert_eq!( plan.provider_for_bidder(&bidder("a-bidder")) .map(|provider| provider.id.as_str()), - Some("a-provider") + Some("a_provider") ); } #[test] - fn compiler_supports_two_instances_of_the_same_profile() { - let mut providers = BTreeMap::new(); - providers.insert(id("pbs-a"), provider("prebid-server")); - providers.insert(id("pbs-b"), provider("prebid-server")); - let plan = AuctionPlan::compile(config(providers)).expect("should compile two PBS plans"); + fn two_sources_can_run_one_implementation_under_their_own_names() { + let plan = AuctionPlan::compile(config(vec![ + ("source_a", table("auction.plain-fixture")), + ("source_b", table("auction.plain-fixture")), + ])) + .expect("should compile two sources of one implementation"); assert_eq!(plan.providers().len(), 2); assert!( - plan.providers().iter().all(|provider| matches!( - provider.profile, - CompiledOpenRtbProfile::PrebidServer(_) - )) + plan.providers() + .iter() + .all(|provider| provider.implementation.id == "auction.plain-fixture"), + "both names should resolve to the same implementation" + ); + assert!( + plan.providers() + .iter() + .all(|provider| provider.demand.as_any().is::()), + "each source should compile its own settings" ); } #[test] - fn profile_defaults_and_explicit_timeout_override_are_resolved() { - let mut providers = BTreeMap::new(); - providers.insert(id("standard-one"), provider("standard")); - providers.insert(id("pbs-one"), provider("prebid-server")); - providers.insert( - id("aps-one"), - ProviderConfig { - endpoint: "https://aps.example/e/pb/bid".to_string(), - profile_config: serde_json::json!({"account_id": "example-account"}), - ..provider("aps") - }, - ); - providers.insert( - id("pbs-override"), - ProviderConfig { - timeout_ms: Some(321), - ..provider("prebid-server") - }, - ); - let plan = AuctionPlan::compile(config(providers)).expect("should resolve timeouts"); + fn implementation_defaults_and_an_explicit_timeout_are_resolved() { + let mut override_table = table("auction.fixture"); + override_table.insert(TIMEOUT_KEY.to_string(), json!(321)); + let plan = AuctionPlan::compile(config(vec![ + ("plain_one", table("auction.plain-fixture")), + ("fixed_one", table("auction.fixture")), + ("fixed_override", override_table), + ])) + .expect("should resolve timeouts"); let timeouts = plan .providers() .iter() .map(|provider| (provider.id.as_str(), provider.timeout_ms)) .collect::>(); - assert_eq!(timeouts["standard-one"], 1500); - assert_eq!(timeouts["pbs-one"], 1000); - assert_eq!(timeouts["aps-one"], 800); - assert_eq!(timeouts["pbs-override"], 321); + assert_eq!(timeouts["plain_one"], 1500); + assert_eq!(timeouts["fixed_one"], 1000); + assert_eq!(timeouts["fixed_override"], 321); } #[test] - fn profile_registry_is_independent_of_browser_configuration() { - let ids = crate::auction::profile::profile_registrations() - .iter() - .map(|registration| registration.id) - .collect::>(); - assert_eq!(ids, vec!["standard", "prebid-server", "aps"]); - let mut providers = BTreeMap::new(); - providers.insert(id("pbs"), provider("prebid-server")); - let plan = AuctionPlan::compile(config(providers)) + fn the_plan_reports_which_implementations_it_selected() { + let plan = AuctionPlan::compile(config(vec![("source", table("auction.fixture"))])) .expect("should compile without Settings or browser integration state"); - assert!(!plan.has_profile("aps")); - - let mut providers = BTreeMap::new(); - providers.insert( - id("aps-instance"), - ProviderConfig { - endpoint: "https://aps.example/e/pb/bid".to_string(), - profile_config: serde_json::json!({"account_id": "example-account"}), - ..provider("aps") - }, + assert!(!plan.has_implementation("auction.plain-fixture")); + assert!(plan.has_implementation("auction.fixture")); + + let plan = AuctionPlan::compile(one_source()).expect("should compile a plain source"); + assert!( + plan.has_implementation("auction.plain-fixture"), + "a validated plan should say which implementation it selected" ); - let plan = AuctionPlan::compile(config(providers)).expect("should compile APS plan"); + assert!(!plan.has_implementation("auction.fixture")); assert!( - plan.has_profile("aps"), - "validated plan should expose APS renderer capability" + plan.providers()[0].demand.as_any().is::(), + "the source should compile its own settings" ); - assert!(!plan.has_profile("prebid-server")); } #[test] - fn compiler_rejects_unknown_protocol_profile_and_route() { - let mut unknown_protocol = provider("standard"); - unknown_protocol.protocol = "openrtb-2.5".to_string(); - assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("one"), unknown_protocol)]))).is_err() - ); - assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("one"), provider("unknown"))]))) - .is_err() - ); - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + fn a_bidder_route_must_name_a_selected_source() { + let mut raw = one_source(); raw.bidders.insert( bidder("example"), BidderRouteConfig { - provider: id("missing"), + module: id("missing"), }, ); - assert!(AuctionPlan::compile(raw).is_err()); + let error = AuctionPlan::compile(raw).expect_err("should refuse an unrouted bidder"); + assert!( + error.to_string().contains("missing"), + "should name the source the route wanted: {error:?}" + ); } #[test] fn compiler_canonicalizes_https_endpoints_and_rejects_unsafe_forms() { - let mut canonical = provider("standard"); - canonical.endpoint = "https://BID.EXAMPLE:443/path".to_string(); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), canonical)]))) + let mut canonical = table("auction.plain-fixture"); + canonical.insert( + ENDPOINT_KEY.to_string(), + json!("https://BID.EXAMPLE:443/path"), + ); + let plan = AuctionPlan::compile(config(vec![("one", canonical)])) .expect("should canonicalize endpoint"); assert_eq!( plan.providers()[0].endpoint.as_str(), @@ -1110,52 +1143,68 @@ mod tests { "https://bid.example/path#fragment", "/relative", ] { - let mut raw_provider = provider("standard"); - raw_provider.endpoint = endpoint.to_string(); + let mut raw = table("auction.plain-fixture"); + raw.insert(ENDPOINT_KEY.to_string(), json!(endpoint)); assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("one"), raw_provider)]))).is_err(), + AuctionPlan::compile(config(vec![("one", raw)])).is_err(), "should reject {endpoint}" ); } - let mut aps = provider("aps"); - aps.endpoint = "https://aps.example/e/dtb/bid".to_string(); - aps.profile_config = serde_json::json!({"account_id": "example-account"}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); } #[test] - fn compiler_normalizes_only_prebid_server_origin_and_canonical_paths() { + fn plain_http_is_allowed_only_to_a_loopback_host() { + for endpoint in [ + "http://127.0.0.1:8000/openrtb2/auction", + "http://[::1]:8000/openrtb2/auction", + "http://localhost:8000/openrtb2/auction", + "http://LOCALHOST:8000/openrtb2/auction", + ] { + let mut raw = table("auction.plain-fixture"); + raw.insert(ENDPOINT_KEY.to_string(), json!(endpoint)); + AuctionPlan::compile(config(vec![("local", raw)])) + .unwrap_or_else(|error| panic!("should accept {endpoint}: {error:?}")); + } + for endpoint in [ + "http://192.0.2.10:8000/openrtb2/auction", + "http://bid.example/openrtb2/auction", + "http://localhost.example/openrtb2/auction", + ] { + let mut raw = table("auction.plain-fixture"); + raw.insert(ENDPOINT_KEY.to_string(), json!(endpoint)); + let error = AuctionPlan::compile(config(vec![("remote", raw)])) + .expect_err("should reject plain HTTP off the loopback"); + assert!( + error.to_string().contains("127.0.0.1"), + "should say which hosts plain HTTP may reach: {error:?}" + ); + } + } + + #[test] + fn an_implementation_completes_an_endpoint_and_the_plain_one_leaves_it_as_written() { for (configured, expected) in [ ( - "https://pbs.example", - "https://pbs.example/openrtb2/auction", + "https://exchange.example", + "https://exchange.example/fixture/auction", ), ( - "https://pbs.example/", - "https://pbs.example/openrtb2/auction", + "https://exchange.example/", + "https://exchange.example/fixture/auction", ), ( - "https://pbs.example/openrtb2/auction", - "https://pbs.example/openrtb2/auction", + "https://exchange.example?region=example", + "https://exchange.example/fixture/auction?region=example", ), ( - "https://pbs.example/openrtb2/auction/", - "https://pbs.example/openrtb2/auction", - ), - ( - "https://pbs.example?region=example", - "https://pbs.example/openrtb2/auction?region=example", - ), - ("https://pbs.example/bid", "https://pbs.example/bid"), - ( - "https://pbs.example/custom/pbs", - "https://pbs.example/custom/pbs", + "https://exchange.example/bid", + "https://exchange.example/bid", ), ] { - let mut pbs = provider("prebid-server"); - pbs.endpoint = configured.to_string(); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))) - .expect("should compile Prebid Server endpoint"); + let mut source = table("auction.fixture"); + source.insert(ENDPOINT_KEY.to_string(), json!(configured)); + let plan = AuctionPlan::compile(config(vec![("source", source)])) + .expect("should compile the endpoint"); assert_eq!( plan.providers()[0].endpoint.as_str(), expected, @@ -1163,124 +1212,26 @@ mod tests { ); } - let mut standard = provider("standard"); - standard.endpoint = "https://bid.example/".to_string(); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("standard"), standard)]))) - .expect("should compile standard root endpoint"); + let mut plain = table("auction.plain-fixture"); + plain.insert(ENDPOINT_KEY.to_string(), json!("https://bid.example/")); + let plan = AuctionPlan::compile(config(vec![("plain", plain)])) + .expect("should compile a root endpoint unchanged"); assert_eq!( plan.providers()[0].endpoint.as_str(), "https://bid.example/" ); - - let mut aps = provider("aps"); - aps.endpoint = "https://aps.example/e/pb/bid".to_string(); - aps.profile_config = serde_json::json!({"account_id": "example-account"}); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))) - .expect("should compile APS endpoint"); - assert_eq!( - plan.providers()[0].endpoint.as_str(), - "https://aps.example/e/pb/bid" - ); - } - - #[test] - fn standard_extensions_are_typed_bounded_and_cannot_claim_reserved_fields() { - let mut valid = provider("standard"); - valid.profile_config = serde_json::json!({ - "request_ext": {"fictional_account": "example"}, - "imp_ext": {"placement_group": "display"} - }); - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) - .expect("should compile static extensions"); - let CompiledOpenRtbProfile::Standard(standard) = &plan.providers()[0].profile else { - panic!("should compile standard profile") - }; - assert_eq!( - standard.request_ext.as_object()["fictional_account"], - "example" - ); - - let mut standard_owned_fields = provider("standard"); - standard_owned_fields.profile_config = serde_json::json!({ - "request_ext": { - "account": "example-account", - "sdk": {"source": "example"}, - "prebid": {"example": true} - }, - "imp_ext": {"prebid": {"example": true}} - }); - AuctionPlan::compile(config(BTreeMap::from([( - id("standard-owned-fields"), - standard_owned_fields, - )]))) - .expect("should allow standard static extensions outside common-owned fields"); - - for profile_config in [ - serde_json::json!({"request_ext": "bad"}), - serde_json::json!({"request_ext": {"trusted_server": {}}}), - ] { - let mut invalid = provider("standard"); - invalid.profile_config = profile_config; - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); - } - let oversized = "x".repeat(16 * 1024); - let mut invalid = provider("standard"); - invalid.profile_config = serde_json::json!({"request_ext": {"value": oversized}}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); - - let too_many_keys = (0..257) - .map(|index| (format!("key-{index}"), Value::Bool(true))) - .collect::>(); - let mut invalid = provider("standard"); - invalid.profile_config = serde_json::json!({"imp_ext": too_many_keys}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); - } - - #[test] - fn standard_extension_depth_counts_container_levels() { - let mut object_valid = provider("standard"); - object_valid.profile_config = serde_json::json!({"request_ext": nested_object(8)}); - AuctionPlan::compile(config(BTreeMap::from([(id("object-valid"), object_valid)]))) - .expect("should accept eight nested object levels"); - - let mut object_invalid = provider("standard"); - object_invalid.profile_config = serde_json::json!({"request_ext": nested_object(9)}); - assert!( - AuctionPlan::compile(config(BTreeMap::from([( - id("object-invalid"), - object_invalid, - )]))) - .is_err(), - "should reject nine nested object levels" - ); - - let mut array_valid = provider("standard"); - array_valid.profile_config = serde_json::json!({"request_ext": {"value": nested_array(7)}}); - AuctionPlan::compile(config(BTreeMap::from([(id("array-valid"), array_valid)]))) - .expect("should accept one object plus seven nested array levels"); - - let mut array_invalid = provider("standard"); - array_invalid.profile_config = - serde_json::json!({"request_ext": {"value": nested_array(8)}}); - assert!( - AuctionPlan::compile(config(BTreeMap::from([( - id("array-invalid"), - array_invalid, - )]))) - .is_err(), - "should reject one object plus eight nested array levels" - ); } #[test] fn notification_policy_rejects_duplicates_and_bounds() { - let mut valid = provider("standard"); - valid.notifications = NotificationConfig { - suppress_all: true, - suppress_seats: vec!["seat-b".to_string(), "seat-a".to_string()], - }; - let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) + let mut valid = table("auction.plain-fixture"); + valid.insert( + NOTIFICATIONS_KEY.to_string(), + json!({"suppress_all": true, "suppress_seats": ["seat-b", "seat-a"]}), + ); + let plan = AuctionPlan::compile(config(vec![("one", valid)])) .expect("should compile notifications"); + assert!(plan.providers()[0].notifications.suppress_all); assert_eq!( plan.providers()[0] .notifications @@ -1291,82 +1242,116 @@ mod tests { vec!["seat-a", "seat-b"] ); for seats in [ - vec!["same".to_string(), "same".to_string()], - vec![String::new()], - vec!["bad\nseat".to_string()], - vec!["x".repeat(129)], - (0..129).map(|index| format!("seat-{index}")).collect(), + json!(["same", "same"]), + json!([""]), + json!(["bad\nseat"]), + json!(["x".repeat(129)]), + Value::Array( + (0..129) + .map(|index| json!(format!("seat-{index}"))) + .collect(), + ), ] { - let mut invalid = provider("standard"); - invalid.notifications.suppress_seats = seats; - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + let mut invalid = table("auction.plain-fixture"); + invalid.insert( + NOTIFICATIONS_KEY.to_string(), + json!({"suppress_seats": seats}), + ); + assert!( + AuctionPlan::compile(config(vec![("one", invalid)])).is_err(), + "should refuse {seats}" + ); } } #[test] - fn typed_profile_config_rejects_unknown_fields_and_validates_aps_pairing() { - let mut pbs = provider("prebid-server"); - pbs.profile_config = serde_json::json!({"browser_only": true}); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))).is_err()); - - let mut non_object = provider("standard"); - non_object.profile_config = Value::Null; + fn an_implementation_rejects_settings_it_does_not_know() { + let mut source = table("auction.fixture"); + source.insert("browser_only".to_string(), json!(true)); + let error = AuctionPlan::compile(config(vec![("source", source)])) + .expect_err("should refuse a setting the implementation does not know"); assert!( - AuctionPlan::compile(config(BTreeMap::from([(id("standard"), non_object,)]))).is_err() + format!("{error:?}").contains("browser_only"), + "should name the setting: {error:?}" ); - - let mut aps = provider("aps"); - aps.endpoint = "https://aps.example/e/pb/bid".to_string(); - aps.profile_config = serde_json::json!({ - "account_id": "example-account", - "inventory_domain": "publisher.example" - }); - assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); } #[test] - fn enabled_signing_requires_nonblank_existing_global_store_ids() { - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); - raw.request_signing = Some(RequestSigning { - enabled: true, - config_store_id: " ".to_string(), - secret_store_id: "example-secret-store".to_string(), - }); - assert!( - AuctionPlan::compile(raw).is_err(), - "should reject enabled signing without a config store ID" + fn routing_signing_and_the_selected_ad_server_are_preserved_in_the_plan() { + let mut plain = table("auction.plain-fixture"); + plain.insert(ROUTING_KEY.to_string(), json!("all_eligible")); + let mut raw = config(vec![("one", plain)]); + raw.request_signing = Some(RequestSigning::new(true)); + raw.adserver = ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + Map::from_iter([( + ENDPOINT_KEY.to_string(), + json!("http://127.0.0.1:6767/adserver/mediate"), + )]), + )]), + ); + let plan = AuctionPlan::compile(raw).expect("should compile common policies"); + assert_eq!(plan.providers()[0].routing, RoutingMode::AllEligible); + assert!(plan.signing_enabled()); + assert_eq!( + plan.adserver().map(|adserver| adserver.id.as_str()), + Some("fixture") ); - let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); - raw.request_signing = Some(RequestSigning { - enabled: true, - config_store_id: "example-config-store".to_string(), - secret_store_id: "\t".to_string(), - }); + let mut invalid = config(Vec::new()); + invalid.adserver = + ProviderChoice::new(Some("fictional_adserver".to_string()), BTreeMap::new()); + let error = AuctionPlan::compile(invalid) + .expect_err("should refuse an ad server this build does not have"); assert!( - AuctionPlan::compile(raw).is_err(), - "should reject enabled signing without a secret store ID" + error.to_string().contains("fictional_adserver"), + "should name the ad server: {error:?}" ); } #[test] - fn routing_signing_and_static_mediator_are_preserved_in_plan() { - let mut provider = provider("standard"); - provider.routing = RoutingMode::AllEligible; - let mut raw = config(BTreeMap::from([(id("one"), provider)])); - raw.request_signing = Some(RequestSigning { - enabled: true, - config_store_id: "example-config-store".to_string(), - secret_store_id: "example-secret-store".to_string(), - }); - raw.mediator = Some(MOCK_MEDIATOR_ID.to_string()); - let plan = AuctionPlan::compile(raw).expect("should compile common policies"); - assert_eq!(plan.providers()[0].routing, RoutingMode::AllEligible); - assert!(plan.signing_enabled()); - assert_eq!(plan.mediator(), Some(MOCK_MEDIATOR_ID)); - - let mut invalid = config(BTreeMap::new()); - invalid.mediator = Some("generic-mediator".to_string()); - assert!(AuctionPlan::compile(invalid).is_err()); + fn an_ad_server_is_built_from_its_own_table() { + let mut raw = one_source(); + raw.adserver = ProviderChoice::new( + Some("house".to_string()), + BTreeMap::from([( + "house".to_string(), + Map::from_iter([ + (IMPLEMENTATION_KEY.to_string(), json!("fixture")), + ( + ENDPOINT_KEY.to_string(), + json!("https://adserver.example/mediate"), + ), + ("timeout_ms".to_string(), json!(750)), + ]), + )]), + ); + let plan = AuctionPlan::compile(raw).expect("should compile a named ad server"); + let adserver = plan.adserver().expect("should select an ad server"); + assert_eq!(adserver.id.as_str(), "house"); + assert_eq!(adserver.implementation.id, "ad-server.fixture"); + + let mut unknown_setting = one_source(); + unknown_setting.adserver = ProviderChoice::new( + Some("fixture".to_string()), + BTreeMap::from([( + "fixture".to_string(), + Map::from_iter([ + ( + ENDPOINT_KEY.to_string(), + json!("https://adserver.example/mediate"), + ), + ("enabled".to_string(), json!(true)), + ]), + )]), + ); + let error = AuctionPlan::compile(unknown_setting) + .expect_err("should refuse a setting the ad server does not know"); + assert!( + format!("{error:?}").contains("enabled"), + "should name the setting: {error:?}" + ); } } diff --git a/crates/trusted-server-core/src/auction/profile.rs b/crates/trusted-server-core/src/auction/profile.rs deleted file mode 100644 index 51a91e69c6..0000000000 --- a/crates/trusted-server-core/src/auction/profile.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! Compile-time `OpenRTB` profile registry and typed profile plans. - -use std::collections::BTreeMap; - -use error_stack::Report; -use serde::Deserialize; -use serde_json::{Map, Value}; - -use crate::consent_config::ConsentForwardingMode; -use crate::error::TrustedServerError; -use crate::integrations::aps::compile_profile_config as compile_aps_profile_config; -use crate::integrations::prebid::{ - BidParamOverrideEngine, BidParamOverrideRule, compile_profile_override_rules, -}; - -const STANDARD_PROFILE_ID: &str = "standard"; -const PREBID_PROFILE_ID: &str = "prebid-server"; -const APS_PROFILE_ID: &str = "aps"; -const STATIC_EXTENSION_MAX_BYTES: usize = 16 * 1024; -const STATIC_EXTENSION_MAX_DEPTH: usize = 8; -const STATIC_EXTENSION_MAX_KEYS: usize = 256; - -/// A registered profile's provider-timeout default. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProfileTimeoutDefault { - /// Inherit the configured auction timeout. - Auction, - /// Use this fixed profile timeout. - Fixed(u32), -} - -/// Compile-time profile registration. -#[derive(Clone, Copy)] -pub struct OpenRtbProfileRegistration { - /// Stable profile identifier used by configuration. - pub id: &'static str, - /// Profile timeout used when a provider has no explicit override. - pub default_timeout: ProfileTimeoutDefault, - compile: fn(&Value) -> Result>, -} - -impl core::fmt::Debug for OpenRtbProfileRegistration { - fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - formatter - .debug_struct("OpenRtbProfileRegistration") - .field("id", &self.id) - .field("default_timeout", &self.default_timeout) - .finish_non_exhaustive() - } -} - -impl OpenRtbProfileRegistration { - pub(crate) fn compile( - self, - config: &Value, - ) -> Result> { - (self.compile)(config) - } -} - -/// Immutable, typed profile behavior selected during plan compilation. -#[derive(Debug, Clone)] -pub enum CompiledOpenRtbProfile { - /// Generic `OpenRTB` 2.6 profile. - Standard(StandardProfilePlan), - /// Prebid Server compatibility profile. - PrebidServer(PrebidProfilePlan), - /// APS `OpenRTB` compatibility profile. - Aps(ApsProfilePlan), -} - -impl CompiledOpenRtbProfile { - /// Return the stable profile identifier. - #[must_use] - pub fn id(&self) -> &'static str { - match self { - Self::Standard(_) => STANDARD_PROFILE_ID, - Self::PrebidServer(_) => PREBID_PROFILE_ID, - Self::Aps(_) => APS_PROFILE_ID, - } - } - - /// Return whether this plan uses the Prebid Server profile. - #[must_use] - pub(crate) fn is_prebid_server(&self) -> bool { - matches!(self, Self::PrebidServer(_)) - } -} - -/// Validated static extension object. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct StaticExtension(Map); - -impl StaticExtension { - /// Borrow the validated extension object. - #[must_use] - pub fn as_object(&self) -> &Map { - &self.0 - } -} - -/// Compiled generic `OpenRTB` profile configuration. -#[derive(Debug, Clone, Default)] -pub struct StandardProfilePlan { - /// Static request-level extension fields. - pub request_ext: StaticExtension, - /// Static impression-level extension fields. - pub imp_ext: StaticExtension, -} - -/// Compiled Prebid profile configuration. -#[derive(Debug, Clone)] -pub struct PrebidProfilePlan { - /// Include Prebid HTTP exchange diagnostics. - pub debug: bool, - /// Set `OpenRTB` test mode. - pub test_mode: bool, - /// Optional query fragment appended to the page URL under legacy rules. - pub debug_query_params: Option, - /// Compiled override matching and merge index. - pub(crate) override_engine: BidParamOverrideEngine, - /// Consent transport policy. - pub consent_forwarding: ConsentForwardingMode, -} - -/// Compiled APS profile configuration. -#[derive(Debug, Clone)] -pub struct ApsProfilePlan { - /// APS account identifier. - pub account_id: String, - /// Include APS request/response diagnostics. - pub debug: bool, - /// Permit APS script creatives. - pub allow_script_creatives: bool, - /// Optional authorized inventory domain. - pub inventory_domain: Option, - /// Optional canonical inventory page origin. - pub inventory_page_origin: Option, -} - -#[derive(Debug, Deserialize, Default)] -#[serde(deny_unknown_fields)] -struct StandardProfileConfig { - #[serde(default)] - request_ext: Option, - #[serde(default)] - imp_ext: Option, -} - -/// Typed operator configuration compiled into a [`PrebidProfilePlan`]. -#[derive(Debug, Deserialize, Default)] -#[serde(deny_unknown_fields)] -pub(crate) struct PrebidProfileConfig { - #[serde(default)] - debug: bool, - #[serde(default)] - test_mode: bool, - #[serde(default)] - debug_query_params: Option, - #[serde(default)] - bid_param_zone_overrides: BTreeMap>>, - #[serde(default)] - bid_param_overrides: BTreeMap>, - #[serde(default)] - bid_param_override_rules: Vec, - #[serde(default)] - consent_forwarding: ConsentForwardingMode, -} - -const PROFILE_REGISTRATIONS: [OpenRtbProfileRegistration; 3] = [ - OpenRtbProfileRegistration { - id: STANDARD_PROFILE_ID, - default_timeout: ProfileTimeoutDefault::Auction, - compile: compile_standard, - }, - OpenRtbProfileRegistration { - id: PREBID_PROFILE_ID, - default_timeout: ProfileTimeoutDefault::Fixed(1000), - compile: compile_prebid, - }, - OpenRtbProfileRegistration { - id: APS_PROFILE_ID, - default_timeout: ProfileTimeoutDefault::Fixed(800), - compile: compile_aps, - }, -]; - -/// Return the compile-time profile registry. -#[must_use] -pub fn profile_registrations() -> &'static [OpenRtbProfileRegistration] { - &PROFILE_REGISTRATIONS -} - -pub(crate) fn find_profile(id: &str) -> Option { - profile_registrations() - .iter() - .copied() - .find(|registration| registration.id == id) -} - -fn configuration_error(message: impl Into) -> Report { - Report::new(TrustedServerError::Configuration { - message: message.into(), - }) -} - -fn deserialize_profile(id: &str, value: &Value) -> Result> -where - T: for<'de> Deserialize<'de>, -{ - T::deserialize(value) - .map_err(|error| configuration_error(format!("invalid `{id}` profile_config: {error}"))) -} - -fn compile_standard(value: &Value) -> Result> { - let config: StandardProfileConfig = deserialize_profile(STANDARD_PROFILE_ID, value)?; - Ok(CompiledOpenRtbProfile::Standard(StandardProfilePlan { - request_ext: validate_static_extension("request_ext", config.request_ext)?, - imp_ext: validate_static_extension("imp_ext", config.imp_ext)?, - })) -} - -fn compile_prebid(value: &Value) -> Result> { - let config: PrebidProfileConfig = deserialize_profile(PREBID_PROFILE_ID, value)?; - let override_engine = compile_profile_override_rules( - &config.bid_param_zone_overrides, - &config.bid_param_overrides, - &config.bid_param_override_rules, - )?; - Ok(CompiledOpenRtbProfile::PrebidServer(PrebidProfilePlan { - debug: config.debug, - test_mode: config.test_mode, - debug_query_params: config.debug_query_params, - override_engine, - consent_forwarding: config.consent_forwarding, - })) -} - -fn compile_aps(value: &Value) -> Result> { - let config = compile_aps_profile_config(value.clone())?; - Ok(CompiledOpenRtbProfile::Aps(ApsProfilePlan { - account_id: config.account_id, - debug: config.debug, - allow_script_creatives: config.allow_script_creatives, - inventory_domain: config.inventory_domain, - inventory_page_origin: config.inventory_page_origin, - })) -} - -fn validate_static_extension( - field: &str, - value: Option, -) -> Result> { - let Some(value) = value else { - return Ok(StaticExtension::default()); - }; - let object = value.as_object().ok_or_else(|| { - configuration_error(format!("standard profile {field} must be an object")) - })?; - let size = serde_json::to_vec(&value) - .map_err(|error| configuration_error(format!("cannot serialize {field}: {error}")))? - .len(); - if size > STATIC_EXTENSION_MAX_BYTES { - return Err(configuration_error(format!( - "standard profile {field} exceeds {STATIC_EXTENSION_MAX_BYTES} bytes" - ))); - } - validate_extension_value(field, &value, 0)?; - reject_reserved_fields(field, object)?; - Ok(StaticExtension(object.clone())) -} - -fn validate_extension_value( - field: &str, - value: &Value, - container_depth: usize, -) -> Result<(), Report> { - match value { - Value::Object(object) => { - let container_depth = container_depth + 1; - if container_depth > STATIC_EXTENSION_MAX_DEPTH { - return Err(configuration_error(format!( - "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" - ))); - } - if object.len() > STATIC_EXTENSION_MAX_KEYS { - return Err(configuration_error(format!( - "standard profile {field} object exceeds {STATIC_EXTENSION_MAX_KEYS} keys" - ))); - } - for nested in object.values() { - validate_extension_value(field, nested, container_depth)?; - } - } - Value::Array(array) => { - let container_depth = container_depth + 1; - if container_depth > STATIC_EXTENSION_MAX_DEPTH { - return Err(configuration_error(format!( - "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" - ))); - } - for nested in array { - validate_extension_value(field, nested, container_depth)?; - } - } - _ => {} - } - Ok(()) -} - -fn reject_reserved_fields( - field: &str, - object: &Map, -) -> Result<(), Report> { - let reserved: &[&str] = match field { - "request_ext" => &["trusted_server"], - _ => &[], - }; - if let Some(key) = reserved.iter().find(|key| object.contains_key(**key)) { - return Err(configuration_error(format!( - "standard profile {field} cannot claim reserved field `{key}`" - ))); - } - Ok(()) -} diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 62b2ddc8fc..d727629862 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -6,11 +6,8 @@ use std::collections::HashSet; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; -use http::{Method, Request, StatusCode, header}; -use serde_json::{Value, json}; - -use crate::integrations::aps::{ApsDebugRequest, parse_planned_aps_response}; -use crate::integrations::prebid::{apply_prebid_transport_headers, parse_planned_prebid_response}; +use http::{Method, Request, header}; +use serde_json::json; use crate::error::TrustedServerError; use crate::platform::{ @@ -18,25 +15,23 @@ use crate::platform::{ }; use crate::request_signing::{RequestSigner, SigningParams}; +use super::demand::{CompiledDemand, DemandResponse, DemandTransport}; use super::openrtb::{ OpenRtbBuildOutcome, RequestFinalization, apply_notification_policy, build_request, - extract_standard_response, unused_bidder_params_count, + unused_bidder_params_count, }; use super::plan::ProviderPlan; -use super::profile::CompiledOpenRtbProfile; use super::routing::{ProviderAuctionInput, RoutedAuction}; use super::types::{AuctionContext, AuctionRequest, AuctionResponse}; -const MAX_PLANNED_RESPONSE_BYTES: usize = 1024 * 1024; - fn attach_provider_routing_metadata( response: &mut AuctionResponse, - profile: &CompiledOpenRtbProfile, + demand: &dyn CompiledDemand, input: &ProviderAuctionInput, ) { response.metadata.insert( "routing".to_string(), - json!({"unused_bidder_params_count": unused_bidder_params_count(profile, input)}), + json!({"unused_bidder_params_count": unused_bidder_params_count(demand, input)}), ); } @@ -197,27 +192,14 @@ pub(crate) struct GenericOpenRtbProvider { plan: ProviderPlan, } -/// Typed state created by and returned to one [`GenericOpenRtbProvider`]. -#[allow( - dead_code, - clippy::large_enum_variant, - reason = "typed Stage 6 state avoids provider-state confusion; Stage 7/8 replace profile variants" -)] -pub(crate) enum GenericOpenRtbParseState { - Standard { - provider_id: String, - input: ProviderAuctionInput, - }, - Prebid { - provider_id: String, - auction_id: String, - input: ProviderAuctionInput, - }, - Aps { - provider_id: String, - input: ProviderAuctionInput, - debug_request: Option, - }, +/// State created by and returned to one [`GenericOpenRtbProvider`]. +/// +/// `captured` holds whatever the demand implementation kept from its own +/// request, and only that implementation reads it back. +pub(crate) struct GenericOpenRtbParseState { + provider_id: String, + input: ProviderAuctionInput, + captured: Option>, } impl GenericOpenRtbProvider { @@ -233,25 +215,13 @@ impl GenericOpenRtbProvider { self.plan.timeout_ms } - #[cfg(test)] + #[cfg(any(test, feature = "test-utils"))] pub(crate) fn parse_state_for_test(&self, input: ProviderAuctionInput) -> ProviderParseState { - let state = match &self.plan.profile { - CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { - provider_id: self.provider_name().to_string(), - input, - }, - CompiledOpenRtbProfile::PrebidServer(_) => GenericOpenRtbParseState::Prebid { - provider_id: self.provider_name().to_string(), - auction_id: input.common_request().id.clone(), - input, - }, - CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { - provider_id: self.provider_name().to_string(), - input, - debug_request: None, - }, - }; - Box::new(state) + Box::new(GenericOpenRtbParseState { + provider_id: self.provider_name().to_string(), + input, + captured: None, + }) } /// Build, register, and start exactly one routed provider request. @@ -341,26 +311,39 @@ impl GenericOpenRtbProvider { })); } + let sent_impression_ids = request + .imp + .iter() + .filter_map(|impression| impression.id.as_deref()) + .collect::>(); let body = serde_json::to_vec(&request).change_context(TrustedServerError::Auction { message: format!( "Provider {} request serialization failed", self.provider_name() ), })?; - let mut outbound = Request::builder() + let demand = self.plan.demand.as_ref(); + let mut builder = Request::builder() .method(Method::POST) .uri(self.plan.endpoint.as_str()) .header(header::CONTENT_TYPE, "application/json"); - if matches!(&self.plan.profile, CompiledOpenRtbProfile::Standard(_)) { - outbound = outbound.header(header::ACCEPT, "application/json"); + if demand.field_policy().accept_json { + builder = builder.header(header::ACCEPT, "application/json"); } - let aps_debug_body = matches!( - &self.plan.profile, - CompiledOpenRtbProfile::Aps(profile) if profile.debug - ) - .then(|| body.clone()); - let mut outbound = - outbound + if let Some(headers) = builder.headers_mut() { + demand.prepare_outbound( + headers, + DemandTransport { + headers: routed.transport_headers(), + attested_client_ip: routed.attested_client_ip(), + }, + ); + } + let captured = builder + .headers_ref() + .and_then(|headers| demand.capture_request(&body, headers)); + let outbound = + builder .body(EdgeBody::from(body)) .change_context(TrustedServerError::Auction { message: format!( @@ -368,17 +351,6 @@ impl GenericOpenRtbProvider { self.provider_name() ), })?; - let aps_debug_request = aps_debug_body - .as_deref() - .map(|body| ApsDebugRequest::capture(body, outbound.headers())); - if let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile { - apply_prebid_transport_headers( - routed.prebid_transport_headers(), - &mut outbound, - profile.consent_forwarding, - routed.attested_client_ip(), - ); - } let pending = services .http_client() .send_async(PlatformHttpRequest::new(outbound, backend_name.clone())) @@ -394,42 +366,23 @@ impl GenericOpenRtbProvider { ), })); } - let parse_state = match &self.plan.profile { - CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { - provider_id: self.provider_name().to_string(), - input: input.clone(), - }, - CompiledOpenRtbProfile::PrebidServer(_) => { - debug_assert!( - request.imp.iter().all(|impression| impression.id.is_some()), - "should populate every PBS impression ID" - ); - let sent_impression_ids = request - .imp - .iter() - .filter_map(|impression| impression.id.as_deref()) - .collect::>(); - GenericOpenRtbParseState::Prebid { - provider_id: self.provider_name().to_string(), - auction_id: input.common_request().id.clone(), - input: input.filtered_slots(|slot| { - sent_impression_ids.contains(slot.slot().id.as_str()) - }), - } - } - CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { - provider_id: self.provider_name().to_string(), - input: input.clone(), - debug_request: aps_debug_request, - }, - }; Ok(ProviderRequestOutcome::pending_with_state( pending, - Box::new(parse_state), + Box::new(GenericOpenRtbParseState { + provider_id: self.provider_name().to_string(), + input: input + .filtered_slots(|slot| sent_impression_ids.contains(slot.slot().id.as_str())), + captured, + }), )) } /// Parse a response using state created by this exact provider instance. + /// + /// The implementation reads its own response and reports an unusable one as + /// an error response. The driver then applies the common notification + /// policy and routing diagnostics, so every demand source reports them the + /// same way. pub(crate) async fn parse_response_with_state( &self, response: PlatformResponse, @@ -446,153 +399,31 @@ impl GenericOpenRtbProvider { ), }) })?; - let state_provider_id = match parse_state { - GenericOpenRtbParseState::Standard { provider_id, .. } - | GenericOpenRtbParseState::Prebid { provider_id, .. } - | GenericOpenRtbParseState::Aps { provider_id, .. } => provider_id, - }; - if state_provider_id != self.provider_name() { + if parse_state.provider_id != self.provider_name() { return Err(Report::new(TrustedServerError::Auction { message: format!( "Provider {} received response state owned by provider {}", self.provider_name(), - state_provider_id + parse_state.provider_id ), })); } - if let GenericOpenRtbParseState::Prebid { - auction_id, input, .. - } = parse_state - { - let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile else { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} received PBS response state for profile {}", - self.provider_name(), - self.plan.profile.id() - ), - })); - }; - let mut parsed = match parse_planned_prebid_response( - self.provider_name(), - profile, - input, + let demand = self.plan.demand.as_ref(); + let mut parsed = demand + .parse_response( + DemandResponse { + provider_id: self.provider_name(), + endpoint: self.plan.endpoint.as_str(), + input: &parse_state.input, + response_time_ms, + captured: parse_state.captured.as_deref(), + }, response, - response_time_ms, - auction_id, ) - .await - { - Ok(parsed) => parsed, - Err(error) => { - log::warn!( - "Provider '{}' PBS response parse failed: {:?}", - self.provider_name(), - error - ); - AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("parse_response")) - } - }; - apply_notification_policy(&mut parsed.bids, &self.plan.notifications); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - - if let GenericOpenRtbParseState::Aps { - input, - debug_request, - .. - } = parse_state - { - let CompiledOpenRtbProfile::Aps(profile) = &self.plan.profile else { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} received APS response state for profile {}", - self.provider_name(), - self.plan.profile.id() - ), - })); - }; - let mut parsed = match parse_planned_aps_response( - self.provider_name(), - profile, - self.plan.endpoint.as_str(), - input, - response, - response_time_ms, - debug_request.clone(), - ) - .await - { - Ok(parsed) => parsed, - Err(error) => { - log::warn!( - "Provider '{}' APS response parse failed: {:?}", - self.provider_name(), - error - ); - let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("parse_response")); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - parsed - } - }; - apply_notification_policy(&mut parsed.bids, &self.plan.notifications); - return Ok(parsed); - } - - let response = response.response; - let status = response.status(); - let GenericOpenRtbParseState::Standard { input, .. } = parse_state else { - unreachable!("profile-specific states are handled before standard parsing"); - }; - if status == StatusCode::NO_CONTENT { - let mut parsed = AuctionResponse::no_bid(self.provider_name(), response_time_ms); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - if !status.is_success() { - if status.is_redirection() { - log::warn!( - "Provider '{}' returned a redirect; generic OpenRTB redirects are refused", - self.provider_name() - ); - } - let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("http_status")) - .with_metadata("http_status", json!(status.as_u16())); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - - let body = response - .into_body() - .into_bytes_bounded(MAX_PLANNED_RESPONSE_BYTES) - .await - .change_context(TrustedServerError::Auction { - message: format!("Provider {} response body failed", self.provider_name()), - })?; - let value: Value = match serde_json::from_slice(&body) { - Ok(value) => value, - Err(error) => { - log::warn!( - "Provider '{}' response JSON was invalid: {}", - self.provider_name(), - error - ); - let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) - .with_metadata("error_type", json!("parse_response")); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); - return Ok(parsed); - } - }; - - let mut parsed = - extract_standard_response(self.provider_name(), input, &value, response_time_ms); + .await?; apply_notification_policy(&mut parsed.bids, &self.plan.notifications); - attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + attach_provider_routing_metadata(&mut parsed, demand, &parse_state.input); Ok(parsed) } } diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs index 2b8135da3c..594726ffb9 100644 --- a/crates/trusted-server-core/src/auction/routing.rs +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -17,8 +17,8 @@ const STORED_REQUEST_FIELD: &str = "storedRequest"; /// Maximum bidder entries admitted from one browser `bidderParams` envelope. pub(crate) const MAX_BIDDER_ENTRIES: usize = 128; -/// Maximum UTF-8 byte length of the optional Prebid zone fact. -pub(crate) const MAX_PREBID_ZONE_BYTES: usize = 256; +/// Maximum UTF-8 byte length of the zone a browser may name for a slot. +pub(crate) const MAX_ZONE_BYTES: usize = 256; /// Immutable provider-local routing output in deterministic provider-ID order. #[derive(Debug, Clone)] @@ -26,7 +26,7 @@ pub(crate) struct RoutedAuction { inputs: Vec, skipped_no_eligible_provider_ids: Vec, diagnostics: RoutingDiagnostics, - transport_headers: PrebidTransportHeaders, + transport_headers: TransportHeaders, attested_client_ip: Option, dnt: Option, } @@ -47,8 +47,8 @@ impl RoutedAuction { self.diagnostics } - /// Request headers approved for later Prebid transport forwarding. - pub(crate) fn prebid_transport_headers(&self) -> &PrebidTransportHeaders { + /// Request headers approved for forwarding to a demand source. + pub(crate) fn transport_headers(&self) -> &TransportHeaders { &self.transport_headers } @@ -122,7 +122,7 @@ impl RoutingDiagnostics { /// Provider-local immutable auction input. #[derive(Debug, Clone)] -pub(crate) struct ProviderAuctionInput { +pub struct ProviderAuctionInput { provider_id: ProviderId, #[cfg_attr( not(test), @@ -137,7 +137,9 @@ pub(crate) struct ProviderAuctionInput { } impl ProviderAuctionInput { - pub(crate) fn provider_id(&self) -> &ProviderId { + /// The configured name of the demand source this input is routed to. + #[must_use] + pub fn provider_id(&self) -> &ProviderId { &self.provider_id } @@ -147,11 +149,14 @@ impl ProviderAuctionInput { } /// Common privacy-approved request data. Its slot list is always empty. - pub(crate) fn common_request(&self) -> &AuctionRequest { + #[must_use] + pub fn common_request(&self) -> &AuctionRequest { &self.common_request } - pub(crate) fn slots(&self) -> &[ProviderSlotInput] { + /// The eligible slots routed to this demand source. + #[must_use] + pub fn slots(&self) -> &[ProviderSlotInput] { &self.slots } @@ -165,25 +170,30 @@ impl ProviderAuctionInput { /// One eligible slot with only the demand assigned to this provider. #[derive(Debug, Clone)] -pub(crate) struct ProviderSlotInput { +pub struct ProviderSlotInput { slot: AdSlot, bidder_params: BTreeMap, - prebid_zone: Option, + zone: Option, stored_request: StoredRequestIntent, } impl ProviderSlotInput { /// Common slot facts. The legacy `bidders` map is always empty. - pub(crate) fn slot(&self) -> &AdSlot { + #[must_use] + pub fn slot(&self) -> &AdSlot { &self.slot } - pub(crate) fn bidder_params(&self) -> &BTreeMap { + /// The bidder parameters routed to this demand source for this slot. + #[must_use] + pub fn bidder_params(&self) -> &BTreeMap { &self.bidder_params } - pub(crate) fn prebid_zone(&self) -> Option<&str> { - self.prebid_zone.as_deref() + /// The zone the browser named for this slot, where one was admitted. + #[must_use] + pub fn zone(&self) -> Option<&str> { + self.zone.as_deref() } #[cfg(test)] @@ -191,38 +201,49 @@ impl ProviderSlotInput { self.bidder_params.is_empty() && self.allows_stored_fallback() } - pub(crate) fn allows_stored_fallback(&self) -> bool { + /// Whether a stored request may stand in for this slot when it is left + /// with no usable inline bidder parameters. + #[must_use] + pub fn allows_stored_fallback(&self) -> bool { self.stored_request .allows_fallback(!self.bidder_params.is_empty()) } } -/// Request headers approved for later Prebid transport forwarding. +/// Request headers approved for forwarding to a demand source. /// /// Values remain as raw [`HeaderValue`] instances so non-ASCII bytes retain /// the same legacy handling. Client-supplied `X-Forwarded-For` is never read. #[derive(Debug, Clone, Default)] -pub(crate) struct PrebidTransportHeaders { +pub struct TransportHeaders { cookie: Option, user_agent: Option, referer: Option, accept_language: Option, } -impl PrebidTransportHeaders { - pub(crate) fn cookie(&self) -> Option<&HeaderValue> { +impl TransportHeaders { + /// The admitted `Cookie` header. + #[must_use] + pub fn cookie(&self) -> Option<&HeaderValue> { self.cookie.as_ref() } - pub(crate) fn user_agent(&self) -> Option<&HeaderValue> { + /// The admitted `User-Agent` header. + #[must_use] + pub fn user_agent(&self) -> Option<&HeaderValue> { self.user_agent.as_ref() } - pub(crate) fn referer(&self) -> Option<&HeaderValue> { + /// The admitted `Referer` header. + #[must_use] + pub fn referer(&self) -> Option<&HeaderValue> { self.referer.as_ref() } - pub(crate) fn accept_language(&self) -> Option<&HeaderValue> { + /// The admitted `Accept-Language` header. + #[must_use] + pub fn accept_language(&self) -> Option<&HeaderValue> { self.accept_language.as_ref() } @@ -286,14 +307,14 @@ impl StoredRequestIntent { struct NormalizedSlotDemand { bidder_params: BTreeMap, stored_request: StoredRequestIntent, - prebid_zone: Option, + zone: Option, } #[derive(Debug)] struct ProviderInputBuilder { provider_id: ProviderId, timeout_ms: u32, - is_prebid: bool, + serves_stored_requests: bool, routing: RoutingMode, slots: Vec, } @@ -330,7 +351,7 @@ pub(crate) fn route_auction_with_trusted_routes( ) -> RoutedAuction { let slots = std::mem::take(&mut request.slots); let common_request = request; - let transport_headers = PrebidTransportHeaders::snapshot(inbound_request); + let transport_headers = TransportHeaders::snapshot(inbound_request); let dnt = inbound_request .headers() .get("dnt") @@ -344,7 +365,7 @@ pub(crate) fn route_auction_with_trusted_routes( .map(|provider| ProviderInputBuilder { provider_id: provider.id.clone(), timeout_ms: provider.timeout_ms, - is_prebid: provider.profile.is_prebid_server(), + serves_stored_requests: provider.implementation.serves_stored_requests, routing: provider.routing, slots: Vec::new(), }) @@ -385,7 +406,7 @@ pub(crate) fn route_auction_with_trusted_routes( let trusted_route = trusted_provider_indices.contains(&provider_index); let include = !bidder_params.is_empty() || trusted_route - || if builder.is_prebid { + || if builder.serves_stored_requests { demand.stored_request.allows_fallback(false) } else { builder.routing == RoutingMode::AllEligible @@ -396,9 +417,9 @@ pub(crate) fn route_auction_with_trusted_routes( builder.slots.push(ProviderSlotInput { slot: common_slot.clone(), bidder_params, - prebid_zone: builder - .is_prebid - .then(|| demand.prebid_zone.clone()) + zone: builder + .serves_stored_requests + .then(|| demand.zone.clone()) .flatten(), stored_request: demand.stored_request, }); @@ -513,9 +534,9 @@ fn normalize_envelope(envelope: &Value) -> Option { }) { return None; } - let prebid_zone = match object.get(ZONE_FIELD) { + let zone = match object.get(ZONE_FIELD) { None => None, - Some(Value::String(zone)) if zone.len() <= MAX_PREBID_ZONE_BYTES => Some(zone.clone()), + Some(Value::String(zone)) if zone.len() <= MAX_ZONE_BYTES => Some(zone.clone()), Some(_) => return None, }; let params = match object.get(BIDDER_PARAMS_FIELD) { @@ -533,7 +554,7 @@ fn normalize_envelope(envelope: &Value) -> Option { let Some(params) = params else { return Some(NormalizedSlotDemand { stored_request, - prebid_zone, + zone, ..Default::default() }); }; @@ -552,7 +573,7 @@ fn normalize_envelope(envelope: &Value) -> Option { Some(NormalizedSlotDemand { bidder_params, stored_request, - prebid_zone, + zone, }) } @@ -565,97 +586,68 @@ mod tests { use std::str::FromStr as _; use super::*; - use crate::auction::plan::{ - AuctionPlanConfig, BidderRouteConfig, NotificationConfig, ProviderConfig, - }; + use crate::auction::plan::BidderRouteConfig; + use crate::auction::test_support::{demand_table, plan_config}; use crate::auction::types::{AdFormat, DeviceInfo, PublisherInfo, SiteInfo, UserInfo}; use http::HeaderName; use serde_json::{Map, json}; - fn provider(profile: &str, routing: RoutingMode) -> ProviderConfig { - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: profile.to_string(), - endpoint: format!("https://{profile}.example.test/openrtb"), - timeout_ms: None, - routing, - notifications: NotificationConfig::default(), - profile_config: if profile == "aps" { - json!({"account_id": "example-account"}) - } else { - json!({}) - }, + fn provider(implementation: &str, routing: RoutingMode) -> Map { + let mut table = demand_table( + implementation, + &format!("https://{implementation}.example.test/openrtb"), + ); + if routing == RoutingMode::AllEligible { + table.insert("routing".to_string(), json!("all_eligible")); } + table + } + + fn routing_plan_config(open_routing: RoutingMode) -> crate::auction::plan::AuctionPlanConfig { + let mut config = plan_config(vec![ + ( + "open_primary", + provider("auction.plain-fixture", open_routing), + ), + ( + "stored_a", + provider("auction.fixture", RoutingMode::Explicit), + ), + ( + "stored_b", + provider("auction.fixture", RoutingMode::Explicit), + ), + ( + "openrtb_direct", + provider("auction.plain-fixture", RoutingMode::Explicit), + ), + ]); + config.timeout_ms = 900; + config } fn plan() -> AuctionPlan { - AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 900, - providers: BTreeMap::from([ - ( - ProviderId::from_str("aps-primary").expect("should parse provider"), - provider("aps", RoutingMode::AllEligible), - ), - ( - ProviderId::from_str("pbs-a").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("pbs-b").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("standard-direct").expect("should parse provider"), - provider("standard", RoutingMode::Explicit), - ), - ]), - bidders: BTreeMap::from([ - ( - BidderId::from_str("alpha").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("pbs-a").expect("should parse provider"), - }, - ), - ( - BidderId::from_str("beta").expect("should parse bidder"), - BidderRouteConfig { - provider: ProviderId::from_str("standard-direct") - .expect("should parse provider"), - }, - ), - ]), - mediator: None, - request_signing: None, - }) - .expect("should compile plan") + let mut config = routing_plan_config(RoutingMode::AllEligible); + config.bidders = BTreeMap::from([ + ( + BidderId::from_str("alpha").expect("should parse bidder"), + BidderRouteConfig { + module: ProviderId::from_str("stored_a").expect("should parse provider"), + }, + ), + ( + BidderId::from_str("beta").expect("should parse bidder"), + BidderRouteConfig { + module: ProviderId::from_str("openrtb_direct").expect("should parse provider"), + }, + ), + ]); + AuctionPlan::compile(config).expect("should compile plan") } fn explicit_plan() -> AuctionPlan { - AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 900, - providers: BTreeMap::from([ - ( - ProviderId::from_str("aps-primary").expect("should parse provider"), - provider("aps", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("pbs-a").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("pbs-b").expect("should parse provider"), - provider("prebid-server", RoutingMode::Explicit), - ), - ( - ProviderId::from_str("standard-direct").expect("should parse provider"), - provider("standard", RoutingMode::Explicit), - ), - ]), - bidders: BTreeMap::new(), - mediator: None, - request_signing: None, - }) - .expect("should compile explicit plan") + AuctionPlan::compile(routing_plan_config(RoutingMode::Explicit)) + .expect("should compile explicit plan") } fn slot(bidders: HashMap) -> AdSlot { @@ -689,6 +681,7 @@ mod tests { user_agent: None, ip: None, geo: None, + attributes: None, }), site: Some(SiteInfo { domain: "publisher.example.test".to_string(), @@ -722,8 +715,8 @@ mod tests { } #[test] - fn pbs_admission_respects_intent_without_changing_aps_eligibility() { - for (params, pbs_ids) in [ + fn stored_admission_respects_intent_without_changing_all_eligible_routing() { + for (params, stored_ids) in [ (json!({"bidderParams":{}, "storedRequest":false}), vec![]), ( json!({"bidderParams":{"unknown":{"id":1}}, "storedRequest":false}), @@ -732,18 +725,18 @@ mod tests { (json!({"bidderParams":{"unknown":{"id":1}}}), vec![]), ( json!({"bidderParams":{"alpha":{}}, "storedRequest":false}), - vec!["pbs-a"], + vec!["stored_a"], ), - (json!({"bidderParams":{"alpha":{}}}), vec!["pbs-a"]), + (json!({"bidderParams":{"alpha":{}}}), vec!["stored_a"]), ( json!({"bidderParams":{"alpha":{"id":1}}, "storedRequest":true}), - vec!["pbs-a", "pbs-b"], + vec!["stored_a", "stored_b"], ), ( json!({"bidderParams":{}, "storedRequest":true}), - vec!["pbs-a", "pbs-b"], + vec!["stored_a", "stored_b"], ), - (json!({"bidderParams":{}}), vec!["pbs-a", "pbs-b"]), + (json!({"bidderParams":{}}), vec!["stored_a", "stored_b"]), ] { let routed = route_auction( request(vec![slot(HashMap::from([( @@ -754,8 +747,8 @@ mod tests { &plan(), None, ); - let mut expected = vec!["aps-primary"]; - expected.extend(pbs_ids); + let mut expected = vec!["open_primary"]; + expected.extend(stored_ids); assert_eq!( routed .inputs() @@ -790,11 +783,11 @@ mod tests { route_auction(request(vec![slot(bidders)]), &inbound(), &plan(), None); assert_eq!(routed.diagnostics().malformed_envelope_count(), 1); assert_eq!(routed.inputs().len(), if direct { 2 } else { 1 }); - assert_eq!(routed.inputs()[0].provider_id().as_str(), "aps-primary"); + assert_eq!(routed.inputs()[0].provider_id().as_str(), "open_primary"); if direct { - let slot = &input(&routed, "pbs-a").slots()[0]; + let slot = &input(&routed, "stored_a").slots()[0]; assert!(!slot.allows_stored_fallback()); - assert_eq!(slot.prebid_zone(), None); + assert_eq!(slot.zone(), None); assert_eq!( slot.bidder_params().values().collect::>(), vec![&json!({"direct":1})] @@ -810,11 +803,11 @@ mod tests { for (params, expected) in [ ( json!({"bidderParams": {}, "storedRequest": true}), - vec!["aps-primary", "pbs-a", "pbs-b"], + vec!["open_primary", "stored_a", "stored_b"], ), ( json!({"bidderParams": {"alpha": {"placement": 1}}, "storedRequest": false}), - vec!["aps-primary", "pbs-a"], + vec!["open_primary", "stored_a"], ), ] { let routed = route_auction( @@ -862,16 +855,16 @@ mod tests { .collect::>(); assert_eq!( ids, - vec!["aps-primary", "pbs-a", "pbs-b"], - "{name} should fan out to both PBS providers while APS remains all-eligible" + vec!["open_primary", "stored_a", "stored_b"], + "{name} should fan out to both stored-request sources while the all-eligible source keeps every slot" ); assert!( - input(&routed, "pbs-a").slots()[0].allows_stored_fallback_without_candidates(), + input(&routed, "stored_a").slots()[0].allows_stored_fallback_without_candidates(), "{name} should create stored intent" ); assert!( - input(&routed, "pbs-b").slots()[0].allows_stored_fallback_without_candidates(), - "{name} should create stored intent for every PBS provider" + input(&routed, "stored_b").slots()[0].allows_stored_fallback_without_candidates(), + "{name} should create stored intent for every stored-request source" ); } } @@ -885,12 +878,12 @@ mod tests { None, ); assert!( - input(&routed, "pbs-a").slots()[0].allows_stored_fallback_without_candidates(), + input(&routed, "stored_a").slots()[0].allows_stored_fallback_without_candidates(), "empty canonical demand should preserve stored-request behavior" ); assert!( - input(&routed, "pbs-b").slots()[0].allows_stored_fallback_without_candidates(), - "empty canonical demand should fan out to same-profile PBS plans" + input(&routed, "stored_b").slots()[0].allows_stored_fallback_without_candidates(), + "empty canonical demand should fan out to the sources of one implementation" ); } @@ -921,7 +914,7 @@ mod tests { ("too many bidders", envelope(Some(too_many))), ( "oversized zone", - json!({"bidderParams": {}, "zone": "z".repeat(MAX_PREBID_ZONE_BYTES + 1)}), + json!({"bidderParams": {}, "zone": "z".repeat(MAX_ZONE_BYTES + 1)}), ), ("nonstring zone", json!({"bidderParams": {}, "zone": 1})), ]; @@ -938,13 +931,13 @@ mod tests { ); assert!( routed.inputs().iter().all(|provider| { - provider.provider_id().as_str() != "pbs-a" - && provider.provider_id().as_str() != "pbs-b" + provider.provider_id().as_str() != "stored_a" + && provider.provider_id().as_str() != "stored_b" }), - "{name} should not produce stored or inline PBS demand" + "{name} should not produce stored or inline demand" ); assert_eq!( - input(&routed, "standard-direct").slots()[0] + input(&routed, "openrtb_direct").slots()[0] .bidder_params() .len(), 1, @@ -954,7 +947,7 @@ mod tests { routed .inputs() .iter() - .any(|provider| provider.provider_id().as_str() == "aps-primary"), + .any(|provider| provider.provider_id().as_str() == "open_primary"), "{name} should preserve independent all-eligible participation" ); } @@ -1025,12 +1018,12 @@ mod tests { assert_eq!( routed.inputs().len(), 1, - "only all-eligible APS should remain" + "only the all-eligible source should remain" ); assert_eq!( routed.inputs()[0].provider_id().as_str(), - "aps-primary", - "unknown demand should not cause PBS fallback" + "open_primary", + "unknown demand should not cause a stored fallback" ); } @@ -1055,7 +1048,7 @@ mod tests { None, ); assert_eq!( - input(&routed, "pbs-a").slots()[0].bidder_params() + input(&routed, "stored_a").slots()[0].bidder_params() [&BidderId::from_str("alpha").expect("should parse bidder")]["source"], expected_source, "{name} should follow deterministic collision semantics" @@ -1110,7 +1103,7 @@ mod tests { TRUSTED_SERVER_ENVELOPE.to_string(), json!({"zone": "home", "bidderParams": null}), ), - ("alpha".to_string(), json!({"placement": "pbs"})), + ("alpha".to_string(), json!({"placement": "stored"})), ("beta".to_string(), json!({"placement": "direct"})), ]))]), &inbound(), @@ -1124,31 +1117,35 @@ mod tests { .collect::>(); assert_eq!( ids, - vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + vec!["open_primary", "stored_a", "stored_b", "openrtb_direct"], "inputs should follow deterministic provider-ID order" ); - let aps = input(&routed, "aps-primary") + let open = input(&routed, "open_primary") .slots() .first() .expect("should have slot"); assert!( - aps.bidder_params().is_empty(), - "APS must receive no foreign params" + open.bidder_params().is_empty(), + "a source that takes no bidder params must receive none" + ); + assert_eq!( + open.zone(), + None, + "a source that takes no bidder params must receive no zone" ); - assert_eq!(aps.prebid_zone(), None, "APS must receive no Prebid zone"); - let pbs_a = &input(&routed, "pbs-a").slots()[0]; - assert_eq!(pbs_a.bidder_params().len(), 1); + let stored_a = &input(&routed, "stored_a").slots()[0]; + assert_eq!(stored_a.bidder_params().len(), 1); assert!( - !pbs_a.allows_stored_fallback_without_candidates(), - "inline params should win for this PBS provider" + !stored_a.allows_stored_fallback_without_candidates(), + "inline params should win for this source" ); - assert_eq!(pbs_a.prebid_zone(), Some("home")); - let pbs_b = &input(&routed, "pbs-b").slots()[0]; - assert!(pbs_b.bidder_params().is_empty()); - assert!(pbs_b.allows_stored_fallback_without_candidates()); - let direct = &input(&routed, "standard-direct").slots()[0]; + assert_eq!(stored_a.zone(), Some("home")); + let stored_b = &input(&routed, "stored_b").slots()[0]; + assert!(stored_b.bidder_params().is_empty()); + assert!(stored_b.allows_stored_fallback_without_candidates()); + let direct = &input(&routed, "openrtb_direct").slots()[0]; assert_eq!(direct.bidder_params().len(), 1); - assert!(direct.prebid_zone().is_none()); + assert!(direct.zone().is_none()); for provider in routed.inputs() { assert!(provider.common_request().slots.is_empty()); assert!(provider.slots()[0].slot().bidders.is_empty()); @@ -1198,7 +1195,7 @@ mod tests { assert_eq!( routed.inputs().len(), 3, - "APS and two PBS providers should receive the eligible slot" + "the all-eligible source and two stored-request sources should receive the eligible slot" ); for provider in routed.inputs() { assert_eq!(provider.slots().len(), 1); @@ -1225,7 +1222,7 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + vec!["open_primary", "stored_a", "stored_b", "openrtb_direct"], "no-banner auction should retain every provider's deterministic skip outcome" ); } @@ -1247,17 +1244,17 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["pbs-a", "pbs-b", "standard-direct"], + vec!["stored_a", "stored_b", "openrtb_direct"], "explicit providers with no routed demand should be retained as skipped" ); } #[test] - fn trusted_routes_admit_explicit_aps_and_standard_and_ignore_unknown_provider() { + fn trusted_routes_admit_explicit_sources_and_ignore_unknown_provider() { let trusted_routes = TrustedProviderRoutes::new(vec![vec![ - ProviderId::from_str("aps-primary").expect("should parse provider"), - ProviderId::from_str("standard-direct").expect("should parse provider"), - ProviderId::from_str("unknown-provider").expect("should parse provider"), + ProviderId::from_str("open_primary").expect("should parse provider"), + ProviderId::from_str("openrtb_direct").expect("should parse provider"), + ProviderId::from_str("unknown_provider").expect("should parse provider"), ]]); let routed = route_auction_with_trusted_routes( request(vec![slot(HashMap::from([( @@ -1275,7 +1272,7 @@ mod tests { .iter() .map(|input| input.provider_id().as_str()) .collect::>(), - vec!["aps-primary", "standard-direct"], + vec!["open_primary", "openrtb_direct"], "only known server-owned provider routes should admit explicit providers" ); assert_eq!( @@ -1289,7 +1286,7 @@ mod tests { .iter() .map(ProviderId::as_str) .collect::>(), - vec!["pbs-a", "pbs-b"], + vec!["stored_a", "stored_b"], "unrouted explicit providers should retain skip outcomes" ); } @@ -1335,7 +1332,7 @@ mod tests { &plan(), Some(attested), ); - let headers = routed.prebid_transport_headers(); + let headers = routed.transport_headers(); assert_eq!(headers.cookie(), Some(&HeaderValue::from_static("first=1"))); assert_eq!( headers.user_agent().expect("should retain UA").as_bytes(), @@ -1355,8 +1352,8 @@ mod tests { assert_eq!(routed.dnt(), Some(true)); for provider in routed.inputs() { let expected_timeout = match provider.provider_id().as_str() { - "aps-primary" => 800, - id if id.starts_with("pbs-") => 1000, + "open_primary" => 900, + id if id.starts_with("stored_") => 1000, _ => 900, }; assert_eq!(provider.timeout_ms(), expected_timeout); diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index e0a819ba38..cebd063102 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -76,7 +76,7 @@ impl AuctionTerminalStatus { pub struct AbandonedProviderCall { /// Provider name. pub provider: String, - /// Provider role, usually `bidder` or `mediator`. + /// Provider role, usually `bidder` or `adserver`. pub provider_role: &'static str, /// Optional elapsed time for this provider at abandonment. pub response_time_ms: Option, @@ -338,7 +338,7 @@ pub struct AuctionEventRow { pub winning_bid_count: Option, /// Provider name. pub provider: Option, - /// `bidder` or `mediator`. + /// `bidder` or `adserver`. pub provider_role: Option, /// Provider-call status. pub status: Option, @@ -576,13 +576,13 @@ pub fn build_auction_events( &result.provider_responses, "bidder", ); - if let Some(mediator_response) = &result.mediator_response { + if let Some(adserver_response) = &result.adserver_response { push_provider_row( &mut rows, &observation, &event_ts, - mediator_response, - "mediator", + adserver_response, + "adserver", ); } push_bid_rows( @@ -775,14 +775,14 @@ fn push_bid_rows( } } - if let Some(mediator_response) = &result.mediator_response { + if let Some(adserver_response) = &result.adserver_response { for (slot_id, winning) in &result.winning_bids { if delivered_winner_slots.is_some_and(|slots| !slots.contains(slot_id)) || matched_wins.contains(slot_id) { continue; } - if mediator_response + if adserver_response .bids .iter() .any(|bid| bid_matches_winning_bid(bid, winning)) @@ -791,7 +791,7 @@ fn push_bid_rows( observation, event_ts, request, - &mediator_response.provider, + &adserver_response.provider, winning, 1, winning.price, @@ -1129,6 +1129,7 @@ mod tests { user_agent: Some(TEST_USER_AGENT.to_owned()), ip: None, geo: None, + attributes: None, }); let from_request = AuctionObservationContext::from_auction_request( @@ -1208,7 +1209,7 @@ mod tests { let winning = provider_success.bids[0].clone(); let result = OrchestrationResult { provider_responses: vec![provider_success, provider_no_bid, provider_error], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), winning)]), total_time_ms: 99, metadata: HashMap::new(), @@ -1274,7 +1275,7 @@ mod tests { let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); let result = OrchestrationResult { provider_responses: vec![provider], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), total_time_ms: 12, metadata: HashMap::new(), @@ -1310,7 +1311,7 @@ mod tests { vec![fallback_bid.clone()], 12, )], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), fallback_bid)]), total_time_ms: 12, metadata: HashMap::new(), @@ -1334,15 +1335,15 @@ mod tests { } #[test] - fn mediated_aps_telemetry_retains_provider_upstream_seat_and_delivery_identity() { + fn adserver_aps_telemetry_retains_provider_upstream_seat_and_delivery_identity() { let request = test_request("ts-ec-derived-id"); let mut aps_bid = bid("slot-1", "aps", Some("ad-1"), Some(1.25)); aps_bid.returned_seat = Some("upstream-seat".to_string()); let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); - let mediator = AuctionResponse::success("adserver_mock", vec![aps_bid.clone()], 3); + let adserver = AuctionResponse::success("adserver_mock", vec![aps_bid.clone()], 3); let result = OrchestrationResult { provider_responses: vec![provider], - mediator_response: Some(mediator), + adserver_response: Some(adserver), winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), total_time_ms: 15, metadata: HashMap::new(), @@ -1382,7 +1383,7 @@ mod tests { ); let result = OrchestrationResult { provider_responses: vec![provider_success.clone()], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::from([("slot-1".to_owned(), provider_success.bids[0].clone())]), total_time_ms: 42, metadata: HashMap::new(), @@ -1423,7 +1424,7 @@ mod tests { .with_metadata("status", json!(403)); let result = OrchestrationResult { provider_responses: vec![provider_http_error], - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 12, metadata: HashMap::new(), @@ -1452,17 +1453,17 @@ mod tests { } #[test] - fn mediated_win_marks_original_bid_once() { + fn adserver_win_marks_original_bid_once() { let request = test_request("req"); let original_bid = bid("slot-1", "kargo", Some("ad-1"), None); let provider_success = AuctionResponse::success("prebid", vec![original_bid], 42); - let mediator_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); - let mediator_response = - AuctionResponse::success("adserver_mock", vec![mediator_bid.clone()], 15); + let adserver_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); + let adserver_response = + AuctionResponse::success("adserver_mock", vec![adserver_bid.clone()], 15); let result = OrchestrationResult { provider_responses: vec![provider_success], - mediator_response: Some(mediator_response), - winning_bids: HashMap::from([("slot-1".to_owned(), mediator_bid)]), + adserver_response: Some(adserver_response), + winning_bids: HashMap::from([("slot-1".to_owned(), adserver_bid)]), total_time_ms: 80, metadata: HashMap::new(), }; @@ -1487,12 +1488,12 @@ mod tests { assert_eq!( winning_rows[0].provider.as_deref(), Some("prebid"), - "should mark original provider row when mediator winner matches" + "should mark original provider row when adserver winner matches" ); assert_eq!( winning_rows[0].price_cpm, Some(2.0), - "should copy mediator decoded price onto original null-price bid" + "should copy adserver decoded price onto original null-price bid" ); } @@ -1501,7 +1502,7 @@ mod tests { let request = test_request("ts-ec-derived-id"); let result = OrchestrationResult { provider_responses: Vec::new(), - mediator_response: None, + adserver_response: None, winning_bids: HashMap::new(), total_time_ms: 1, metadata: HashMap::new(), diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..ee6f77ea7b 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -1,3 +1,6 @@ +//! Helpers for tests of the auction, in core and in an implementation's own +//! crate. + use std::collections::HashMap; use std::sync::LazyLock; @@ -17,7 +20,8 @@ use crate::settings::Settings; static TEST_SERVICES: LazyLock = LazyLock::new(noop_services); -pub(crate) fn create_test_auction_context<'a>( +/// An auction context over services that do nothing. +pub fn create_test_auction_context<'a>( settings: &'a Settings, request: &'a Request, timeout_ms: u32, @@ -33,13 +37,13 @@ pub(crate) fn create_test_auction_context<'a>( } } -/// Build canonical request facts shared by the PBS and APS Stage 1 wire goldens. +/// Build canonical request facts shared by the implementations' wire goldens. /// -/// The supported and unsupported formats deliberately exercise each profile's -/// existing filtering and field-ownership policy. `trustedServer` bidder -/// parameters are included to pin that PBS consumes them while APS ignores -/// them. -pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { +/// The supported and unsupported formats deliberately exercise each +/// implementation's filtering and field-ownership policy. `trustedServer` +/// bidder parameters are included to pin which implementation consumes them +/// and which ignores them. +pub fn canonical_parity_auction_request() -> AuctionRequest { AuctionRequest { id: "fictional-auction".to_string(), slots: vec![AdSlot { @@ -114,8 +118,648 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { region: Some("CA".to_string()), asn: None, }), + attributes: None, }), site: None, context: HashMap::new(), } } + +/// The signing input the request goldens were captured with. +pub fn golden_signing_params() -> crate::request_signing::SigningParams { + crate::request_signing::SigningParams { + request_id: "fictional-auction".to_string(), + request_host: "publisher.example".to_string(), + request_scheme: "https".to_string(), + timestamp: 1_706_900_000, + } +} + +/// The one-source plan the request goldens were captured with: a source +/// called `fictional_provider`, running `implementation` with `settings`. +/// +/// The source takes every eligible slot when `all_eligible` is set. Otherwise +/// the bidder `exampleBidder` is routed to it, which is how a source that +/// takes only routed bidders is reached. +pub fn golden_plan_config( + implementation: &str, + settings: serde_json::Value, + all_eligible: bool, + extra: &[crate::integrations::IntegrationBuilder], +) -> crate::auction::plan::AuctionPlanConfig { + let mut table = demand_table(implementation, "https://exchange.example.test/openrtb"); + table.insert("timeout_ms".to_string(), json!(321)); + if all_eligible { + table.insert("routing".to_string(), json!("all_eligible")); + } + if let serde_json::Value::Object(settings) = settings { + table.extend(settings); + } + let mut config = plan_config_with(vec![("fictional_provider", table)], extra); + config.timeout_ms = 321; + if !all_eligible { + config.bidders.insert( + "exampleBidder".parse().expect("should parse bidder ID"), + crate::auction::plan::BidderRouteConfig { + module: "fictional_provider" + .parse() + .expect("should parse provider ID"), + }, + ); + } + config +} + +/// The inbound request the request goldens were captured with. +pub fn golden_inbound_request() -> Request { + Request::builder() + .uri("https://publisher.example/auction") + .header( + http::header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request") +} + +/// A signer over one fixed key, so a signed golden is the same on every run. +pub fn deterministic_signer() -> crate::request_signing::RequestSigner { + use base64::Engine as _; + + use crate::platform::test_support::{ + HashMapConfigStore, HashMapSecretStore, NoopHttpClient, + build_services_with_config_secret_and_http_client, + }; + + let config_data = HashMap::from([("current-kid".to_string(), "fictional-kid".to_string())]); + let secret_data = HashMap::from([( + "fictional-kid".to_string(), + base64::engine::general_purpose::STANDARD + .encode([7_u8; 32]) + .into_bytes(), + )]); + let services = build_services_with_config_secret_and_http_client( + HashMapConfigStore::new(config_data), + HashMapSecretStore::new(secret_data), + std::sync::Arc::new(NoopHttpClient), + ); + crate::request_signing::RequestSigner::from_services(&services) + .expect("should load deterministic signer") +} + +/// Builds the request the driver would send to the first demand source of +/// `plan` once `request` is routed to it, or `None` when the source keeps no +/// impression. +/// +/// A test in an implementation's own crate checks what its field policy and +/// its extensions do to a request through this. The signature, when `signer` +/// is given, is made over [`golden_signing_params`]. +/// +/// # Errors +/// +/// Returns the error the request builder returns. +pub fn build_for_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, + transport_timeout_ms: u32, + signer: Option<&crate::request_signing::RequestSigner>, +) -> Result< + Option, + error_stack::Report, +> { + use crate::auction::openrtb::{OpenRtbBuildOutcome, RequestFinalization, build_request}; + + let routed = crate::auction::routing::route_auction(request, inbound, plan, None); + let outcome = build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + transport_timeout_ms, + &RequestFinalization { + signer, + signing_params: golden_signing_params(), + }, + )?; + Ok(match outcome { + OpenRtbBuildOutcome::Ready(request) => Some(request), + OpenRtbBuildOutcome::NoImpressions => None, + }) +} + +/// Routes `request` through `plan` and returns the input its first demand +/// source receives, or `None` when no source takes a slot. +/// +/// A test in an implementation's own crate hands this input to its own +/// response reader. +#[must_use] +pub fn route_to_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, +) -> Option { + crate::auction::routing::route_auction(request, inbound, plan, None) + .inputs() + .first() + .cloned() +} + +/// The transport headers a routed auction admits from `inbound`, with the +/// attested client address it carries to every source's outbound request. +/// +/// A test in an implementation's own crate checks what it forwards upstream +/// through this. +#[must_use] +pub fn routed_transport( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + inbound: &Request, + attested_client_ip: Option, +) -> ( + crate::auction::demand::TransportHeaders, + Option, +) { + let routed = crate::auction::routing::route_auction(request, inbound, plan, attested_client_ip); + ( + routed.transport_headers().clone(), + routed.attested_client_ip(), + ) +} + +/// Reads `response` as the first demand source of `plan` would, once +/// `request` is routed to it. +/// +/// A test in an implementation's own crate checks how its implementation +/// answers through this, with no upstream. +/// +/// # Errors +/// +/// Returns the error the source's own reading returns. +pub async fn parse_as_first_source( + plan: &crate::auction::AuctionPlan, + request: AuctionRequest, + response: crate::platform::PlatformResponse, + response_time_ms: u64, +) -> Result< + crate::auction::types::AuctionResponse, + error_stack::Report, +> { + let inbound = Request::new(EdgeBody::empty()); + let routed = crate::auction::routing::route_auction(request, &inbound, plan, None); + let provider = + crate::auction::provider::GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + provider + .parse_response_with_state(response, response_time_ms, Some(state.as_ref())) + .await +} + +/// One `[demand.]` table naming an implementation, for tests that need a +/// compiled plan. +pub fn demand_table( + implementation: &str, + endpoint: &str, +) -> serde_json::Map { + serde_json::Map::from_iter([ + ("implementation".to_string(), json!(implementation)), + ("endpoint".to_string(), json!(endpoint)), + ]) +} + +/// An `[demand]` table selecting every name given, in the order given. +pub fn demand_selection( + tables: Vec<(&str, serde_json::Map)>, +) -> crate::provider_table::ProviderList { + let selected = tables + .iter() + .map(|(name, _)| (*name).to_string()) + .collect::>(); + let tables = tables + .into_iter() + .map(|(name, table)| (name.to_string(), table)) + .collect::>(); + crate::provider_table::ProviderList::new(selected, tables) +} + +/// A plan configuration selecting the named demand tables, with every +/// implementation the built-in builders register. +pub fn plan_config( + tables: Vec<(&str, serde_json::Map)>, +) -> crate::auction::plan::AuctionPlanConfig { + plan_config_with(tables, &[]) +} + +/// A plan configuration selecting the named demand tables, with every +/// implementation the built-in builders register followed by the ones +/// `extra` registers, which is how a crate's own tests reach its +/// implementation. +pub fn plan_config_with( + tables: Vec<(&str, serde_json::Map)>, + extra: &[crate::integrations::IntegrationBuilder], +) -> crate::auction::plan::AuctionPlanConfig { + let builders = crate::integrations::all_builders(extra).collect::>(); + crate::auction::plan::AuctionPlanConfig { + timeout_ms: 1000, + demand: demand_selection(tables), + demand_implementations: builders + .iter() + .filter_map(crate::integrations::IntegrationBuilder::demand) + .collect(), + adserver_implementations: builders + .iter() + .filter_map(crate::integrations::IntegrationBuilder::adserver) + .collect(), + ..crate::auction::plan::AuctionPlanConfig::default() + } +} + +/// A `[demand]` selection of sources under the names given, each running +/// `implementation` at an endpoint of its own and taking every eligible slot. +pub fn demand_named(implementation: &str, names: &[&str]) -> crate::provider_table::ProviderList { + demand_selection( + names + .iter() + .map(|name| { + let mut table = demand_table( + implementation, + &format!("https://{name}.example/openrtb2/auction"), + ); + table.insert("routing".to_string(), json!("all_eligible")); + (*name, table) + }) + .collect(), + ) +} + +/// The legacy test orchestrator's configuration for these settings, carrying +/// the demand source names `[demand] modules` selects. +/// +/// Production compiles its sources from the plan instead, so this exists only +/// so the parity tests can drive the pre-plan orchestrator. +#[cfg(test)] +pub(crate) fn legacy_auction_config(settings: &Settings) -> crate::auction::AuctionConfig { + let mut config = settings.auction.clone(); + config.provider_names = settings + .demand + .selected() + .into_iter() + .map(str::to_string) + .collect(); + config +} + +/// A stand-in ad server implementation for core's own tests of the ad server +/// seam, selected with `[ad-server] module = "fixture"`. +/// +/// It reads an `endpoint`, which has to be a URL, and an optional +/// `timeout_ms` from its table and refuses any other setting, as an +/// implementation of a vendor's does. Its provider answers every request +/// with no bid and calls nothing. +#[cfg(test)] +pub(crate) mod adserver_fixture { + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + + use crate::auction::demand::AdServerImplementation; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name the stand-in is selected by, which `[ad-server]` shortens to + /// `fixture`. + pub(crate) const MODULE: &str = "ad-server.fixture"; + + /// The stand-in ad server implementation. + pub(crate) static ADSERVER: AdServerImplementation = + AdServerImplementation { id: MODULE, build }; + + #[derive(Debug, Deserialize)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + endpoint: String, + #[serde(default = "default_timeout_ms")] + timeout_ms: u32, + } + + fn default_timeout_ms() -> u32 { + 500 + } + + fn build( + name: &str, + settings: &serde_json::Map, + ) -> Result, Report> { + let settings: FixtureSettings = serde_json::from_value(serde_json::Value::Object( + settings.clone(), + )) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + url::Url::parse(&settings.endpoint).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` endpoint: {error}"), + }) + })?; + Ok(Arc::new(FixtureAdServer::new(name, settings.timeout_ms))) + } + + /// The stand-in's provider. + pub(crate) struct FixtureAdServer { + name: String, + timeout_ms: u32, + } + + impl FixtureAdServer { + /// A provider called `name`. + pub(crate) fn new(name: &str, timeout_ms: u32) -> Self { + Self { + name: name.to_owned(), + timeout_ms, + } + } + } + + #[async_trait(?Send)] + impl AuctionProvider for FixtureAdServer { + fn provider_name(&self) -> &str { + &self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.name.clone(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::no_bid(self.name.clone(), response_time_ms)) + } + + fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + } +} + +/// A stand-in demand implementation for core's own tests of what the driver +/// does for an implementation that departs from the plain one. It serves +/// stored requests, forbids all-eligible routing, takes bidder parameters, +/// keeps the unsigned request identity and completes an endpoint that names +/// a host alone. +/// +/// It writes each impression's usable bidder parameters under +/// `ext.fixture.bidder`, or the slot id under `ext.fixture.stored` where the +/// slot allows a stored request, and leaves out an impression that has +/// neither. +#[cfg(test)] +pub(crate) mod demand_fixture { + use core::any::Any; + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + use serde_json::{Map, Value, json}; + use url::Url; + + use crate::auction::demand::{ + CompiledDemand, DemandFieldPolicy, DemandImplementation, DemandResponse, + DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, + }; + use crate::auction::types::AuctionResponse; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name an `implementation` line gives the stand-in. + pub(crate) const MODULE: &str = "auction.fixture"; + + /// The path the stand-in completes a bare host with. + pub(crate) const AUCTION_PATH: &str = "/fixture/auction"; + + /// The stand-in demand implementation. + pub(crate) static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Fixed(1000), + allows_all_eligible: false, + serves_stored_requests: true, + canonicalize_endpoint, + compile, + }; + + /// One compiled stand-in source. + #[derive(Debug, Clone, Default)] + pub(crate) struct FixtureDemand; + + #[derive(Debug, Deserialize, Default)] + #[serde(deny_unknown_fields)] + struct FixtureSettings {} + + fn compile( + settings: &Map, + ) -> Result, Report> { + FixtureSettings::deserialize(Value::Object(settings.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + Ok(Arc::new(FixtureDemand)) + } + + fn canonicalize_endpoint(endpoint: &mut Url) -> Result<(), String> { + if matches!(endpoint.path(), "" | "/") { + endpoint.set_path(AUCTION_PATH); + } + Ok(()) + } + + #[async_trait(?Send)] + impl CompiledDemand for FixtureDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + imp_tagid: true, + site_ref: true, + precise_geo: true, + additional_consent: true, + unsigned_request_identity: true, + consumes_bidder_params: true, + regs: RegsPolicy::Jurisdiction, + ..DemandFieldPolicy::default() + } + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + for impression in &mut extensions.impressions { + let slot = impression.slot; + let bidder = slot + .bidder_params() + .iter() + .filter(|(_, params)| { + params.as_object().is_some_and(|params| !params.is_empty()) + }) + .map(|(bidder, params)| (bidder.as_str().to_string(), params.clone())) + .collect::>(); + let mut fixture = Map::new(); + if !bidder.is_empty() { + fixture.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.allows_stored_fallback() { + fixture.insert("stored".to_string(), json!(slot.slot().id)); + } + if fixture.is_empty() { + impression.omitted = true; + continue; + } + *impression.ext = Some(Map::from_iter([( + "fixture".to_string(), + Value::Object(fixture), + )])); + } + *extensions.request = Some(Map::from_iter([( + "fixture".to_string(), + Value::Object(Map::new()), + )])); + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + crate::auction::openrtb::parse_openrtb_response(context, response).await + } + + fn as_any(&self) -> &dyn Any { + self + } + } +} + +/// A stand-in for the plainest demand implementation there can be, which +/// core's own tests of the plan compiler and the request driver name where +/// they need a source. +/// +/// It takes every eligible slot, inherits the auction's timeout, leaves its +/// endpoint as written and has its response read the ordinary way. Its table +/// may set `request_ext` and `imp_ext`, which it writes into the request and +/// into every impression as given, so the driver's tests have an +/// implementation that adds extensions. +#[cfg(test)] +pub(crate) mod plain_fixture { + use core::any::Any; + use std::sync::Arc; + + use async_trait::async_trait; + use error_stack::Report; + use serde::Deserialize; + use serde_json::{Map, Value}; + + use crate::auction::demand::{ + CONSERVATIVE_LANGUAGE_MAX_BYTES, CompiledDemand, DemandFieldPolicy, DemandImplementation, + DemandResponse, DemandTimeoutDefault, ProviderAuctionInput, RegsPolicy, RequestExtensions, + accept_endpoint, + }; + use crate::auction::types::AuctionResponse; + use crate::error::TrustedServerError; + use crate::platform::PlatformResponse; + + /// The name an `implementation` line gives the stand-in. + pub(crate) const MODULE: &str = "auction.plain-fixture"; + + /// The stand-in demand implementation. + pub(crate) static DEMAND: DemandImplementation = DemandImplementation { + id: MODULE, + default_timeout: DemandTimeoutDefault::Auction, + allows_all_eligible: true, + serves_stored_requests: false, + canonicalize_endpoint: accept_endpoint, + compile, + }; + + /// One compiled stand-in source. + #[derive(Debug, Clone, Default)] + pub(crate) struct PlainDemand { + request_ext: Map, + imp_ext: Map, + } + + #[derive(Debug, Deserialize, Default)] + #[serde(deny_unknown_fields)] + struct PlainSettings { + #[serde(default)] + request_ext: Map, + #[serde(default)] + imp_ext: Map, + } + + fn compile( + settings: &Map, + ) -> Result, Report> { + let settings = + PlainSettings::deserialize(Value::Object(settings.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `{MODULE}` settings: {error}"), + }) + })?; + Ok(Arc::new(PlainDemand { + request_ext: settings.request_ext, + imp_ext: settings.imp_ext, + })) + } + + #[async_trait(?Send)] + impl CompiledDemand for PlainDemand { + fn field_policy(&self) -> DemandFieldPolicy { + DemandFieldPolicy { + language_max_bytes: Some(CONSERVATIVE_LANGUAGE_MAX_BYTES), + regs: RegsPolicy::Jurisdiction, + accept_json: true, + ..DemandFieldPolicy::default() + } + } + + fn augment_request( + &self, + extensions: &mut RequestExtensions<'_>, + _input: &ProviderAuctionInput, + ) -> Result<(), Report> { + *extensions.request = (!self.request_ext.is_empty()).then(|| self.request_ext.clone()); + for impression in &mut extensions.impressions { + *impression.ext = (!self.imp_ext.is_empty()).then(|| self.imp_ext.clone()); + } + Ok(()) + } + + async fn parse_response( + &self, + context: DemandResponse<'_>, + response: PlatformResponse, + ) -> Result> { + crate::auction::openrtb::parse_openrtb_response(context, response).await + } + + fn as_any(&self) -> &dyn Any { + self + } + } +} diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 4069157068..9d4bbcb47d 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -1,11 +1,14 @@ //! Core types for auction requests and responses. use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _, bail, ensure}; use http::Request; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use crate::auction::context::ContextValue; +use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -93,7 +96,7 @@ pub struct UserInfo { /// cookies/headers, not from stored data. #[serde(skip)] pub consent: Option, - /// Extended User IDs parsed from the [`crate::constants::COOKIE_TS_EIDS`] cookie. + /// Extended User IDs parsed from the [`crate::constants::COOKIE_EIDS`] cookie. /// /// Raw (un-gated) values from the browser; consent gating via /// [`crate::consent::gate_eids_by_consent`] is applied centrally in the @@ -110,6 +113,14 @@ pub struct DeviceInfo { pub user_agent: Option, pub ip: Option, pub geo: Option, + /// Attributes a device module resolved, for the bid request's device + /// object. + /// + /// Absent when no module resolved any, which is the default. Defaulted + /// on deserialize so a stored request from before this field existed still + /// reads. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attributes: Option, } /// Site information. @@ -129,14 +140,14 @@ pub struct SiteInfo { /// (DNT, User-Agent, cookies, X-* customs) directly off it. /// /// In the **collect path** ([`collect_dispatched_auction`][collect]) the -/// mediator is invoked with a synthetic placeholder request +/// ad server is invoked with a synthetic placeholder request /// (`https://placeholder.invalid/`), because the real client request has /// already been consumed by `send_async` during dispatch and the host pipeline -/// can't lend it across the `.await`. **Mediators must not depend on reading +/// can't lend it across the `.await`. **Ad servers must not depend on reading /// client state from `context.request`** — the placeholder has none of the -/// real headers. If a future mediator needs that data, snapshot it into a new +/// real headers. If a future ad server needs that data, snapshot it into a new /// field on this struct at dispatch time and stash it on the -/// [`DispatchedAuction`] token so collect can attach it to the mediator's +/// [`DispatchedAuction`] token so collect can attach it to the ad server's /// context. See /// (P2-1) for the open follow-up. /// @@ -154,15 +165,15 @@ pub struct AuctionContext<'a> { /// encode timers. Providers that register a backend should use this value /// for transport timers while retaining `timeout_ms` for logical policy. pub transport_timeout_ms: u32, - /// Provider responses from the bidding phase, used by mediators. - /// This is `None` for regular bidders and `Some` when calling a mediator. + /// Provider responses from the bidding phase, used by ad servers. + /// This is `None` for regular bidders and `Some` when calling an ad server. pub provider_responses: Option<&'a [AuctionResponse]>, /// Platform services (config store, secret store, etc.) for use by providers. pub services: &'a RuntimeServices, } -/// URL used by the orchestrator when invoking a mediator from the collect -/// path. Providers can `debug_assert` against this value to catch a mediator +/// URL used by the orchestrator when invoking an ad server from the collect +/// path. Providers can `debug_assert` against this value to catch an ad server /// that has accidentally started depending on `context.request` carrying real /// client headers. pub const MEDIATOR_PLACEHOLDER_URL: &str = "https://placeholder.invalid/"; @@ -182,56 +193,203 @@ pub struct AuctionResponse { pub metadata: HashMap, } -/// APS creative tag type accepted by the Trusted Server renderer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ApsTagType { - /// APS loads the creative URL in a nested iframe. - Iframe, - /// APS fetches creative HTML and executes it in its nested renderer frame. - Script, -} - -/// Version 1 APS renderer descriptor shared with browser clients. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ApsRendererV1 { - /// Renderer contract version. - pub version: u8, - /// APS account identifier used to initialize the fixed runner. - pub account_id: String, - /// Selected `OpenRTB` bid identifier. - pub bid_id: String, - /// Optional `OpenRTB` creative identifier. - #[serde(skip_serializing_if = "Option::is_none")] - pub creative_id: Option, - /// APS creative delivery mode. - pub tag_type: ApsTagType, - /// HTTPS creative URL consumed by the fixed APS runner. - pub creative_url: String, - /// Base64-encoded exact one-bid APS response envelope. - pub aax_response: String, - /// Creative width. - pub width: u32, - /// Creative height. - pub height: u32, -} +/// Wire key carrying the renderer type tag. +/// +/// A payload may not use this key, since it would collide with the tag when +/// the descriptor is serialized flat. +const RENDERER_TYPE_KEY: &str = "type"; -/// Typed browser renderer capability carried by a bid. +/// Browser renderer capability carried by a bid: a type tag and the payload +/// the auction provider that produced the bid defines. +/// +/// Serialized flat, as `{"type": "", ...payload}`, so a page receives +/// the same bytes whether the provider lives in core or in its own crate. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum BidRenderer { - /// APS renderer version 1. - Aps(ApsRendererV1), +pub struct BidRenderer { + #[serde(rename = "type")] + renderer_type: String, + #[serde(flatten)] + payload: serde_json::Map, + /// The payload key the code that built the descriptor said holds the + /// identifier the renderer picks its bid by. A statement about the + /// payload and no part of what is sent. + #[serde(skip)] + bid_id_key: Option<&'static str>, } impl BidRenderer { - /// Return the APS renderer descriptor when this is an APS renderer. + /// Build a descriptor from a type tag and the provider's JSON payload. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Auction`] when `payload` is not a JSON + /// object, or when it carries its own `type` key, which would collide with + /// the tag once the descriptor is serialized flat. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let renderer = BidRenderer::new("example", json!({ "version": 1 })) + /// .expect("should accept an object payload"); + /// assert_eq!(renderer.renderer_type(), "example"); + /// ``` + pub fn new( + renderer_type: &str, + payload: serde_json::Value, + ) -> Result> { + let serde_json::Value::Object(payload) = payload else { + bail!(TrustedServerError::Auction { + message: format!("Renderer '{renderer_type}' payload must be a JSON object"), + }); + }; + ensure!( + !payload.contains_key(RENDERER_TYPE_KEY), + TrustedServerError::Auction { + message: format!( + "Renderer '{renderer_type}' payload must not carry a '{RENDERER_TYPE_KEY}' key" + ), + } + ); + Ok(Self { + renderer_type: renderer_type.to_string(), + payload, + bid_id_key: None, + }) + } + + /// Build a descriptor by serializing a provider's own payload type. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Auction`] when `payload` cannot be + /// serialized, and when the serialized form is rejected by + /// [`new`](Self::new). + /// + /// # Examples + /// + /// ``` + /// use serde::Serialize; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// #[derive(Serialize)] + /// struct ExampleRendererV1 { + /// version: u8, + /// } + /// + /// let renderer = BidRenderer::from_typed("example", &ExampleRendererV1 { version: 1 }) + /// .expect("should accept a struct payload"); + /// assert_eq!(renderer.renderer_type(), "example"); + /// ``` + pub fn from_typed( + renderer_type: &str, + payload: &T, + ) -> Result> { + let payload = + serde_json::to_value(payload).change_context(TrustedServerError::Auction { + message: format!("Failed to serialize renderer '{renderer_type}' payload"), + })?; + Self::new(renderer_type, payload) + } + + /// Return the renderer type tag a page reads to select its renderer. + #[must_use] + pub fn renderer_type(&self) -> &str { + &self.renderer_type + } + + /// Deserialize the payload into the provider's own descriptor type. + /// + /// Returns `None` when the descriptor carries a different tag, and when the + /// payload does not match `T`. + /// + /// Clones the whole payload map and deserializes all of it, so use + /// [`payload_field`](Self::payload_field) when the caller wants one field. + /// An APS payload carries a base64 encoding of a creative envelope of up + /// to 256 KB. + #[must_use] + pub fn payload_as(&self, renderer_type: &str) -> Option { + if self.renderer_type != renderer_type { + return None; + } + serde_json::from_value(serde_json::Value::Object(self.payload.clone())).ok() + } + + /// Borrow one field of the payload, copying nothing. + /// + /// Returns `None` when the descriptor carries a different tag, and when + /// the payload has no such key. `key` is the wire key, so a payload type + /// that renames its fields for serialization must be asked for the + /// renamed form. + /// + /// Unlike [`payload_as`](Self::payload_as) this reads the one field + /// asked for and does not check that the rest of the payload matches the + /// provider's descriptor type. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let renderer = BidRenderer::new("example", json!({ "bidId": "fictional-bid-id" })) + /// .expect("should accept an object payload"); + /// + /// assert_eq!( + /// renderer + /// .payload_field("example", "bidId") + /// .and_then(serde_json::Value::as_str), + /// Some("fictional-bid-id"), + /// ); + /// assert!(renderer.payload_field("other", "bidId").is_none()); + /// ``` #[must_use] - pub fn as_aps(&self) -> Option<&ApsRendererV1> { - match self { - Self::Aps(renderer) => Some(renderer), + pub fn payload_field(&self, renderer_type: &str, key: &str) -> Option<&serde_json::Value> { + if self.renderer_type != renderer_type { + return None; } + self.payload.get(key) + } + + /// States that the payload key `key` holds the identifier this renderer + /// picks its bid by. + /// + /// The statement is the builder's own about its payload. A descriptor + /// built without it has no such identifier, whatever its payload holds, + /// so a key of the same name under another renderer's type tag is not + /// read. + #[must_use] + pub fn picking_bid_by(mut self, key: &'static str) -> Self { + self.bid_id_key = Some(key); + self + } + + /// The identifier the renderer picks its bid by, when the code that built + /// the descriptor said which payload key holds it. + /// + /// # Examples + /// + /// ``` + /// use serde_json::json; + /// use trusted_server_core::auction::types::BidRenderer; + /// + /// let payload = json!({ "bidId": "fictional-bid-id" }); + /// let unstated = BidRenderer::new("example", payload.clone()) + /// .expect("should accept an object payload"); + /// assert_eq!(unstated.bid_id(), None); + /// + /// let stated = BidRenderer::new("example", payload) + /// .expect("should accept an object payload") + /// .picking_bid_by("bidId"); + /// assert_eq!(stated.bid_id(), Some("fictional-bid-id")); + /// ``` + #[must_use] + pub fn bid_id(&self) -> Option<&str> { + self.payload + .get(self.bid_id_key?) + .and_then(serde_json::Value::as_str) } } @@ -599,59 +757,177 @@ mod tests { ); } - #[test] - fn aps_renderer_serializes_to_versioned_camel_case_contract() { - let renderer = BidRenderer::Aps(ApsRendererV1 { + /// A provider's own payload type, as an implementation in a crate of its + /// own defines one. + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + struct ExampleRenderer { + version: u8, + bid_id: String, + creative_url: String, + } + + const EXAMPLE_RENDERER_TYPE: &str = "example"; + + fn example_descriptor() -> ExampleRenderer { + ExampleRenderer { version: 1, - account_id: "example-account-id".to_string(), bid_id: "fictional-bid-id".to_string(), - creative_id: Some("fictional-creative-id".to_string()), - tag_type: ApsTagType::Iframe, creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }); + } + } - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + #[test] + fn renderer_serializes_flat_under_its_type_tag() { + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &example_descriptor()) + .expect("should build renderer descriptor"); assert_eq!( - serialized, + serde_json::to_value(&renderer).expect("should serialize renderer"), json!({ - "type": "aps", + "type": "example", "version": 1, - "accountId": "example-account-id", "bidId": "fictional-bid-id", - "creativeId": "fictional-creative-id", - "tagType": "iframe", - "creativeUrl": "https://creative.example/render", - "aaxResponse": "base64-data", - "width": 300, - "height": 250 + "creativeUrl": "https://creative.example/render" }), - "should match renderer wire contract" + "should carry the tag beside the payload's own keys" ); } #[test] - fn aps_renderer_omits_absent_creative_id() { - let renderer = BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account-id".to_string(), - bid_id: "fictional-bid-id".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "base64-data".to_string(), - width: 300, - height: 250, - }); + fn renderer_round_trips_through_its_wire_form() { + let descriptor = example_descriptor(); + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &descriptor) + .expect("should build renderer descriptor"); - let serialized = serde_json::to_value(&renderer).expect("should serialize renderer"); + let serialized = serde_json::to_string(&renderer).expect("should serialize renderer"); + let restored: BidRenderer = + serde_json::from_str(&serialized).expect("should deserialize renderer"); + assert_eq!( + restored.renderer_type(), + EXAMPLE_RENDERER_TYPE, + "should round-trip the renderer type tag" + ); + assert_eq!( + restored + .payload_as::(EXAMPLE_RENDERER_TYPE) + .expect("should deserialize the payload"), + descriptor, + "should round-trip the provider payload" + ); + } + + #[test] + fn renderer_payload_is_hidden_from_a_different_type_tag() { + let renderer = BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!({ "version": 1 })) + .expect("should build renderer descriptor"); + + assert!( + renderer.payload_as::("other").is_none(), + "should refuse a payload requested under a different tag" + ); + } + + #[test] + fn renderer_payload_field_borrows_the_same_value_the_whole_descriptor_carries() { + let descriptor = example_descriptor(); + let renderer = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &descriptor) + .expect("should build renderer descriptor"); + + assert_eq!( + renderer + .payload_field(EXAMPLE_RENDERER_TYPE, "bidId") + .and_then(serde_json::Value::as_str), + Some(descriptor.bid_id.as_str()), + "should read the same bid id the whole descriptor carries" + ); + assert_eq!( + renderer + .payload_field(EXAMPLE_RENDERER_TYPE, "bidId") + .and_then(serde_json::Value::as_str), + renderer + .payload_as::(EXAMPLE_RENDERER_TYPE) + .as_ref() + .map(|full| full.bid_id.as_str()), + "should agree with the field read through the whole descriptor" + ); + assert!( + renderer + .payload_field(EXAMPLE_RENDERER_TYPE, "notAKey") + .is_none(), + "should return nothing for a key the payload does not carry" + ); + } + + #[test] + fn renderer_payload_field_is_hidden_from_a_different_type_tag() { + let renderer = BidRenderer::new( + EXAMPLE_RENDERER_TYPE, + json!({ "bidId": "fictional-bid-id" }), + ) + .expect("should build renderer descriptor"); + + assert!( + renderer.payload_field("other", "bidId").is_none(), + "should refuse a field requested under a different tag" + ); + } + + #[test] + fn renderer_names_the_bid_it_picks_only_when_its_builder_said_which_key_holds_it() { + let unstated = BidRenderer::from_typed(EXAMPLE_RENDERER_TYPE, &example_descriptor()) + .expect("should build renderer descriptor"); + assert_eq!( + unstated.bid_id(), + None, + "should name no bid for a descriptor whose builder made no statement" + ); + + let stated = unstated.clone().picking_bid_by("bidId"); + assert_eq!( + stated.bid_id(), + Some("fictional-bid-id"), + "should read the key the builder stated" + ); + assert_eq!( + stated.clone().bid_id(), + Some("fictional-bid-id"), + "should keep the statement when the descriptor is cloned" + ); + assert_eq!( + serde_json::to_value(&stated).expect("should serialize renderer"), + serde_json::to_value(&unstated).expect("should serialize renderer"), + "should send the same bytes with or without the statement" + ); + + let absent = BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!({ "version": 1 })) + .expect("should build renderer descriptor") + .picking_bid_by("bidId"); + assert_eq!( + absent.bid_id(), + None, + "should name no bid when the payload does not carry the stated key" + ); + } + + #[test] + fn renderer_rejects_a_payload_that_is_not_an_object() { + assert!( + BidRenderer::new(EXAMPLE_RENDERER_TYPE, json!("not-an-object")).is_err(), + "should reject a payload that is not a JSON object" + ); + } + + #[test] + fn renderer_rejects_a_payload_carrying_its_own_type_key() { assert!( - serialized.get("creativeId").is_none(), - "should omit absent creative ID" + BidRenderer::new( + EXAMPLE_RENDERER_TYPE, + json!({ "type": "other", "version": 1 }) + ) + .is_err(), + "should reject a payload that would collide with the type tag" ); } diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index 56171e1ea5..9a7f5ab019 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -1,20 +1,22 @@ //! Auction configuration types shared by settings and auction planning. -use serde::de::{Error as _, MapAccess, SeqAccess, Visitor, value::MapAccessDeserializer}; +use serde::de::Error as _; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; use validator::Validate; -const LEGACY_PROVIDER_LIST_MESSAGE: &str = "Configuration field `auction.providers` uses the removed list schema; migrate to `[auction.providers.]` map entries as described in the CHANGELOG.md breaking migration"; +const MOVED_PROVIDERS_MESSAGE: &str = "`[auction.providers]` has moved. Select demand sources with `[demand] modules = [...]` and give each its settings in `[demand.]`, as the configuration rules describe"; + +const MOVED_MEDIATOR_MESSAGE: &str = "`[auction] mediator` has moved. Select the ad server with `[ad-server] module = \"\"` and give it its settings in `[ad-server.]`, as the configuration rules describe"; + +use crate::provider_table::SectionModules; pub use crate::auction::plan::{ - BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + BidderId, BidderRouteConfig, NotificationConfig, ProviderId, RoutingMode, }; /// Auction orchestration configuration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] pub struct AuctionConfig { /// Enable the auction orchestrator #[serde(default)] @@ -49,18 +51,45 @@ pub struct AuctionConfig { )] pub rewrite_creatives: bool, - /// Operator-defined bidder-provider instances, keyed by provider ID. - #[serde(default, deserialize_with = "deserialize_provider_map")] - pub providers: BTreeMap, + /// Refuses the removed `providers` table with a message naming its new + /// home, so an old configuration fails with the fix rather than as an + /// unknown field. + #[serde(default, skip_serializing, deserialize_with = "reject_moved_providers")] + #[allow( + dead_code, + reason = "the field exists so deserialization refuses the removed table" + )] + pub(crate) providers: MovedSetting, - /// Client-visible bidder routes, keyed by bidder code. + /// Client-visible bidder routes, keyed by bidder code, each naming the + /// `[demand]` provider the bidder is sent to. #[serde(default)] pub bidders: BTreeMap, - /// Optional separately registered mediator provider name. - /// When set, runs parallel mediation strategy (bidders in parallel, then mediator decides) - /// When omitted, runs parallel only strategy (bidders in parallel, highest CPM wins) - pub mediator: Option, + /// The ad server name the legacy test orchestrator runs. + /// + /// Production selects its ad server in `[ad-server]` instead, so this never + /// appears in a configuration file. + #[cfg(test)] + #[serde(skip)] + pub(crate) adserver_name: Option, + + /// The demand source names the legacy test orchestrator runs. + /// + /// Production compiles its sources from `[demand]` instead, so this never + /// appears in a configuration file. + #[cfg(test)] + #[serde(skip)] + pub(crate) provider_names: Vec, + + /// Refuses the removed `mediator` setting with a message naming its new + /// home. + #[serde(default, skip_serializing, deserialize_with = "reject_moved_mediator")] + #[allow( + dead_code, + reason = "the field exists so deserialization refuses the removed setting" + )] + pub(crate) mediator: MovedSetting, /// Timeout in milliseconds #[serde(default = "default_timeout")] @@ -68,7 +97,10 @@ pub struct AuctionConfig { pub timeout_ms: u32, /// KV store name for creative storage (deprecated: creatives are now delivered inline) - #[serde(default = "default_creative_store")] + #[serde( + default = "default_creative_store", + serialize_with = "crate::redacted::sensitive" + )] pub creative_store: String, /// Keys allowed in the auction request context map. @@ -79,6 +111,11 @@ pub struct AuctionConfig { /// deduplicated and serialized in sorted order so config hashes are stable. #[serde(default = "default_allowed_context_keys")] pub allowed_context_keys: BTreeSet, + + /// The modules this section selects, with each one's settings in the + /// table at its name. + #[serde(flatten)] + pub modules: SectionModules, } impl Default for AuctionConfig { @@ -87,47 +124,47 @@ impl Default for AuctionConfig { enabled: false, sanitize_creatives: default_sanitize_creatives(), rewrite_creatives: default_rewrite_creatives(), - providers: BTreeMap::new(), + providers: MovedSetting, + #[cfg(test)] + adserver_name: None, + #[cfg(test)] + provider_names: Vec::new(), bidders: BTreeMap::new(), - mediator: None, + mediator: MovedSetting, timeout_ms: default_timeout(), creative_store: default_creative_store(), allowed_context_keys: BTreeSet::new(), + modules: SectionModules::default(), } } } -fn deserialize_provider_map<'de, D>( - deserializer: D, -) -> Result, D::Error> +#[cfg(test)] +impl AuctionConfig { + /// Whether the legacy test orchestrator runs an ad server. + pub(crate) fn has_adserver(&self) -> bool { + self.adserver_name.is_some() + } +} + +/// A setting that has moved elsewhere. It deserializes only by failing. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct MovedSetting; + +fn reject_moved_providers<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { - struct ProviderMapVisitor; - - impl<'de> Visitor<'de> for ProviderMapVisitor { - type Value = BTreeMap; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a map of auction provider IDs to provider configurations") - } - - fn visit_map(self, map: A) -> Result - where - A: MapAccess<'de>, - { - Self::Value::deserialize(MapAccessDeserializer::new(map)) - } - - fn visit_seq(self, _sequence: A) -> Result - where - A: SeqAccess<'de>, - { - Err(A::Error::custom(LEGACY_PROVIDER_LIST_MESSAGE)) - } - } + let _ = serde::de::IgnoredAny::deserialize(deserializer)?; + Err(D::Error::custom(MOVED_PROVIDERS_MESSAGE)) +} - deserializer.deserialize_any(ProviderMapVisitor) +fn reject_moved_mediator<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let _ = serde::de::IgnoredAny::deserialize(deserializer)?; + Err(D::Error::custom(MOVED_MEDIATOR_MESSAGE)) } fn default_timeout() -> u32 { @@ -159,36 +196,6 @@ fn default_allowed_context_keys() -> BTreeSet { BTreeSet::new() } -impl AuctionConfig { - #[cfg(test)] - pub(crate) fn legacy_provider_map(names: &[&str]) -> BTreeMap { - names - .iter() - .map(|name| { - let id = ProviderId::unchecked_for_legacy_test(name); - ( - id, - ProviderConfig { - protocol: "openrtb-2.6".to_string(), - profile: "standard".to_string(), - endpoint: format!("https://{name}.example/openrtb2/auction"), - timeout_ms: None, - routing: RoutingMode::AllEligible, - notifications: NotificationConfig::default(), - profile_config: serde_json::json!({}), - }, - ) - }) - .collect() - } - - /// Check if this config has a mediator configured. - #[must_use] - pub fn has_mediator(&self) -> bool { - self.mediator.is_some() - } -} - #[cfg(test)] mod tests { use super::*; @@ -283,35 +290,50 @@ mod tests { } #[test] - fn provider_list_shape_is_rejected() { + fn moved_providers_fail_naming_the_demand_table() { + for providers in [ + serde_json::json!(["prebid"]), + serde_json::json!({ "pbs_main": { "endpoint": "https://prebid.example" } }), + ] { + let error = serde_json::from_value::(serde_json::json!({ + "providers": providers + })) + .expect_err("should refuse the moved providers table"); + + assert!( + error.to_string().contains("[demand] modules"), + "should name the new home: {error}" + ); + } + } + + #[test] + fn moved_mediator_fails_naming_the_adserver_table() { let error = serde_json::from_value::(serde_json::json!({ - "providers": ["prebid"] + "mediator": "adserver_mock" })) - .expect_err("should reject the removed provider-list schema"); + .expect_err("should refuse the moved mediator setting"); assert!( - error.to_string().contains("map") || error.to_string().contains("object"), - "should require map-shaped providers: {error}" + error.to_string().contains("[ad-server] module"), + "should name the new home: {error}" ); } #[test] - fn map_schema_round_trips_provider_and_bidder_routes() { + fn bidder_routes_round_trip() { let config: AuctionConfig = serde_json::from_value(serde_json::json!({ - "providers": { - "pbs-main": { - "protocol": "openrtb-2.6", - "profile": "prebid-server", - "endpoint": "https://prebid.example/openrtb2/auction" - } - }, "bidders": { - "example-bidder": { "provider": "pbs-main" } + "example-bidder": { "module": "pbs_main" } } })) - .expect("should parse map-shaped auction config"); + .expect("should parse bidder routes"); - assert_eq!(config.providers.len(), 1); assert_eq!(config.bidders.len(), 1); + let serialized = serde_json::to_value(&config).expect("should serialize"); + assert!( + serialized.get("providers").is_none() && serialized.get("mediator").is_none(), + "should never write the moved settings back" + ); } } diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs deleted file mode 100644 index 9bbd40b69a..0000000000 --- a/crates/trusted-server-core/src/auth.rs +++ /dev/null @@ -1,607 +0,0 @@ -use base64::{Engine as _, engine::general_purpose::STANDARD}; -use edgezero_core::body::Body as EdgeBody; -use error_stack::Report; -use http::header; -use http::{Request, Response, StatusCode}; -use sha2::{Digest as _, Sha256}; -use subtle::ConstantTimeEq as _; - -use crate::error::TrustedServerError; -use crate::settings::Settings; - -const BASIC_AUTH_REALM: &str = r#"Basic realm="Trusted Server""#; - -/// Marks the single `Authorization` value Trusted Server validated. -/// -/// The shared template cache may exempt this value from its normal authorization -/// bypass. [`enforce_basic_auth`] clears any existing marker before checking and -/// inserts a digest-bound marker only after successful authentication. -#[derive(Debug, Clone)] -pub(crate) struct EdgeTerminatedAuthorization([u8; 32]); - -impl EdgeTerminatedAuthorization { - fn digest(value: &[u8]) -> [u8; 32] { - Sha256::digest(value).into() - } - - pub(crate) fn matches(&self, headers: &http::HeaderMap) -> bool { - let mut values = headers.get_all(header::AUTHORIZATION).iter(); - let Some(value) = values.next() else { - return false; - }; - values.next().is_none() && self.0 == Self::digest(value.as_bytes()) - } - - /// Builds the marker without performing a credential check. - /// - /// Test-only. Production code obtains this marker exclusively by passing - /// [`enforce_basic_auth`], which is what makes it meaningful. - #[cfg(test)] - #[must_use] - pub(crate) fn for_test(value: &str) -> Self { - Self(Self::digest(value.as_bytes())) - } -} - -/// Enforces HTTP Basic authentication for configured handler paths. -/// -/// Returns `Ok(None)` when the request does not target a protected handler or -/// when the supplied credentials are valid. Returns `Ok(Some(Response))` with -/// the auth challenge when credentials are missing or invalid. -/// -/// Admin endpoints are protected by requiring a handler during settings -/// finalization; see [`Settings::from_toml`]. Credential checks use constant-time -/// comparison for both username and password, and evaluate both regardless of -/// individual match results to avoid timing oracles. Runtime requests within -/// the reserved admin namespace fail closed if no handler matches, providing -/// defense in depth for malformed and parameterized paths. -/// -/// # Request mutation -/// -/// Takes `req` mutably because it owns [`EdgeTerminatedAuthorization`]. Any -/// inherited marker is cleared on entry, and a fresh one is inserted only on the -/// success path, so the marker present after this call always describes this -/// call's own decision. Nothing else about the request is touched — in -/// particular the `Authorization` header is left in place and still reaches the -/// publisher origin. -/// -/// That last point is a stated assumption: a credential this edge terminates is -/// treated as reader-neutral, which holds unless the origin *also* authenticates -/// on the same header. An origin that does so declares `Vary: Authorization`, -/// which the template-cache store refuses as an uncovered `Vary` name. An origin -/// that varies on `Authorization` without declaring it would defeat any HTTP -/// cache, and is out of scope here. -/// -/// # Errors -/// -/// Returns an error when handler configuration is invalid, such as an -/// un-compilable path regex. -pub fn enforce_basic_auth( - settings: &Settings, - req: &mut Request, -) -> Result>, Report> { - // Cleared before any early return so no inherited marker can survive a call - // that did not itself validate a credential. Without this, a request marked - // upstream and then routed to an unprotected path would keep an assertion - // nothing checked. - req.extensions_mut().remove::(); - - let path = req.uri().path(); - let Some(handler) = settings.handler_for_path(path)? else { - if Settings::is_admin_path(path) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!("Admin path `{path}` has no configured handler"), - })); - } - return Ok(None); - }; - - let Some((username, password, authorization_digest)) = extract_credentials(req) else { - return Ok(Some(unauthorized_response())); - }; - - // Hash before comparing to normalise lengths — `ct_eq` on raw byte slices - // short-circuits when lengths differ, which would leak credential length. - // SHA-256 produces fixed-size digests so the comparison is truly constant-time. - // - // Note: constant-time guarantees are best-effort on WASM targets because the - // runtime optimiser/JIT may re-introduce variable-time paths. This is an - // inherent limitation of all constant-time code in managed runtimes. - let username_match = Sha256::digest(handler.username.expose().as_bytes()) - .ct_eq(&Sha256::digest(username.as_bytes())); - let password_match = Sha256::digest(handler.password.expose().as_bytes()) - .ct_eq(&Sha256::digest(password.as_bytes())); - - if bool::from(username_match & password_match) { - // Record that TS itself consumed this credential, so the shared template - // cache can distinguish it from a credential meant for the origin. - req.extensions_mut() - .insert(EdgeTerminatedAuthorization(authorization_digest)); - Ok(None) - } else { - log::warn!("Basic auth failed for path: {}", req.uri().path()); - Ok(Some(unauthorized_response())) - } -} - -/// Username from a basic-auth request, for audit logging. -/// -/// Returns the username alone. The password is a shared static secret and must never reach -/// a log line. -/// -/// This parses a header; it verifies nothing. Call it only on a request -/// [`enforce_basic_auth`] has already accepted, where the username identifies which -/// operator credential was used. -#[must_use] -pub fn authenticated_username(req: &Request) -> Option { - extract_credentials(req).map(|(username, _password, _digest)| username) -} - -fn extract_credentials(req: &Request) -> Option<(String, String, [u8; 32])> { - let mut header_values = req.headers().get_all(header::AUTHORIZATION).iter(); - let header_value = header_values.next()?; - if header_values.next().is_some() { - return None; - } - let authorization_digest = EdgeTerminatedAuthorization::digest(header_value.as_bytes()); - let header_value = header_value.to_str().ok()?; - - let mut parts = header_value.splitn(2, ' '); - let scheme = parts.next()?.trim(); - if !scheme.eq_ignore_ascii_case("basic") { - return None; - } - - let token = parts.next()?.trim(); - if token.is_empty() { - return None; - } - - let decoded = STANDARD.decode(token).ok()?; - let credentials = String::from_utf8(decoded).ok()?; - - let mut credentials_parts = credentials.splitn(2, ':'); - let username = credentials_parts.next()?.to_owned(); - let password = credentials_parts.next()?.to_owned(); - - Some((username, password, authorization_digest)) -} - -fn unauthorized_response() -> Response { - Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header(header::WWW_AUTHENTICATE, BASIC_AUTH_REALM) - .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") - .body(EdgeBody::from(b"Unauthorized".as_ref())) - .expect("should build unauthorized response") -} - -#[cfg(test)] -mod tests { - use super::*; - use base64::engine::general_purpose::STANDARD; - use http::{HeaderValue, Method, header}; - - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; - - fn build_request(method: Method, uri: &str) -> Request { - Request::builder() - .method(method) - .uri(uri) - .body(EdgeBody::empty()) - .expect("should build request") - } - - fn set_authorization(req: &mut Request, value: &str) { - req.headers_mut().insert( - header::AUTHORIZATION, - HeaderValue::from_str(value).expect("should build authorization header"), - ); - } - - #[test] - fn encoded_admin_separator_path_is_auth_gated() { - // `^/_ts/admin` matches the raw path, so a percent-encoded separator - // still consumes admin credentials. The publisher-fallback boundary - // reserves the same paths so those credentials are never forwarded - // upstream (see `ec::admin::deny_admin_diagnostic_fallback`). - let settings = create_test_settings(); - - for path in ["/_ts/admin%2Fec", "/_ts/admin%2fec"] { - let mut req = build_request(Method::GET, &format!("https://example.com{path}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .unwrap_or_else(|| panic!("should challenge {path}")); - - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "should require credentials for {path}" - ); - } - } - - #[test] - fn valid_credentials_mark_the_request_as_edge_terminated() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "valid credentials should be admitted" - ); - let marker = req - .extensions() - .get::() - .expect("should mark a credential this edge consumed"); - assert!( - marker.matches(req.headers()), - "the marker should match the unchanged validated authorization" - ); - - req.headers_mut().insert( - header::AUTHORIZATION, - HeaderValue::from_static("Bearer publisher-origin-credential"), - ); - let marker = req - .extensions() - .get::() - .expect("should retain the marker after an unrelated mutation"); - assert!( - !marker.matches(req.headers()), - "the marker must not match a replacement authorization value" - ); - } - - #[test] - fn repeated_authorization_values_are_rejected_without_a_marker() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - req.headers_mut().append( - header::AUTHORIZATION, - HeaderValue::from_static("Bearer publisher-origin-credential"), - ); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge an ambiguous credential"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - req.extensions() - .get::() - .is_none(), - "a repeated authorization field must remain pass-through rather than being marked safe" - ); - } - - #[test] - fn an_inherited_marker_is_cleared_on_an_unprotected_path() { - // The marker asserts "this edge already checked a credential". A request - // routed to a path no handler protects was never checked here, so a marker - // it arrived with must not survive to grant shared-template eligibility. - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/open"); - set_authorization(&mut req, "Basic dXNlcjpwYXNz"); - req.extensions_mut() - .insert(EdgeTerminatedAuthorization::for_test("Basic dXNlcjpwYXNz")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "an unprotected path should not challenge" - ); - assert!( - req.extensions() - .get::() - .is_none(), - "a marker no credential check produced must not survive this call" - ); - } - - #[test] - fn an_inherited_marker_is_cleared_when_credentials_are_rejected() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:wrong-pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - req.extensions_mut() - .insert(EdgeTerminatedAuthorization::for_test( - "Basic dXNlcjp3cm9uZy1wYXNz", - )); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - req.extensions() - .get::() - .is_none(), - "a failed check must strip an inherited marker rather than honour it" - ); - } - - #[test] - fn a_non_protected_path_leaves_authorization_unmarked() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/open"); - set_authorization(&mut req, "Basic dXNlcjpwYXNz"); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "an unprotected path should not challenge" - ); - assert!( - req.extensions() - .get::() - .is_none(), - "a credential no handler consumed is pass-through and must stay disqualifying" - ); - } - - #[test] - fn rejected_credentials_leave_the_request_unmarked() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - let encoded = STANDARD.encode("user:wrong-pass"); - set_authorization(&mut req, &format!("Basic {encoded}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!( - req.extensions() - .get::() - .is_none(), - "a failed credential must never be marked as terminated" - ); - } - - #[test] - fn no_challenge_for_non_protected_path() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/open"); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none() - ); - } - - #[test] - fn challenge_when_missing_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - let realm = response - .headers() - .get(header::WWW_AUTHENTICATE) - .expect("should have WWW-Authenticate header"); - assert_eq!(realm, BASIC_AUTH_REALM); - } - - #[test] - fn allow_when_credentials_match() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("user:pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none() - ); - } - - #[test] - fn challenge_when_both_credentials_wrong() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("wrong:wrong"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn challenge_when_username_wrong_password_correct() { - // Validates that both fields are always evaluated — no short-circuit username oracle. - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("wrong-user:pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "should reject wrong username even with correct password" - ); - } - - #[test] - fn challenge_when_username_correct_password_wrong() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("user:wrong-pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "should reject correct username with wrong password" - ); - } - - #[test] - fn challenge_when_scheme_is_not_basic() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://example.com/secure"); - set_authorization(&mut req, "Bearer token"); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn returns_error_for_invalid_handler_regex_without_panicking() { - let config = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "(""#); - let err = Settings::from_toml(&config).expect_err("should reject invalid handler regex"); - assert!( - err.to_string() - .contains("Handler path regex `(` failed to compile"), - "should describe the invalid handler regex" - ); - } - - #[test] - fn allow_admin_path_with_valid_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::POST, "https://example.com/_ts/admin/keys/rotate"); - let token = STANDARD.encode("admin:admin-pass"); - set_authorization(&mut req, &format!("Basic {token}")); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "should allow admin path with correct credentials" - ); - } - - #[test] - fn challenge_admin_path_with_wrong_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::POST, "https://example.com/_ts/admin/keys/rotate"); - let token = STANDARD.encode("admin:wrong"); - set_authorization(&mut req, &format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge admin path with wrong credentials"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - /// A handler regex is matched against the whole request path, so a broad - /// pattern such as `^/_ts` covers browser-facing endpoints in that - /// namespace — `/_ts/page-bids` and `/_ts/api/v1/identify` — not just the - /// admin routes it was probably meant for. Anonymous browser fetches never - /// carry Basic credentials, so those endpoints answer `401` for every - /// visitor. - /// - /// This pins the behaviour rather than exempting the paths: which paths a - /// handler covers is the operator's decision, and silently carving holes in - /// it would be worse than a documented constraint. Operators must scope - /// handler patterns to the paths they mean (`^/_ts/admin`) — see the - /// configuration guide. The tsjs client's `/__ts/page-bids` fallback keeps - /// affected deployments serving SPA ads until they do, but it disappears - /// with the alias in IABTechLab/trusted-server#970. - #[test] - fn broad_handler_regex_also_covers_browser_facing_endpoints() { - let config = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "^/_ts""#); - let settings = Settings::from_toml(&config).expect("should parse broad handler regex"); - - for path in [ - "https://example.com/_ts/page-bids?path=/article", - "https://example.com/_ts/api/v1/identify", - ] { - let mut req = build_request(Method::GET, path); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .unwrap_or_else(|| panic!("should challenge {path} under a `^/_ts` handler")); - assert_eq!( - response.status(), - StatusCode::UNAUTHORIZED, - "a `^/_ts` handler should challenge {path}" - ); - } - } - - #[test] - fn challenge_admin_path_with_missing_credentials() { - let settings = create_test_settings(); - let mut req = build_request(Method::POST, "https://example.com/_ts/admin/keys/rotate"); - - let response = enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .expect("should challenge admin path with missing credentials"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn concrete_admin_path_without_matching_handler_fails_closed() { - let config = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[{]id[}]$" - username = "admin" - password = "strong-test-password""#, - ); - let settings: Settings = - toml::from_str(&config).expect("should deserialize settings without finalization"); - let ec_id = format!("{}.abc123", "a".repeat(64)); - let mut req = build_request( - Method::GET, - &format!("https://example.com/_ts/admin/ec/{ec_id}"), - ); - - let error = enforce_basic_auth(&settings, &mut req) - .expect_err("should fail closed without a matching admin handler"); - assert!( - error.to_string().contains("no configured handler"), - "should describe the missing admin handler" - ); - } - - #[test] - fn similar_non_admin_prefix_without_handler_remains_public() { - let config = crate_test_settings_str().replace( - r#"path = "^/_ts/admin""#, - r#"path = "^/_ts/admin/keys/rotate$""#, - ); - let settings: Settings = - toml::from_str(&config).expect("should deserialize settings without finalization"); - let mut req = build_request(Method::GET, "https://example.com/_ts/administrator"); - - assert!( - enforce_basic_auth(&settings, &mut req) - .expect("should evaluate auth") - .is_none(), - "should not classify a similar prefix as the admin namespace" - ); - } -} diff --git a/crates/trusted-server-core/src/build_info.rs b/crates/trusted-server-core/src/build_info.rs new file mode 100644 index 0000000000..38466d2904 --- /dev/null +++ b/crates/trusted-server-core/src/build_info.rs @@ -0,0 +1,89 @@ +//! What this build reports about itself, fixed when it was compiled. +//! +//! `env!` is resolved by the compiler. Some hosts give a WebAssembly guest no +//! custom environment variable when it runs, so a fact read then would read +//! as absent there. +//! +//! | Constant | Build input | Without it | +//! | --- | --- | --- | +//! | [`BUILT_AT`] | `SOURCE_DATE_EPOCH`, in seconds | The time of the build | +//! | [`COMMIT`] | `TRUSTED_SERVER_COMMIT` | `unknown` | +//! | [`BUILD_RUN`] | `TRUSTED_SERVER_BUILD_RUN` | `unknown` | +//! | [`BUILD_TAG`] | `TRUSTED_SERVER_BUILD_TAG` | `unknown` | +//! +//! Each is the builder's own report. Nothing here measures the binary. + +/// The crate version. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// When this build was made, RFC 3339 in UTC to the second. +pub const BUILT_AT: &str = env!("TRUSTED_SERVER_BUILT_AT"); + +/// The commit this build was made from, or `unknown`. +pub const COMMIT: &str = env!("TRUSTED_SERVER_COMMIT"); + +/// The build run that made this binary, or `unknown`. +pub const BUILD_RUN: &str = env!("TRUSTED_SERVER_BUILD_RUN"); + +/// The tag this build was made from, as the builder names it, or `unknown`. +pub const BUILD_TAG: &str = env!("TRUSTED_SERVER_BUILD_TAG"); + +/// The build script's date arithmetic, compiled here so a test can call it. +#[cfg(test)] +#[path = "../build_time.rs"] +mod build_time; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_build_time_is_rfc_3339_in_utc() { + let shape = regex::Regex::new(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + .expect("should compile the pattern"); + + assert!( + shape.is_match(BUILT_AT), + "should be RFC 3339 in UTC: {BUILT_AT}" + ); + } + + #[test] + fn the_commit_the_run_and_the_tag_are_never_empty() { + for (name, value) in [ + ("commit", COMMIT), + ("build run", BUILD_RUN), + ("build tag", BUILD_TAG), + ] { + assert!( + !value.is_empty() && !value.chars().any(char::is_control), + "the {name} should be the builder's value or `unknown`, got {value:?}" + ); + } + } + + #[test] + fn the_build_script_writes_known_instants_as_rfc_3339() { + for (seconds, expected, instant) in [ + (0, "1970-01-01T00:00:00Z", "the epoch"), + ( + 951_782_400, + "2000-02-29T00:00:00Z", + "the leap day of a 400th year", + ), + (1_000_000_000, "2001-09-09T01:46:40Z", "a time of day"), + (4_102_444_800, "2100-01-01T00:00:00Z", "the start of 2100"), + ( + 4_107_542_400, + "2100-03-01T00:00:00Z", + "the day after 28 February 2100", + ), + ] { + assert_eq!( + build_time::rfc3339_utc(seconds), + expected, + "should write {instant}, {seconds} seconds after the epoch" + ); + } + } +} diff --git a/crates/trusted-server-core/src/cache_purge.rs b/crates/trusted-server-core/src/cache_purge.rs deleted file mode 100644 index f77f43e4dd..0000000000 --- a/crates/trusted-server-core/src/cache_purge.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! The operator-facing purge endpoint for templates and tagged origin responses. -//! -//! `POST /_ts/admin/cache/purge` with `{"scope":"all"}` or -//! `{"scope":"url","url":"https://example.com/page"}`. -//! -//! # Why every method is registered, not just `POST` -//! -//! A named route only claims the methods it lists. A method it does not claim falls through -//! to the publisher, and `enforce_basic_auth` leaves the `Authorization` header in place, so -//! a `GET` to this path would authenticate and then ship the shared admin credential to the -//! publisher origin. The route therefore claims every method and this handler answers the -//! non-`POST` ones with 405 itself. -//! -//! # Why `Content-Type` is enforced exactly -//! -//! Browsers attach basic-auth credentials automatically. A cross-origin form POST with -//! `enctype="text/plain"` is not preflighted, so requiring `POST` alone does not stop CSRF; -//! requiring a `Content-Type` that a form cannot produce does. - -use edgezero_core::body::Body as EdgeBody; -use error_stack::Report; -use http::{Method, Request, Response, StatusCode, header}; -use serde::{Deserialize, Serialize}; - -use crate::error::TrustedServerError; -use crate::http_util::enforce_max_body_size; -use crate::platform::{RuntimeServices, reader_url_surrogate_key}; - -/// Purge bodies name a scope and, for a URL purge, one URL. Nothing here is unbounded. -const PURGE_MAX_BODY_BYTES: usize = 4096; - -/// The request body, before its scope is validated. -/// -/// Deserialized as a flat struct rather than an internally-tagged enum on purpose. -/// `deny_unknown_fields` does not reach the unit variant of such an enum, so -/// `{"scope":"all","url":"…"}` would parse as a full flush — an operator who typed the -/// wrong scope while intending to purge one page would empty the whole cache and be told -/// it succeeded. Validating the pair by hand makes that combination an error. -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct PurgeBody { - scope: String, - url: Option, -} - -/// What an operator asked to purge. -#[derive(Debug, PartialEq, Eq)] -enum PurgeRequest { - /// Every template and tagged origin response this service has cached. - All, - /// One reader-facing URL, as a reader would type it; canonicalized before hashing. - Url(String), -} - -impl PurgeRequest { - /// Parse and validate a purge body. - /// - /// # Errors - /// - /// Returns a message naming the problem when the JSON is malformed, the scope is not - /// one of the two documented values, or the scope and `url` field disagree. - fn parse(bytes: &[u8]) -> Result { - let body: PurgeBody = - serde_json::from_slice(bytes).map_err(|error| format!("invalid JSON: {error}"))?; - - match (body.scope.as_str(), body.url) { - ("all", None) => Ok(Self::All), - ("all", Some(_)) => Err( - "scope \"all\" takes no url; did you mean {\"scope\":\"url\",\"url\":…}?" - .to_owned(), - ), - // Parsed, not merely non-empty. `reader_url_surrogate_key` hashes whatever it - // is given, so a path, a scheme-less host, or an `ftp://` URL would be - // acknowledged as purged under a key nothing was ever stored with — the exact - // silent no-op that key's own documentation calls the failure that matters. - // A CMS webhook sending paths would purge nothing and never find out. - ("url", Some(url)) - if url::Url::parse(&url).is_ok_and(|parsed| { - matches!(parsed.scheme(), "http" | "https") && parsed.host_str().is_some() - }) => - { - Ok(Self::Url(url)) - } - ("url", _) => Err( - "scope \"url\" requires an absolute http(s) url, as a reader addresses the page" - .to_owned(), - ), - (other, _) => Err(format!( - "unknown scope {other:?}; expected \"all\" or \"url\"" - )), - } - } -} - -/// What the purge did, in terms an operator mid-incident can act on. -#[derive(Debug, Serialize)] -struct PurgeResponse { - purged: bool, - scope: &'static str, - /// Present on a URL purge, so an operator can confirm which key was hit. - #[serde(skip_serializing_if = "Option::is_none")] - surrogate_key: Option, - message: &'static str, -} - -/// Handle a purge request. -/// -/// # Errors -/// -/// Returns an error when the body cannot be read, exceeds [`PURGE_MAX_BODY_BYTES`], is not -/// valid JSON for a known scope, or when the platform's purge fails. -pub async fn handle_cache_purge( - services: &RuntimeServices, - req: Request, - authenticated_principal: Option<&str>, -) -> Result, Report> { - if req.method() != Method::POST { - return Ok(method_not_allowed()); - } - - // Checked before the body is read: a request that cannot be a legitimate API call - // should not have its payload parsed at all. - let content_type = req - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default(); - if !is_exactly_json(content_type) { - return Ok(json_response( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - r#"{"purged":false,"error":"Content-Type must be application/json"}"#, - )); - } - - let body = req.into_body(); - if body.is_stream() { - return Err(Report::new(TrustedServerError::BadRequest { - message: "cache-purge request body must be buffered, not streamed".into(), - })); - } - let bytes = body.into_bytes().unwrap_or_default(); - enforce_max_body_size(&bytes, PURGE_MAX_BODY_BYTES, "cache-purge")?; - - let request = - PurgeRequest::parse(&bytes).map_err(|message| TrustedServerError::BadRequest { - message: format!("invalid cache-purge request: {message}"), - })?; - - match request { - PurgeRequest::All => { - // An unbounded flush behind one shared static credential is also an - // origin-stampede lever, and there is no rate-limit primitive on this path, so - // who used it is recorded every time. - log::warn!( - "Cache purge: ALL templates purged by {}", - authenticated_principal.unwrap_or("") - ); - services - .template_cache() - .purge_all() - .await - .map_err(|error| TrustedServerError::Configuration { - message: format!("cache purge failed: {error}"), - })?; - Ok(purge_ok(&PurgeResponse { - purged: true, - scope: "all", - surrogate_key: None, - // One surrogate-key call, so there is no partial state to reason about: - // an error means nothing was purged and the call is safe to retry. - message: "All templates purged. Purge is idempotent and safe to repeat.", - })) - } - PurgeRequest::Url(url) => { - // The reader-facing key, so callers pass the URL a reader would see and never - // have to replay this service's origin rewriting. - let surrogate_key = reader_url_surrogate_key(&url); - log::info!( - "Cache purge: url {url} (key {surrogate_key}) by {}", - authenticated_principal.unwrap_or("") - ); - services - .template_cache() - .purge_url_surrogate_key(&surrogate_key) - .await - .map_err(|error| TrustedServerError::Configuration { - message: format!("cache purge failed: {error}"), - })?; - Ok(purge_ok(&PurgeResponse { - purged: true, - scope: "url", - surrogate_key: Some(surrogate_key), - message: "URL purged. Purge is idempotent and safe to repeat.", - })) - } - } -} - -/// Exactly `application/json`, with optional parameters and whitespace. -/// -/// Deliberately strict: accepting `application/json`-ish media types would readmit the -/// form-post CSRF shape this check exists to close. -fn is_exactly_json(content_type: &str) -> bool { - content_type - .split(';') - .next() - .unwrap_or_default() - .trim() - .eq_ignore_ascii_case("application/json") -} - -fn method_not_allowed() -> Response { - let mut response = json_response( - StatusCode::METHOD_NOT_ALLOWED, - r#"{"purged":false,"error":"cache purge requires POST"}"#, - ); - response - .headers_mut() - .insert(header::ALLOW, header::HeaderValue::from_static("POST")); - response -} - -fn purge_ok(payload: &PurgeResponse) -> Response { - let body = serde_json::to_string(payload) - .unwrap_or_else(|_| r#"{"purged":true,"scope":"unknown"}"#.to_owned()); - json_response(StatusCode::OK, &body) -} - -/// Purge answers are per-operator and per-moment; nothing may store one. -fn json_response(status: StatusCode, body: &str) -> Response { - Response::builder() - .status(status) - .header(header::CONTENT_TYPE, "application/json") - .header(header::CACHE_CONTROL, "private, no-store") - .body(EdgeBody::from(body.to_owned())) - .expect("should build a cache-purge response") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn content_type_must_be_exactly_json() { - assert!(is_exactly_json("application/json")); - assert!(is_exactly_json("application/json; charset=utf-8")); - assert!(is_exactly_json(" APPLICATION/JSON ")); - } - - #[test] - fn form_postable_content_types_are_refused() { - // These three are the only types a cross-origin form can send, and a form POST - // carries basic-auth credentials without a preflight. - for content_type in [ - "text/plain", - "application/x-www-form-urlencoded", - "multipart/form-data", - "", - ] { - assert!( - !is_exactly_json(content_type), - "{content_type} must not be accepted" - ); - } - } - - #[test] - fn json_lookalike_types_are_refused() { - for content_type in [ - "application/jsonrequest", - "text/json", - "application/ld+json", - ] { - assert!( - !is_exactly_json(content_type), - "{content_type} must not be accepted" - ); - } - } - - #[test] - fn a_non_post_answer_names_the_method_it_accepts() { - let response = method_not_allowed(); - assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); - assert_eq!( - response - .headers() - .get(header::ALLOW) - .and_then(|value| value.to_str().ok()), - Some("POST") - ); - } - - #[test] - fn every_answer_forbids_storage() { - for response in [method_not_allowed(), json_response(StatusCode::OK, "{}")] { - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, no-store"), - "a purge answer must never be stored" - ); - } - } - - #[test] - fn scope_parsing_accepts_both_documented_shapes() { - assert_eq!( - PurgeRequest::parse(br#"{"scope":"all"}"#).expect("should parse"), - PurgeRequest::All - ); - assert_eq!( - PurgeRequest::parse(br#"{"scope":"url","url":"https://example.com/a"}"#) - .expect("should parse"), - PurgeRequest::Url("https://example.com/a".to_owned()) - ); - } - - #[test] - fn a_url_alongside_scope_all_is_refused_rather_than_flushing_everything() { - // The dangerous typo: an operator means to purge one page, mistypes the scope, and - // would otherwise be told a full flush succeeded. - let error = PurgeRequest::parse(br#"{"scope":"all","url":"https://example.com/a"}"#) - .expect_err("should refuse"); - assert!( - error.contains("takes no url"), - "the error must point at the confusion, got: {error}" - ); - } - - #[test] - fn an_unknown_or_malformed_scope_is_refused() { - // A typo must not silently widen into a full flush. - for body in [ - &br#"{"scope":"everything"}"#[..], - br#"{"scope":"url"}"#, - br#"{"scope":"url","url":" "}"#, - // Hashed as given, these would each be acknowledged as purged under a key - // that can never match a stored object. - br#"{"scope":"url","url":"/article"}"#, - br#"{"scope":"url","url":"example.com/article"}"#, - br#"{"scope":"url","url":"ftp://example.com/article"}"#, - br#"{"scope":"all","url":"https://example.com/a"}"#, - br#"{}"#, - br#"{"scope":"ALL"}"#, - br#"{"scope":"all","extra":1}"#, - br#"not json"#, - ] { - assert!( - PurgeRequest::parse(body).is_err(), - "{} must not parse", - String::from_utf8_lossy(body) - ); - } - } - - #[test] - fn a_reader_facing_url_still_parses() { - for body in [ - &br#"{"scope":"url","url":"https://ts.example.com/article"}"#[..], - br#"{"scope":"url","url":"http://ts.example.com/article?a=1"}"#, - br#"{"scope":"url","url":"https://ts.example.com"}"#, - ] { - assert!( - PurgeRequest::parse(body).is_ok(), - "{} is a URL a reader can address and must still purge", - String::from_utf8_lossy(body) - ); - } - } -} diff --git a/crates/trusted-server-core/src/closed_paths.rs b/crates/trusted-server-core/src/closed_paths.rs new file mode 100644 index 0000000000..bac9b398c2 --- /dev/null +++ b/crates/trusted-server-core/src/closed_paths.rs @@ -0,0 +1,256 @@ +//! Paths this server answers itself and never forwards to the publisher's origin. +//! +//! A request to `/_ts/admin`, or to the `/admin/keys` addresses beside it, is +//! an administration request. It can carry an operator's `Authorization` +//! header and a payload meant for this server alone. No route serves either +//! prefix, and forwarding such a request would hand both to the origin, so +//! every adapter asks [`closed_path_response`] before its publisher fallback +//! and the request is answered `404` here. + +use std::borrow::Cow; + +use edgezero_core::body::Body as EdgeBody; +use http::{Request, Response, StatusCode, header}; + +/// The server's administration prefix. +/// +/// It is matched as a bare prefix, with no `/` after it, so that a +/// percent-encoded separator such as `/_ts/admin%2Fec` is closed with the +/// paths it decodes to. A check for a literal slash would miss that spelling. +const ADMIN_NAMESPACE_PREFIX: &str = "/_ts/admin"; + +/// The key administration alias outside `/_ts`. +/// +/// Only the alias itself and its separator descendants are closed. A bare +/// prefix here would also close unrelated publisher paths such as +/// `/admin/keystore`. +const ADMIN_KEYS_PREFIX: &str = "/admin/keys"; + +/// Maximum percent-decoding rounds applied when testing a path. +/// +/// Bounds the work a `%25`-chained path can force while still reaching the +/// fixed point of any separator encoding a proxy chain would plausibly decode. +const MAX_PERCENT_DECODE_ROUNDS: usize = 4; + +/// Returns the local `404 Not Found` for a request to a closed path, and +/// `None` for any other request, which the publisher fallback then answers. +/// +/// The path is tested as sent and after each of a bounded number of +/// percent-decoding rounds, so `/_ts/admin%2Fec` and `/admin%252Fkeys/rotate` +/// are closed with the paths they decode to. +#[must_use] +pub fn closed_path_response(req: &Request) -> Option> { + is_closed(req.uri().path()).then(not_found) +} + +/// Returns whether `path` is closed either as sent or after any bounded number +/// of percent-decoding rounds. +/// +/// A separator encoded twice, as in `/admin%252Fkeys/rotate`, survives one +/// decode as `/admin%2Fkeys/rotate`, so the check is repeated to a fixed point +/// rather than applied once. +fn is_closed(path: &str) -> bool { + if is_closed_as_written(path) { + return true; + } + + let mut current = path.to_owned(); + for _ in 0..MAX_PERCENT_DECODE_ROUNDS { + let Some(decoded) = percent_decoded_path(¤t) else { + return false; + }; + + // A path whose remaining `%` sequences are not decodable escapes is a + // fixed point, so further rounds would repeat the same comparison. + if decoded == current { + return false; + } + + if is_closed_as_written(&decoded) { + return true; + } + + current = decoded; + } + + false +} + +fn is_closed_as_written(path: &str) -> bool { + path.starts_with(ADMIN_NAMESPACE_PREFIX) + || path == ADMIN_KEYS_PREFIX + || path + .strip_prefix(ADMIN_KEYS_PREFIX) + .is_some_and(|remainder| remainder.starts_with('/')) +} + +/// Percent-decodes `path` once, returning `None` when the path contains no +/// escape sequence or decodes to invalid UTF-8. +fn percent_decoded_path(path: &str) -> Option { + if !path.contains('%') { + return None; + } + + urlencoding::decode(path).ok().map(Cow::into_owned) +} + +fn not_found() -> Response { + Response::builder() + .status(StatusCode::NOT_FOUND) + .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) + .header(header::CACHE_CONTROL, "no-store") + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") + .body(EdgeBody::from(r#"{"error":"not found"}"#)) + .expect("should build the response for a closed path") +} + +#[cfg(test)] +mod tests { + use http::Method; + + use super::*; + + fn request(method: Method, path: &str) -> Request { + Request::builder() + .method(method) + .uri(format!("https://edge.example.com{path}")) + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn assert_closed(path: &str) { + for method in [ + Method::GET, + Method::POST, + Method::HEAD, + Method::OPTIONS, + Method::PUT, + Method::PATCH, + Method::DELETE, + ] { + let response = closed_path_response(&request(method.clone(), path)) + .unwrap_or_else(|| panic!("should answer {method} {path} here")); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should answer {method} {path} with 404" + ); + assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&http::HeaderValue::from_static("no-store")), + "should prevent caching for {path}" + ); + } + } + + #[test] + fn every_path_beneath_the_administration_prefix_is_closed() { + let ec_id = format!("{}.abc123", "a".repeat(64)); + for path in [ + "/_ts/admin".to_owned(), + "/_ts/admin/".to_owned(), + "/_ts/admin/ec".to_owned(), + "/_ts/admin/ec/".to_owned(), + format!("/_ts/admin/ec/{ec_id}"), + format!("/_ts/admin/ec/{ec_id}/extra"), + "/_ts/admin/eids".to_owned(), + "/_ts/admin/eids.json".to_owned(), + "/_ts/admin/ec;foo".to_owned(), + "/_ts/admin/keys/rotate".to_owned(), + "/_ts/admin/cache/purge".to_owned(), + "/_ts/admin/unknown".to_owned(), + ] { + assert_closed(&path); + } + } + + #[test] + fn an_encoded_separator_does_not_open_a_closed_path() { + // A check for a literal slash would miss `/_ts/admin%2Fec`, and + // reaching the publisher fallback would forward the caller's + // `Authorization` header and body to the origin. + for path in [ + "/_ts/admin%2Fec", + "/_ts/admin%2fec", + "/_ts/admin%2Fkeys/rotate", + "/_ts/admin%252Fec", + "/_ts/admin%5Cec", + "/_ts/adminec", + "/%5Fts/admin/ec", + ] { + assert_closed(path); + } + } + + #[test] + fn the_key_alias_outside_ts_is_closed_with_its_descendants() { + for path in [ + "/admin/keys", + "/admin/keys/", + "/admin/keys/rotate", + "/admin/keys/deactivate", + "/admin/keys/rotate/", + "/admin/keys/rotate/extra", + "/admin/keys%2Frotate", + "/admin/keys%2frotate", + "/admin%2Fkeys/rotate", + "/admin%2fkeys%2Frotate", + ] { + assert_closed(path); + } + } + + #[test] + fn a_separator_encoded_more_than_once_is_still_closed() { + // One decode leaves `/admin%252Fkeys/rotate` as `/admin%2Fkeys/rotate`, + // which no literal check matches. Decoding to a fixed point keeps the + // path closed against a proxy or origin that decodes it more than once. + for path in [ + "/admin%252Fkeys/rotate", + "/admin/keys%252Frotate", + "/admin%25252Fkeys/rotate", + "/_ts%252Fadmin/ec", + ] { + assert_closed(path); + } + } + + #[test] + fn other_publisher_paths_are_left_to_the_fallback() { + for path in [ + "/", + "/articles/example", + "/admin", + "/admin/login", + "/admin/keyboards", + "/admin/keystore", + "/admin/keys%25store", + "/articles/100%25-organic", + "/_ts/api/v1/batch-sync", + "/_ts/data", + ] { + assert!( + closed_path_response(&request(Method::POST, path)).is_none(), + "should leave {path} to the publisher fallback" + ); + } + } + + #[test] + fn the_answer_says_nothing_but_not_found() { + let response = closed_path_response(&request(Method::GET, "/_ts/admin/ec")) + .expect("should answer a closed path"); + + assert_eq!( + response.headers().get(header::CONTENT_TYPE), + Some(&http::HeaderValue::from_static("application/json")) + ); + assert_eq!( + response.headers().get(header::X_CONTENT_TYPE_OPTIONS), + Some(&http::HeaderValue::from_static("nosniff")) + ); + let body = response.into_body().into_bytes().unwrap_or_default(); + assert_eq!(&body[..], br#"{"error":"not found"}"#); + } +} diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index e48f5c2f03..6e939ac7db 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -11,48 +11,16 @@ use std::borrow::Cow; use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use validator::{Validate, ValidationError, ValidationErrors}; +use validator::{Validate, ValidationError, ValidationErrors, ValidationErrorsKind}; +use crate::ec::module::{HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; -use crate::integrations::{ - adserver_mock::AdServerMockConfig, - aps::ApsConfig, - datadome::DataDomeConfig, - didomi::DidomiIntegrationConfig, - google_tag_manager::GoogleTagManagerConfig, - gpt::GptConfig, - gpt_diagnostics::GptDiagnosticsConfig, - js_asset_proxy::{JS_ASSET_PROXY_INTEGRATION_ID, JsAssetProxyConfig}, - lockr::LockrConfig, - nextjs::NextJsIntegrationConfig, - osano::OsanoConfig, - permutive::PermutiveConfig, - prebid, - sourcepoint::SourcepointConfig, - testlight::TestlightConfig, -}; -use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; + +use crate::integrations::IntegrationBuilder; +use crate::settings::{AssetOriginAuth, Ec, MODULE_IMPLEMENTATION_KEY, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; -#[cfg(test)] -const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ - "prebid", - "aps", - "adserver_mock", - "testlight", - "nextjs", - "permutive", - "lockr", - "didomi", - "sourcepoint", - "osano", - "google_tag_manager", - "datadome", - "gpt", - "gpt_diagnostics", - JS_ASSET_PROXY_INTEGRATION_ID, -]; /// Typed app-config root used by the `ts` CLI. /// @@ -116,10 +84,34 @@ impl<'de> Deserialize<'de> for TrustedServerAppConfig { } } +/// The builders [`TrustedServerAppConfig`] validates against besides core's +/// own, set once by the tool that validates. +static DEPLOY_INTEGRATIONS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +/// Registers the integration builders [`TrustedServerAppConfig`] validates +/// against besides core's own. +/// +/// `EdgeZero` validates an app config through the [`Validate`] trait, which +/// takes no arguments, so a tool that validates a deployment's settings +/// registers that deployment's builders here before it validates anything. +/// The first call wins and a later one changes nothing. +pub fn register_deploy_integrations(builders: Vec) { + let _ = DEPLOY_INTEGRATIONS.set(builders); +} + +/// The builders a tool registered for deploy validation, or none. +fn deploy_integrations() -> &'static [IntegrationBuilder] { + DEPLOY_INTEGRATIONS.get().map_or(&[], Vec::as_slice) +} + impl Validate for TrustedServerAppConfig { fn validate(&self) -> Result<(), ValidationErrors> { let mut errors = self.settings.validate().err().unwrap_or_default(); - if let Err(report) = validate_settings_for_deploy(&self.settings) { + remove_labeled_module_secret_errors(&mut errors, &self.settings.ec); + if let Err(report) = + validate_settings_for_deploy_with(&self.settings, deploy_integrations()) + { errors.add( DEPLOY_VALIDATION_FIELD, report_to_validation_error(&report, "trusted_server_deploy_validation"), @@ -133,6 +125,101 @@ impl Validate for TrustedServerAppConfig { } } +/// Removes the passphrase checks on Edge Cookie module blocks written under +/// a label. +/// +/// Push-time validation reads a configuration whose secret fields hold +/// secret-store key names rather than the secrets themselves, so a value check +/// such as the 32-byte passphrase minimum would be judging a key name. +/// `EdgeZero`'s `validate_excluding_secrets` removes those checks for the +/// leaves [`secret_fields`](edgezero_core::app_config::AppConfigMeta::secret_fields) +/// lists, which covers the `[ec.hmac]` block. A block under a label of the +/// operator's choosing has no fixed path that list can hold, so its check is +/// removed here instead. The check itself is unchanged, and runs wherever +/// settings are loaded with their secrets resolved. +fn remove_labeled_module_secret_errors(errors: &mut ValidationErrors, ec: &Ec) { + let Some(ValidationErrorsKind::Struct(ec_errors)) = errors.errors_mut().get_mut("ec") else { + return; + }; + let labeled = ec + .module_blocks + .hmac_blocks() + .map(|(name, _)| name) + .filter(|name| *name != HMAC_MODULE_KEY) + .chain( + ec.module_blocks + .host_signals_blocks() + .map(|(name, _)| name) + .filter(|name| *name != HOST_SIGNALS_MODULE_KEY), + ); + for name in labeled { + let Some(ValidationErrorsKind::Struct(block_errors)) = ec_errors.errors_mut().get_mut(name) + else { + continue; + }; + block_errors.errors_mut().remove("passphrase"); + if block_errors.errors().is_empty() { + ec_errors.errors_mut().remove(name); + } + } + // An `ec` entry holding nothing would keep the whole result an error, the + // same reason `EdgeZero` prunes emptied containers after its own removals. + let ec_is_empty = ec_errors.errors().is_empty(); + if ec_is_empty { + errors.errors_mut().remove("ec"); + } +} + +impl crate::secret_resolution::ConfiguredSecretFields for TrustedServerAppConfig { + /// The passphrase of every Edge Cookie module block that configures a + /// module built into core under a label. + /// + /// [`secret_fields`](edgezero_core::app_config::AppConfigMeta::secret_fields) + /// lists the passphrases of the `[ec.hmac]` and `[ec.host_signals]` + /// blocks, the one path each of those modules' blocks has when its name + /// is its implementation. The same module under a label of the + /// operator's choosing holds that secret at `ec.kept\ + kept"; + + let replaced = without_bundle_hashes(page); + + assert_eq!( + replaced, + "\ + \ + kept\ + kept" + ); + } } } #[cfg(test)] mod tests { use super::*; - use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationHeadInjector, IntegrationHtmlContext, - }; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; use serde_json::json; @@ -739,166 +1194,106 @@ mod tests { fn create_test_config() -> HtmlProcessorConfig { HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_owned(), request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), integrations: IntegrationRegistry::default(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } + /// The client script names its cookies and storage entries after the + /// public prefix, which its tag tells it. A document given no prefix gets + /// no attribute. #[test] - fn integration_attribute_rewriter_can_remove_elements() { - struct RemovingLinkRewriter; - - impl IntegrationAttributeRewriter for RemovingLinkRewriter { - fn integration_id(&self) -> &'static str { - "removing" - } - - fn handles_attribute(&self, attribute: &str) -> bool { - attribute == "href" - } + fn the_bundle_tag_tells_the_client_script_the_public_prefix() { + fn process(public: Option) -> String { + let settings = create_test_settings(); + let registry = IntegrationRegistry::new(&settings).expect("should build the registry"); + let mut config = create_test_config(); + config.integrations = registry.clone(); + config.public_prefix = public; + let mut processor = test_support::create_page_processor(&settings, ®istry, config); + let output = processor + .process_chunk(b"", true) + .expect("should process HTML"); - fn rewrite( - &self, - _attr_name: &str, - attr_value: &str, - _ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if attr_value.contains("remove-me") { - AttributeRewriteAction::remove_element() - } else { - AttributeRewriteAction::keep() - } - } + String::from_utf8(output).expect("should produce valid UTF-8") } - let html = r#" - remove - keep - "#; - - let mut config = create_test_config(); - config.integrations = - IntegrationRegistry::from_rewriters(vec![Arc::new(RemovingLinkRewriter)], Vec::new()); - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - pipeline - .process(Cursor::new(html.as_bytes()), &mut output) - .expect("pipeline should process HTML"); - let processed = String::from_utf8(output).expect("output should be valid UTF-8"); - - assert!(processed.contains("keep-me")); - assert!(!processed.contains("remove-me")); + let told = process(Some( + PublicPrefix::new(["_ex"]).expect("should accept the prefix"), + )); + assert!( + told.contains(r#"data-ts-public-prefix="_ex""#), + "the bundle tag should carry the public prefix: {told}" + ); + let untold = process(None); + assert!(!untold.contains(PUBLIC_PREFIX_ATTRIBUTE), "{untold}"); } + /// A body-open insert is written immediately after the opening body tag, + /// once, however many body tags a fragment carries. The fetch processor's + /// handler is the same function, and the seam probe's page test drives it. #[test] - fn integration_head_injector_prepends_after_tsjs_once() { - struct TestHeadInjector; - - impl IntegrationHeadInjector for TestHeadInjector { - fn integration_id(&self) -> &'static str { - "test" - } - - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - vec!["".to_owned()] - } - } - - let html = "Test"; - - let mut config = create_test_config(); - config.integrations = IntegrationRegistry::from_rewriters_with_head_injectors( - Vec::new(), - Vec::new(), - vec![Arc::new(TestHeadInjector)], - ); - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, + fn a_body_open_insert_is_written_once_after_the_opening_body_tag() { + let plan = MiddlewarePlan { + body_open_inserts: vec!["

first

".to_owned(), "

second

".to_owned()], + ..MiddlewarePlan::default() }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - pipeline - .process(Cursor::new(html.as_bytes()), &mut output) - .expect("pipeline should process HTML"); - let processed = String::from_utf8(output).expect("output should be valid UTF-8"); - let tsjs_marker = "id=\"trustedserver-js\""; - let head_marker = "window.__testHeadInjector=true"; + let output = build_serve_processor(plan) + .process_chunk( + b"

page

", + true, + ) + .expect("should process HTML"); - assert_eq!( - processed.matches(tsjs_marker).count(), - 1, - "should inject unified tsjs tag once" + let out = String::from_utf8(output).expect("should be UTF-8"); + assert!( + out.contains("

first

second

page

"), + "the inserts follow the opening body tag in order: {out}" ); assert_eq!( - processed.matches(head_marker).count(), + out.matches("

first

").count(), 1, - "should inject head snippet once" - ); - - let tsjs_index = processed - .find(tsjs_marker) - .expect("should include unified tsjs tag"); - let head_index = processed - .find(head_marker) - .expect("should include head snippet"); - let title_index = processed - .find("") - .expect("should keep existing head content"); - - assert!( - head_index < tsjs_index, - "should inject config before tsjs bundle so auto-init can read it" - ); - assert!( - tsjs_index < title_index, - "should prepend all injected content before existing head content" + "once per document: {out}" ); } #[test] - fn integration_head_injector_marks_only_attribution_enabled_gpt_bundle() { - fn process(gpt_config: Option<(bool, bool)>) -> String { - let integrations = if let Some((enabled, gam_attribution_enabled)) = gpt_config { - let mut settings = create_test_settings(); + fn a_registration_marks_only_the_bundle_it_asks_to_mark() { + use crate::integrations::registry_test_support::tag_fixture as tag; + + fn process(mark_bundle: Option<bool>) -> String { + let mut settings = create_test_settings(); + if let Some(mark_bundle) = mark_bundle { settings - .integrations - .insert_config( - "gpt", + .insert_module_config( + "testing", + tag::MODULE, &json!({ - "enabled": enabled, - "gam_attribution_enabled": gam_attribution_enabled + "mark_bundle": mark_bundle }), ) - .expect("should insert GPT config"); - IntegrationRegistry::new(&settings).expect("should build GPT registry") - } else { - IntegrationRegistry::empty_for_tests() - }; + .expect("should insert the stand-in's settings"); + test_support::place_on_every_page( + &mut settings, + crate::middleware::MiddlewarePhase::Fetch, + &[tag::MODULE], + ); + } + let registry = IntegrationRegistry::new(&settings).expect("should build the registry"); let mut config = create_test_config(); - config.integrations = integrations; - let mut processor = create_html_processor(config); + config.integrations = registry.clone(); + let mut processor = test_support::create_page_processor(&settings, ®istry, config); let output = processor .process_chunk(b"<html><head></head><body></body></html>", true) .expect("should process HTML"); @@ -906,77 +1301,57 @@ mod tests { String::from_utf8(output).expect("should produce valid UTF-8") } - let attributed = process(Some((true, true))); - let unattributed = process(Some((true, false))); - let disabled_gpt = process(Some((false, true))); - let without_gpt = process(None); + let marked = process(Some(true)); + let unmarked = process(Some(false)); + let without_the_integration = process(None); - for html in [&attributed, &unattributed, &disabled_gpt, &without_gpt] { + for html in [&marked, &unmarked, &without_the_integration] { assert_eq!( html.matches("id=\"trustedserver-js\"").count(), 1, "should emit exactly one publisher bundle tag: {html}" ); } + let attribute = format!("{}=\"true\"", tag::BUNDLE_ATTRIBUTE); assert!( - attributed.contains("data-ts-gam-attribution=\"true\""), - "should mark only an attribution-enabled GPT publisher bundle" - ); - assert!( - !unattributed.contains("data-ts-gam-attribution"), - "should leave an attribution-disabled GPT publisher bundle unmarked" + marked.contains(&attribute), + "should mark the bundle a registration asks to mark" ); assert!( - !disabled_gpt.contains("data-ts-gam-attribution"), - "should let the GPT master switch suppress attribution metadata" + !unmarked.contains(tag::BUNDLE_ATTRIBUTE), + "should leave the bundle unmarked when the registration asks for nothing" ); assert!( - !without_gpt.contains("data-ts-gam-attribution"), - "should leave a non-GPT publisher bundle unmarked" + !without_the_integration.contains(tag::BUNDLE_ATTRIBUTE), + "should leave the bundle unmarked when no section selects the integration" ); - let head_insert_index = attributed - .find("window.__tsjs_installGptShim") - .expect("should include the GPT head insert"); - let publisher_bundle_index = attributed + let head_insert_index = marked + .find(tag::HEAD_FLAG) + .expect("should include the integration's head insert"); + let publisher_bundle_index = marked .find("id=\"trustedserver-js\"") .expect("should include the publisher bundle"); assert!( head_insert_index < publisher_bundle_index, - "should keep integration head inserts before the publisher bundle" + "should keep a middleware's head markup before the publisher bundle" ); } #[test] - fn active_gpt_diagnostics_loads_standalone_after_unified_bundle_once() { + fn what_a_module_left_on_the_request_is_written_around_the_unified_bundle_once() { + use crate::integrations::registry_test_support::request_fixture; + let html = "<html><head><title>Test"; let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &json!({ "enabled": true })) - .expect("should insert GPT diagnostics config"); - - let mut request = http::Request::builder() - .method(http::Method::GET) - .uri("https://publisher.example/page?ts_console=1") - .header("sec-fetch-dest", "document") - .body(edgezero_core::body::Body::empty()) - .expect("should build activation request"); - let decision = - crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) - .expect("should prepare diagnostics request"); + request_fixture::select_and_place(&mut settings); + let registry = + IntegrationRegistry::new(&settings).expect("should build integration registry"); let mut config = create_test_config(); - config.integrations = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should build integration registry"); - config.gpt_diagnostics = Some(decision); + config.integrations = registry.clone(); + config.request_state = request_fixture::marked(); - let processor = create_html_processor(config); + let processor = test_support::create_page_processor(&settings, ®istry, config); let pipeline_config = PipelineConfig { input_compression: Compression::None, output_compression: Compression::None, @@ -989,14 +1364,14 @@ mod tests { .process(Cursor::new(html.as_bytes()), &mut output) .expect("should process HTML"); let processed = String::from_utf8(output).expect("should produce valid UTF-8"); - let bootstrap_marker = "__tsjs_gpt_diagnostics_active"; + let head_marker = request_fixture::HEAD_FLAG; let bundle_marker = "id=\"trustedserver-js\""; - let diagnostics_marker = "tsjs-gpt_diagnostics.min.js"; + let module_marker = request_fixture::MODULE_FILE; assert_eq!( - processed.matches(bootstrap_marker).count(), + processed.matches(head_marker).count(), 1, - "should inject the diagnostics bootstrap once" + "should write the module's head insert once" ); assert_eq!( processed.matches(bundle_marker).count(), @@ -1004,26 +1379,26 @@ mod tests { "should inject the immediate TSJS bundle once" ); assert_eq!( - processed.matches(diagnostics_marker).count(), + processed.matches(module_marker).count(), 1, - "should inject one standalone diagnostics module" + "should write the module's after-bundle insert once" ); - let bootstrap_index = processed - .find(bootstrap_marker) - .expect("should include diagnostics bootstrap"); + let head_index = processed + .find(head_marker) + .expect("should include the head insert"); let bundle_index = processed .find(bundle_marker) .expect("should include immediate TSJS bundle"); - let diagnostics_index = processed - .find(diagnostics_marker) - .expect("should include standalone diagnostics module"); + let module_index = processed + .find(module_marker) + .expect("should include the after-bundle insert"); assert!( - bootstrap_index < bundle_index, - "should activate before core executes" + head_index < bundle_index, + "should write the head insert before the bundle executes" ); assert!( - bundle_index < diagnostics_index, - "should load diagnostics after core" + bundle_index < module_index, + "should write the after-bundle insert after the bundle" ); } @@ -1066,6 +1441,37 @@ mod tests { assert!(!result.contains("origin.example.com")); } + #[test] + fn the_origin_s_address_is_moved_in_srcset_and_imagesrcset() { + let mut processor = create_html_processor(create_test_config()); + + let output = processor + .process_chunk( + br#""#, + true, + ) + .expect("should process the document"); + let processed = String::from_utf8(output).expect("should stay UTF-8"); + + assert!( + processed.contains( + r#"imagesrcset="https://test.example.com/a.png 1x, https://test.example.com/b.png 2x, //test.example.com/c.png 3x""# + ), + "should move the origin's address in every candidate of an imagesrcset: {processed}" + ); + assert!( + processed.contains( + r#"srcset="https://test.example.com/a.png 1x, https://test.example.com/b.png 2x, //test.example.com/c.png 3x, test.example.com/d.png 4x""# + ), + "should move the origin's address in every candidate of a srcset, a bare host \ + included: {processed}" + ); + assert!( + processed.contains(r#""#), + "should leave a srcset that names another host as it is: {processed}" + ); + } + #[test] fn test_html_processor_config_from_settings() { let settings = create_test_settings(); @@ -1090,62 +1496,6 @@ mod tests { assert_eq!(config.request_scheme, "https"); } - #[test] - fn suppressed_datadome_tag_preserves_and_rewrites_publisher_tag() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "datadome", - &json!({ - "enabled": true, - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let config = HtmlProcessorConfig::from_settings( - &settings, - ®istry, - "origin.example.com", - "test.example.com", - "https", - ) - .with_datadome_client_tag_suppression(true); - let mut processor = create_html_processor(config); - - let output = processor - .process_chunk( - br#"content"#, - true, - ) - .expect("should process HTML"); - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - - assert!( - !html.contains("window.ddjskey"), - "should omit the DataDome client configuration" - ); - assert!( - html.contains("id=\"publisher-datadome\""), - "should preserve the publisher-originated DataDome tag" - ); - assert!( - html.contains("src=\"/integrations/datadome/tags.js\""), - "should rewrite the publisher-originated DataDome tag" - ); - assert!( - !html.contains("https://js.datadome.co/tags.js"), - "should remove the original third-party DataDome URL" - ); - assert_eq!( - html.matches("/integrations/datadome/tags.js").count(), - 1, - "should leave exactly one publisher-originated DataDome tag" - ); - } - #[test] fn test_real_publisher_html() { // Test with publisher HTML from test_publisher.html @@ -1223,61 +1573,6 @@ mod tests { ); } - #[test] - fn test_integration_registry_rewrites_integration_scripts() { - let html = r#" - - "#; - - let mut settings = Settings::default(); - let shim_src = "https://edge.example.com/static/testlight.js".to_owned(); - settings - .integrations - .insert_config( - "testlight", - &json!({ - "enabled": true, - "endpoint": "https://example.com/openrtb2/auction", - "rewrite_scripts": true, - "shim_src": shim_src, - }), - ) - .expect("should insert testlight config"); - - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let mut config = create_test_config(); - config.integrations = registry; - - let processor = create_html_processor(config); - let pipeline_config = PipelineConfig { - input_compression: Compression::None, - output_compression: Compression::None, - chunk_size: 8192, - }; - let mut pipeline = StreamingPipeline::new(pipeline_config, processor); - - let mut output = Vec::new(); - let result = pipeline.process(Cursor::new(html.as_bytes()), &mut output); - result.unwrap(); - - let processed = String::from_utf8_lossy(&output); - assert!( - processed.contains(&shim_src), - "Integration shim should replace integration script reference" - ); - assert!( - !processed.contains("cdn.testlight.com"), - "Original integration URL should be removed" - ); - } - #[test] fn test_real_publisher_html_with_gzip() { use flate2::Compression as GzCompression; @@ -1541,6 +1836,8 @@ mod tests { fn injects_ad_slots_at_head_open() { let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1550,10 +1847,10 @@ mod tests { r#""# .to_string(), ), + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1619,6 +1916,8 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1627,10 +1926,10 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1658,6 +1957,8 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1666,10 +1967,10 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); // Malformed HTML with two elements (common in CMS template pages) @@ -1698,16 +1999,18 @@ mod tests { let request_host = "proxy.test-publisher.example.com"; let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.test-publisher.example.com".to_string(), request_host: request_host.to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1752,6 +2055,8 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::InlineBids, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), @@ -1760,10 +2065,10 @@ mod tests { ad_slots_script: Some( r#""#.to_string(), ), + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1784,16 +2089,18 @@ mod tests { let state = std::sync::Arc::new(std::sync::Mutex::new(None)); let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + permissions_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut processor = create_html_processor(config); let output = processor @@ -1809,16 +2116,18 @@ mod tests { fn marker_mode_config(marker: &str, observer: Option>) -> HtmlProcessorConfig { HtmlProcessorConfig { csp_nonce_observed: observer, + public_prefix: None, body_close: BodyCloseInjection::Marker(marker.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + prepared: Arc::new(Prepared::default()), + request_state: IntegrationRequestState::default(), } } @@ -1963,50 +2272,6 @@ mod tests { } } - #[test] - fn nextjs_output_overflow_restores_in_progress_script_at_every_split() { - let mut settings = create_test_settings(); - settings.integrations.insert( - "nextjs".to_owned(), - json!({"enabled": true, "max_combined_payload_bytes": 128}), - ); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), - ) - .expect("should create registry"); - let first = r#""#; - let script = r#"self.__next_f.push([1,"c"])"#; - let padding = "x".repeat(129); - let expected = format!("{first}{padding}"); - - for split in 1..script.len() { - let mut config = create_test_config(); - config.integrations = registry.clone(); - let mut processor = create_html_processor(config); - let mut output = processor - .process_chunk(first.as_bytes(), false) - .expect("should process unresolved RSC group"); - let second = format!("{padding}", &script[split..]); - output.extend( - processor - .process_chunk(third.as_bytes(), true) - .expect("should finish bypassed script"), - ); - assert_eq!( - String::from_utf8(output).expect("should retain UTF-8"), - expected, - "should restore all original bytes when overflow occurs at script split {split}" - ); - } - } - #[test] fn a_nonce_bearing_meta_policy_is_observed() { let observed = Arc::new(AtomicBool::new(false)); @@ -2070,16 +2335,18 @@ mod tests { const MARKER: &str = ""; let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: Arc::new(Prepared::default()), body_close: BodyCloseInjection::Marker(MARKER.to_string()), origin_host: "origin.example.com".to_string(), request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), ad_slots_script: None, + permissions_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let source = format!(r#""#); diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 457e503568..2b9bed48bc 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -152,18 +152,30 @@ impl RequestInfo { /// [`sanitize_forwarded_headers`] at the edge, so `Host` and /// [`ClientInfo`] TLS detection are the only sources that fire. pub fn from_request(req: &Request, client_info: &ClientInfo) -> Self { - let host = extract_request_host(req); - let scheme = detect_request_scheme( - req, - client_info.tls_protocol.as_deref(), - client_info.tls_cipher.as_deref(), - ); - - Self { host, scheme } + Self { + host: request_host(req).to_owned(), + scheme: request_scheme(req, client_info).to_owned(), + } } } -fn extract_request_host(req: &Request) -> String { +/// The host a request was made for, read as [`RequestInfo::from_request`] +/// reads it, borrowed from the request. +pub(crate) fn request_host(req: &Request) -> &str { + extract_request_host(req) +} + +/// The scheme a request was made over, read as +/// [`RequestInfo::from_request`] reads it. +pub(crate) fn request_scheme(req: &Request, client_info: &ClientInfo) -> &'static str { + detect_request_scheme( + req, + client_info.tls_protocol.as_deref(), + client_info.tls_cipher.as_deref(), + ) +} + +fn extract_request_host(req: &Request) -> &str { req.headers() .get("forwarded") .and_then(|h| h.to_str().ok()) @@ -180,7 +192,6 @@ fn extract_request_host(req: &Request) -> String { .and_then(|h| h.to_str().ok()) }) .unwrap_or_default() - .to_owned() } fn parse_forwarded_param<'a>(forwarded: &'a str, param: &str) -> Option<&'a str> { @@ -221,9 +232,15 @@ fn strip_quotes(value: &str) -> &str { } } -fn normalize_scheme(value: &str) -> Option { - let scheme = value.trim().to_ascii_lowercase(); - (scheme == "https" || scheme == "http").then_some(scheme) +fn normalize_scheme(value: &str) -> Option<&'static str> { + let scheme = value.trim(); + if scheme.eq_ignore_ascii_case("https") { + Some("https") + } else if scheme.eq_ignore_ascii_case("http") { + Some("http") + } else { + None + } } /// Detects the request scheme (HTTP or HTTPS) using Fastly SDK methods and headers. @@ -238,17 +255,17 @@ fn detect_request_scheme( req: &Request, tls_protocol: Option<&str>, tls_cipher: Option<&str>, -) -> String { +) -> &'static str { // 1. First try ClientInfo TLS fields populated at the adapter entry point. if let Some(tls_protocol) = tls_protocol { log::debug!("TLS protocol detected: {tls_protocol}"); - return "https".to_owned(); + return "https"; } // Also check TLS cipher - if present, connection is HTTPS. if tls_cipher.is_some() { log::debug!("TLS cipher detected, using HTTPS"); - return "https".to_owned(); + return "https"; } // 2. Try the Forwarded header (RFC 7239) @@ -284,11 +301,11 @@ fn detect_request_scheme( && let Ok(ssl_str) = ssl.to_str() && (ssl_str == "1" || ssl_str.to_lowercase() == "true") { - return "https".to_owned(); + return "https"; } // Default to HTTP - "http".to_owned() + "http" } /// Build a static text response with strong `ETag` and standard caching headers. @@ -483,6 +500,41 @@ pub fn enforce_max_body_size( Ok(()) } +/// The namespace the inspection and attestation endpoints answer under. +pub const ROUTE_NAMESPACE: &str = "/_ts"; + +/// Response headers that describe the deployment and not the page. +const DEPLOYMENT_HEADERS: &[&str] = &["x-cache", "x-cache-hits", "x-served-by"]; + +/// Removes the headers that describe the deployment from a response, unless +/// the request was for a path in `namespace`, which is where a deployment is +/// asked about itself. +/// +/// An adapter calls this before it finalizes a response, so a header that +/// finalizing writes is not removed, an operator's `[response_headers]` +/// among them. The publisher fallback has already removed them from what the +/// origin sent, with [`remove_deployment_headers`], so a response under +/// `namespace` keeps them only when this server answered it. +pub fn strip_deployment_headers(path: &str, namespace: &str, response: &mut Response) { + // A path that only starts with the namespace's text is not in it. `/_tsx` + // is a publisher's page. + let in_namespace = path == namespace + || path + .strip_prefix(namespace) + .is_some_and(|rest| rest.starts_with('/')); + if !in_namespace { + remove_deployment_headers(response); + } +} + +/// Removes the headers that describe the deployment from a response, whatever +/// path it answers. +pub fn remove_deployment_headers(response: &mut Response) { + for name in DEPLOYMENT_HEADERS { + response.headers_mut().remove(*name); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1037,4 +1089,87 @@ mod tests { "should match text/html case-insensitively in fallback" ); } + + const STRIPPED: [&str; 3] = ["x-cache", "x-cache-hits", "x-served-by"]; + + fn response_with(names: &[&'static str]) -> Response<()> { + let mut response = Response::new(()); + for name in names { + response + .headers_mut() + .insert(*name, HeaderValue::from_static("value")); + } + response + } + + #[test] + fn a_publisher_page_loses_every_deployment_header() { + let mut response = response_with(&STRIPPED); + + strip_deployment_headers("/articles/an-article", ROUTE_NAMESPACE, &mut response); + + for name in STRIPPED { + assert!( + response.headers().get(name).is_none(), + "{name} should not reach a reader" + ); + } + } + + #[test] + fn the_servers_own_namespace_keeps_them() { + for path in ["/_ts", "/_ts/permissions", "/_ts/debug/ja4"] { + let mut response = response_with(&STRIPPED); + + strip_deployment_headers(path, ROUTE_NAMESPACE, &mut response); + + for name in STRIPPED { + assert!( + response.headers().get(name).is_some(), + "{path} should keep {name}" + ); + } + } + } + + #[test] + fn a_header_outside_the_list_is_left_alone() { + let kept = ["x-ts-version", "x-geo-info-available", "cache-control"]; + let mut response = response_with(&kept); + + strip_deployment_headers("/articles/an-article", ROUTE_NAMESPACE, &mut response); + + for name in kept { + assert!( + response.headers().get(name).is_some(), + "{name} should be left alone" + ); + } + } + + #[test] + fn a_path_that_only_starts_with_the_namespace_is_not_in_it() { + for path in ["/_tsx", "/_ts-archive", "/_tsfoo/bar"] { + let mut response = response_with(&["x-served-by"]); + + strip_deployment_headers(path, ROUTE_NAMESPACE, &mut response); + + assert!( + response.headers().get("x-served-by").is_none(), + "{path} is a publisher's page" + ); + } + } + + #[test] + fn the_boundary_is_the_namespace_passed_in() { + let mut kept = response_with(&["x-served-by"]); + let mut stripped = response_with(&["x-served-by"]); + + strip_deployment_headers("/_other/permissions", "/_other", &mut kept); + strip_deployment_headers("/_ts/permissions", "/_other", &mut stripped); + + assert!(kept.headers().get("x-served-by").is_some()); + assert!(stripped.headers().get("x-served-by").is_none()); + } } diff --git a/crates/trusted-server-core/src/inspect.rs b/crates/trusted-server-core/src/inspect.rs new file mode 100644 index 0000000000..11ff52fa73 --- /dev/null +++ b/crates/trusted-server-core/src/inspect.rs @@ -0,0 +1,145 @@ +//! What a deployment shows to anyone who asks, with no credential. +//! +//! A publisher can point a reader, a regulator or an auditor at these +//! addresses, and each sees what this deployment decided for their own +//! request without having to take anybody's word for it. Every address is +//! fixed under `/_ts`, so it is the same on every deployment. +//! +//! | Address | What it shows | +//! | --- | --- | +//! | [`permissions::PERMISSIONS_PAGE_PATH`] | The permissions resolved for the request, the signals that produced them and the terms the data is held under | +//! | [`config::CONFIG_PAGE_PATH`] | The settings the deployment is running, with every secret and every value sensitive by default masked | +//! | [`data::DATA_PAGE_PATH`] | What the deployment holds against the Edge Cookie the request carries, with every identifier masked | +//! +//! The first two answer as a page, and as data with `.json` added. The third +//! is about one reader, so it answers a browser opening it as a page and +//! nothing else. +//! +//! Operator data is not here. It is in the platform's own stores, reached +//! with the platform's own authentication, and never through the publisher's +//! domain. + +use serde_json::Value; + +pub mod config; +pub mod data; +pub mod permissions; + +/// What a masked value shows as. +pub const MASK: &str = "XXXX"; + +const DATA_FORM_LEDE: &str = + "Add .json to this address for the same information as data."; + +/// Renders `payload` as a page titled `title`. +#[must_use] +pub fn render_page(title: &str, payload: &Value) -> String { + page(title, DATA_FORM_LEDE, payload) +} + +/// Renders `payload` as [`render_page`] does, under `lede` in place of the +/// line saying where the data form is, for an address that has none. +#[must_use] +pub fn render_page_with_lede(title: &str, lede: &str, payload: &Value) -> String { + page(title, &html_escape(lede), payload) +} + +/// The page, with `lede` written into it as markup. +fn page(title: &str, lede: &str, payload: &Value) -> String { + let pretty = serde_json::to_string_pretty(payload).unwrap_or_else(|_| "{}".to_owned()); + format!( + "\n\ +\n\ +\n\ +\n\ +\n\ +\n\ +{title}\n\ +\n\ +\n\ +\n\ +
\n\ +

{title}

\n\ +

{lede}

\n\ +
{body}
\n\ +
\n\ +\n\ +\n", + title = html_escape(title), + body = html_escape(&pretty), + ) +} + +/// Escapes `value` for HTML text content. +pub(crate) fn html_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn a_page_escapes_whatever_its_payload_and_title_carry() { + let html = render_page( + "A ", + &json!({ "value": "</pre><script>x()</script> & more" }), + ); + + assert!( + !html.contains("<script>x()"), + "should not let a value open a tag" + ); + assert!( + html.contains("</pre><script>x()</script> & more"), + "should show the value as text" + ); + assert!( + html.contains("<title>A <title>"), + "should escape the title as well" + ); + } + + #[test] + fn a_page_tells_the_reader_where_the_data_form_is() { + let html = render_page("Permissions", &json!({})); + + assert!( + html.contains(".json"), + "should say how to get the same information as data" + ); + assert!( + html.contains(""), + "should keep the page out of search indexes" + ); + } + + #[test] + fn a_page_with_a_lede_of_its_own_says_that_instead() { + let html = render_page_with_lede("Your data", "Held for & no other", &json!({})); + + assert!( + html.contains("

Held for <you> & no other

"), + "should show the lede as text" + ); + assert!( + !html.contains(".json"), + "should not point at a data form the address does not have" + ); + } +} diff --git a/crates/trusted-server-core/src/inspect/config.rs b/crates/trusted-server-core/src/inspect/config.rs new file mode 100644 index 0000000000..0e0a5171bf --- /dev/null +++ b/crates/trusted-server-core/src/inspect/config.rs @@ -0,0 +1,714 @@ +//! The configuration a deployment is running, served to anyone at +//! [`CONFIG_PAGE_PATH`] and [`CONFIG_JSON_PATH`]. +//! +//! What is shown is the settings the handler holds, after defaults have been +//! applied, with values masked as [`MASK`]: +//! +//! 1. Every secret, meaning a value the loader filled in from the secret +//! store or a leaf declared as one, whatever the publisher's document says. +//! 2. Every value sensitive by default, being a settings field serialized +//! with [`crate::redacted::sensitive`] and every +//! [`Redacted`](crate::redacted::Redacted) value, unless `[inspect] show` +//! names it. +//! 3. Everything `[inspect] hide` names. + +use std::collections::HashSet; +use std::fmt::{self, Write as _}; + +use edgezero_core::app_config::{AppConfigMeta as _, SecretPathSegment}; +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderValue, Request, Response, StatusCode, header}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::{Map, Value, json}; + +use crate::config::TrustedServerAppConfig; +use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; +use crate::redacted::{SENSITIVE_MARKER, to_marked_value}; +use crate::secret_resolution::unresolved_secrets; +use crate::settings::Settings; + +use super::{MASK, render_page}; + +/// The page form of the configuration. +pub const CONFIG_PAGE_PATH: &str = "/_ts/config"; + +/// The JSON form of the configuration. +pub const CONFIG_JSON_PATH: &str = "/_ts/config.json"; + +/// Both addresses, the page first. +pub const CONFIG_PATHS: [&str; 2] = [CONFIG_PAGE_PATH, CONFIG_JSON_PATH]; + +const NOT_PUBLISHED: &str = "This publisher does not publish its configuration."; + +const PATTERN_FORM: &str = "A pattern is keys joined by `.`, with `[]` for every element of a \ + list or `[N]` for one, such as \ + `proxy.asset_routes[0].origin_url`"; + +/// The `[inspect]` section, which says what the configuration page shows. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct InspectConfig { + /// Whether the configuration is published. `false` answers not found. + #[serde(default = "default_config")] + pub config: bool, + /// Values masked by default that are shown instead. A secret cannot be. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub show: Vec, + /// Values masked as well as the defaults. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hide: Vec, +} + +fn default_config() -> bool { + true +} + +impl Default for InspectConfig { + fn default() -> Self { + Self { + config: default_config(), + show: Vec::new(), + hide: Vec::new(), + } + } +} + +impl InspectConfig { + /// Whether the section says nothing the defaults do not. + #[must_use] + pub fn is_default(&self) -> bool { + *self == Self::default() + } +} + +/// One step of a concrete path through the view. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum PathStep { + /// A key of an object. + Key(String), + /// An index into a list. + Index(usize), +} + +/// A concrete path written the way a pattern is, such as +/// `proxy.asset_routes[0].origin_url`. +#[must_use] +pub fn render_path(steps: &[PathStep]) -> String { + let mut out = String::new(); + for step in steps { + match step { + PathStep::Key(key) => { + if !out.is_empty() { + out.push('.'); + } + out.push_str(key); + } + PathStep::Index(index) => { + let _ = write!(out, "[{index}]"); + } + } + } + out +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum PatternStep { + Key(String), + Each, + Index(usize), +} + +/// Values in the view named by keys joined by `.`, with `[]` for every +/// element of a list and `[N]` for one, such as +/// `proxy.asset_routes[].origin_url` or `proxy.asset_routes[0].prefix`. +/// +/// A pattern names the values at exactly its own depth. A key holding `.`, +/// `[` or `]` cannot be named. +#[derive(Clone, PartialEq, Eq)] +pub struct PathPattern { + text: String, + steps: Vec, +} + +impl PathPattern { + /// Reads a pattern. + /// + /// # Errors + /// + /// A message naming `text` and what is wrong with it. + pub fn parse(text: &str) -> Result { + let malformed = + |why: &str| format!("`{text}` is not a path pattern, because {why}. {PATTERN_FORM}"); + let mut steps = Vec::new(); + let mut rest = text; + loop { + let end = rest.find(['.', '[', ']']).unwrap_or(rest.len()); + let (key, after_key) = rest.split_at(end); + if key.is_empty() { + return Err(malformed("a key is empty")); + } + if key.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err(malformed("a key holds a space or a control character")); + } + steps.push(PatternStep::Key(key.to_owned())); + rest = after_key; + while let Some(open) = rest.strip_prefix('[') { + let Some((inside, after)) = open.split_once(']') else { + return Err(malformed("a `[` is not closed")); + }; + let step = if inside.is_empty() { + PatternStep::Each + } else if inside.bytes().all(|b| b.is_ascii_digit()) { + PatternStep::Index( + inside + .parse() + .map_err(|_| malformed("an index is too large"))?, + ) + } else { + return Err(malformed("only a number may sit between `[` and `]`")); + }; + steps.push(step); + rest = after; + } + if rest.is_empty() { + break; + } + match rest.strip_prefix('.') { + Some(after) => rest = after, + None if rest.starts_with(']') => return Err(malformed("a `]` has no `[`")), + None => return Err(malformed("a `]` is followed by neither `.` nor `[`")), + } + } + Ok(Self { + text: text.to_owned(), + steps, + }) + } + + /// The pattern a secret or sensitive declaration names, or `None` for a + /// segment kind a pattern cannot express. + #[must_use] + pub fn from_segments(segments: &[SecretPathSegment]) -> Option { + let mut steps = Vec::with_capacity(segments.len()); + let mut text = String::new(); + for segment in segments { + match segment { + SecretPathSegment::Field(name) | SecretPathSegment::OptionalField(name) => { + if !text.is_empty() { + text.push('.'); + } + text.push_str(name); + steps.push(PatternStep::Key(name.to_string())); + } + SecretPathSegment::ArrayEach => { + text.push_str("[]"); + steps.push(PatternStep::Each); + } + _ => return None, + } + } + Some(Self { text, steps }) + } + + /// The pattern as written. + #[must_use] + pub fn as_str(&self) -> &str { + &self.text + } + + /// Whether the concrete path `path` is one the pattern names. + #[must_use] + pub fn matches(&self, path: &[PathStep]) -> bool { + self.steps.len() == path.len() + && self + .steps + .iter() + .zip(path) + .all(|(pattern, step)| match (pattern, step) { + (PatternStep::Key(want), PathStep::Key(got)) => want == got, + (PatternStep::Each, PathStep::Index(_)) => true, + (PatternStep::Index(want), PathStep::Index(got)) => want == got, + _ => false, + }) + } +} + +impl fmt::Debug for PathPattern { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.text) + } +} + +impl fmt::Display for PathPattern { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.text) + } +} + +impl Serialize for PathPattern { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.text) + } +} + +impl<'de> Deserialize<'de> for PathPattern { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let text = String::deserialize(deserializer)?; + Self::parse(&text).map_err(serde::de::Error::custom) + } +} + +/// The configuration as the endpoint shows it. +#[derive(Debug, Clone, PartialEq)] +pub struct ConfigView { + /// Every masked path, concrete and sorted. + pub masked: Vec, + /// The settings, each masked value replaced by [`MASK`] and every + /// object's keys sorted. + pub settings: Value, +} + +/// Builds the view of `settings`. +/// +/// # Errors +/// +/// When the settings cannot be represented as JSON. +pub fn build_view(settings: &Settings) -> Result> { + let marked = marked_settings(settings)?; + Ok(Rules::new(settings).render(&marked)) +} + +fn marked_settings(settings: &Settings) -> Result> { + // The serializer's error is left out, because nothing may carry a value + // out of the settings. + to_marked_value(settings).map_err(|_| { + Report::new(TrustedServerError::Configuration { + message: "the settings could not be represented for the configuration view".to_owned(), + }) + }) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum List { + Show, + Hide, +} + +impl fmt::Display for List { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Show => "show", + Self::Hide => "hide", + }) + } +} + +/// Everything that decides whether a value is masked. +#[derive(Clone)] +struct Rules<'a> { + secret_paths: &'a [Vec], + secret_patterns: Vec, + secret_values: &'a [String], + show: Vec<&'a PathPattern>, + hide: Vec<&'a PathPattern>, +} + +impl<'a> Rules<'a> { + fn new(settings: &'a Settings) -> Self { + let secrets = settings.resolved_secrets(); + // Core's own declarations are added here as well as recorded by the + // loader, so settings built any other way still mask them. + let secret_patterns = secrets + .patterns() + .iter() + .cloned() + .chain( + TrustedServerAppConfig::secret_fields() + .iter() + .filter_map(|field| PathPattern::from_segments(&field.path)), + ) + .collect(); + Self { + secret_paths: secrets.paths(), + secret_patterns, + secret_values: secrets.values(), + show: settings.inspect.show.iter().collect(), + hide: settings.inspect.hide.iter().collect(), + } + } + + /// These rules less one of the publisher's patterns. + fn without(&self, list: List, index: usize) -> Self { + let mut rules = self.clone(); + let _removed = match list { + List::Show => rules.show.remove(index), + List::Hide => rules.hide.remove(index), + }; + rules + } + + fn is_secret(&self, path: &[PathStep]) -> bool { + self.secret_paths.iter().any(|secret| secret == path) + || self + .secret_patterns + .iter() + .any(|pattern| pattern.matches(path)) + } + + fn render(&self, marked: &Value) -> ConfigView { + let mut masked = Vec::new(); + let mut path = Vec::new(); + let mut settings = self.mask(marked, &mut path, &mut masked); + self.scan(&mut settings, &mut path, &mut masked); + masked.sort(); + masked.dedup(); + ConfigView { + masked: masked.iter().map(|steps| render_path(steps)).collect(), + settings, + } + } + + /// `node` with every sensitive marker resolved, every masked value + /// replaced and every object's keys sorted. + fn mask( + &self, + node: &Value, + path: &mut Vec, + masked: &mut Vec>, + ) -> Value { + let (inner, marked_sensitive) = match unmark(node) { + Some(inner) => (inner, true), + None => (node, false), + }; + if is_empty(inner) { + return inner.clone(); + } + let shown = self.show.iter().any(|p| p.matches(path)); + if self.is_secret(path) + || self.hide.iter().any(|p| p.matches(path)) + || (marked_sensitive && !shown) + { + masked.push(path.clone()); + return Value::String(MASK.to_owned()); + } + match inner { + Value::Object(map) => { + let mut entries: Vec<(&String, &Value)> = map.iter().collect(); + entries.sort_unstable_by(|left, right| left.0.cmp(right.0)); + let mut out = Map::new(); + for (key, value) in entries { + path.push(PathStep::Key(key.clone())); + out.insert(key.clone(), self.mask(value, path, masked)); + path.pop(); + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + path.push(PathStep::Index(index)); + out.push(self.mask(item, path, masked)); + path.pop(); + } + Value::Array(out) + } + other => other.clone(), + } + } + + /// Masks every string, and every key, that contains a secret value, + /// wherever it sits. A key holding a secret is renamed [`MASK`], or + /// `XXXX-2` and on where that name is taken, and every path already + /// recorded beneath it is replaced by the renamed one, so no recorded path + /// carries the secret. + fn scan(&self, node: &mut Value, path: &mut Vec, masked: &mut Vec>) { + match node { + Value::String(text) => { + if self.holds_secret(text) { + masked.push(path.clone()); + *node = Value::String(MASK.to_owned()); + } + } + Value::Array(items) => { + for (index, item) in items.iter_mut().enumerate() { + path.push(PathStep::Index(index)); + self.scan(item, path, masked); + path.pop(); + } + } + Value::Object(map) => { + let entries = std::mem::take(map); + let kept: HashSet = entries + .keys() + .filter(|key| !self.holds_secret(key)) + .cloned() + .collect(); + let mut sorted = std::collections::BTreeMap::new(); + for (key, mut value) in entries { + if self.holds_secret(&key) { + let mut renamed = MASK.to_owned(); + let mut copy = 1; + while kept.contains(&renamed) || sorted.contains_key(&renamed) { + copy += 1; + renamed = format!("{MASK}-{copy}"); + } + path.push(PathStep::Key(key)); + masked.retain(|recorded| !recorded.starts_with(path)); + path.pop(); + path.push(PathStep::Key(renamed.clone())); + masked.push(path.clone()); + path.pop(); + sorted.insert(renamed, Value::String(MASK.to_owned())); + continue; + } + path.push(PathStep::Key(key.clone())); + self.scan(&mut value, path, masked); + path.pop(); + sorted.insert(key, value); + } + map.extend(sorted); + } + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } + } + + fn holds_secret(&self, text: &str) -> bool { + self.secret_values + .iter() + .any(|secret| !secret.is_empty() && text.contains(secret.as_str())) + } +} + +/// The value a sensitive marker wraps, or `None` when `node` is not one. +fn unmark(node: &Value) -> Option<&Value> { + match node { + Value::Object(map) if map.len() == 1 => map.get(SENSITIVE_MARKER), + _ => None, + } +} + +/// Whether there is nothing in `value` to hide. +fn is_empty(value: &Value) -> bool { + match value { + Value::Null => true, + Value::String(text) => text.is_empty(), + Value::Array(items) => items.is_empty(), + Value::Object(map) => map.is_empty(), + Value::Bool(_) | Value::Number(_) => false, + } +} + +/// The concrete paths of the values in `marked` that `pattern` names. +fn select(marked: &Value, pattern: &PathPattern) -> Vec> { + fn walk( + node: &Value, + steps: &[PatternStep], + path: &mut Vec, + out: &mut Vec>, + ) { + let node = unmark(node).unwrap_or(node); + let Some((step, rest)) = steps.split_first() else { + out.push(path.clone()); + return; + }; + match (step, node) { + (PatternStep::Key(key), Value::Object(map)) => { + if let Some(child) = map.get(key) { + path.push(PathStep::Key(key.clone())); + walk(child, rest, path, out); + path.pop(); + } + } + (PatternStep::Each, Value::Array(items)) => { + for (index, item) in items.iter().enumerate() { + path.push(PathStep::Index(index)); + walk(item, rest, path, out); + path.pop(); + } + } + (PatternStep::Index(index), Value::Array(items)) => { + if let Some(item) = items.get(*index) { + path.push(PathStep::Index(*index)); + walk(item, rest, path, out); + path.pop(); + } + } + _ => {} + } + } + let mut out = Vec::new(); + walk(marked, &pattern.steps, &mut Vec::new(), &mut out); + out +} + +/// Refuses an `[inspect]` pattern that would not be honored as written, +/// being one that matches nothing, a `show` naming a secret, one another +/// pattern of its list already covers, a `show` a `hide` covers, one that +/// changes nothing the view shows, and any pattern beside `config = false`. +/// Costs nothing when the section has no patterns. +/// +/// # Errors +/// +/// Naming the first pattern at fault. +pub(crate) fn validate_patterns(settings: &Settings) -> Result<(), Report> { + let inspect = &settings.inspect; + if inspect.show.is_empty() && inspect.hide.is_empty() { + return Ok(()); + } + if !inspect.config { + return Err(configuration( + "[inspect] show and hide say what the configuration page shows, and config = false \ + publishes no page, so they change nothing. Remove them, or publish the page" + .to_owned(), + )); + } + let marked = marked_settings(settings)?; + let rules = Rules::new(settings); + let full = rules.render(&marked); + for (list, patterns) in [(List::Show, &inspect.show), (List::Hide, &inspect.hide)] { + for (index, pattern) in patterns.iter().enumerate() { + let selected = select(&marked, pattern); + if selected.is_empty() { + return Err(configuration(format!( + "[inspect] {list} names `{pattern}`, which matches no value in this \ + configuration, so it changes nothing" + ))); + } + if list == List::Show && selected.iter().any(|path| rules.is_secret(path)) { + return Err(configuration(format!( + "[inspect] show names `{pattern}`, which is a secret. A secret is always \ + masked, so it cannot be shown" + ))); + } + let covers = |other: &PathPattern| { + let covered = select(&marked, other); + selected.iter().all(|path| covered.contains(path)) + }; + if let Some(other) = patterns + .iter() + .enumerate() + .find(|(other_index, other)| *other_index != index && covers(other)) + .map(|(_, other)| other) + { + return Err(configuration(format!( + "[inspect] {list} names `{pattern}`, which changes nothing, because \ + `{other}` in the same list already names everything it does" + ))); + } + if list == List::Show + && let Some(hide) = inspect.hide.iter().find(|hide| covers(hide)) + { + return Err(configuration(format!( + "[inspect] show names `{pattern}`, which changes nothing, because hide \ + names it as `{hide}` and hide wins over show" + ))); + } + if rules.without(list, index).render(&marked) == full { + let reason = match list { + List::Show => "nothing it names is masked by default", + List::Hide => "everything it names is masked or empty already", + }; + return Err(configuration(format!( + "[inspect] {list} names `{pattern}`, which changes nothing, because {reason}" + ))); + } + } + } + Ok(()) +} + +/// [`validate_patterns`] for a deployment's settings before they are pushed, +/// when their secret fields still hold key names. The view is handed every +/// secret leaf the load will look for, the ones only the configuration and +/// `extra_integrations` name included, so a pattern the load refuses is +/// refused here. +/// +/// # Errors +/// +/// As [`validate_patterns`]. +pub(crate) fn validate_patterns_for_deploy( + settings: &Settings, + extra_integrations: &[IntegrationBuilder], +) -> Result<(), Report> { + if settings.inspect.show.is_empty() && settings.inspect.hide.is_empty() { + return Ok(()); + } + // The serializer's error is left out, as in `marked_settings`. + let data = serde_json::to_value(settings).map_err(|_| { + configuration("the settings could not be represented for the configuration view".to_owned()) + })?; + let builders = crate::integrations::all_builders(extra_integrations).collect::>(); + let mut settings = settings.clone(); + settings.set_resolved_secrets(unresolved_secrets::( + &data, + crate::module_secrets::secret_fields(&builders), + )); + validate_patterns(&settings) +} + +fn configuration(message: String) -> Report { + Report::new(TrustedServerError::Configuration { message }) +} + +/// Answers [`CONFIG_PAGE_PATH`] with the page and [`CONFIG_JSON_PATH`] with +/// the JSON, or not found when the publisher does not publish its +/// configuration. +#[must_use] +pub fn handle_config(settings: &Settings, req: &Request) -> Response { + let json = req.uri().path() == CONFIG_JSON_PATH; + if !settings.inspect.config { + let payload = json!({ "error": NOT_PUBLISHED }); + return answer(StatusCode::NOT_FOUND, json, &payload); + } + match build_view(settings) { + Ok(view) => answer(StatusCode::OK, json, &view_payload(&view)), + Err(_) => { + log::error!("the configuration view could not be built"); + let payload = json!({ "error": "The configuration could not be shown." }); + answer(StatusCode::INTERNAL_SERVER_ERROR, json, &payload) + } + } +} + +/// What the endpoint serves for `view`, with the version of the core crate +/// that answered. +#[must_use] +pub fn view_payload(view: &ConfigView) -> Value { + json!({ + "version": env!("CARGO_PKG_VERSION"), + "masked": view.masked, + "settings": view.settings, + }) +} + +fn answer(status: StatusCode, json: bool, payload: &Value) -> Response { + let (content_type, body) = if json { + ("application/json", payload.to_string()) + } else { + ( + "text/html; charset=utf-8", + render_page("Configuration", payload), + ) + }; + Response::builder() + .status(status) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store")) + .header( + header::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("*"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build configuration response") +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/inspect/config/tests.rs b/crates/trusted-server-core/src/inspect/config/tests.rs new file mode 100644 index 0000000000..f58d3f0296 --- /dev/null +++ b/crates/trusted-server-core/src/inspect/config/tests.rs @@ -0,0 +1,926 @@ +use std::collections::BTreeSet; + +use edgezero_core::blob_envelope::BlobEnvelope; + +use super::*; +use crate::config_payload::settings_from_config_blob_with; +use crate::integrations::{IntegrationBuilder, ModuleSecretSetting}; +use crate::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; +use crate::secret_resolution::ResolvedSecrets; +use crate::settings::ProxyAssetRoute; +use crate::test_support::tests::crate_test_settings_str; + +/// Every value this store hands back starts with this, so a view can be +/// searched for all of them at once. +const CANARY: &str = "canary-"; + +/// Resolves every key name to a value of its own, long enough for any +/// minimum length a secret has. +struct CanaryStore; + +impl PlatformSecretStore for CanaryStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + Ok(format!("{CANARY}{key}-value-0123456789abcdef").into_bytes()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } +} + +fn settings() -> Settings { + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") +} + +/// The test settings with `extra` appended, as a stored document holds +/// them, so every secret in them is the name of one. +fn document(extra: &str) -> Value { + document_from(&format!("{}\n{extra}\n", crate_test_settings_str())) +} + +/// `toml` as a stored document holds it, with the allowed domains a load +/// needs for the test settings' asset hosts. +fn document_from(toml: &str) -> Value { + let mut settings = Settings::from_toml(toml).expect("should parse the document"); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + serde_json::to_value(&settings).expect("should serialize the document") +} + +fn load_with( + data: Value, + builders: &[IntegrationBuilder], +) -> Result> { + let envelope = BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_owned()); + let envelope = serde_json::to_string(&envelope).expect("should serialize the envelope"); + settings_from_config_blob_with( + &envelope, + &CanaryStore, + &StoreName::from("ts_secrets"), + builders, + ) +} + +fn load(data: Value) -> Result> { + load_with(data, &[]) +} + +fn view(settings: &Settings) -> ConfigView { + build_view(settings).expect("should build the view") +} + +fn request(uri: &str) -> Request { + Request::builder() + .uri(uri) + .body(EdgeBody::empty()) + .expect("should build a request") +} + +fn body_text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("a buffered body") + .to_vec(); + String::from_utf8(bytes).expect("a text body") +} + +/// The JSON and the page the endpoint serves for `settings`. +fn served(settings: &Settings) -> (String, String) { + ( + body_text(handle_config(settings, &request(CONFIG_JSON_PATH))), + body_text(handle_config(settings, &request(CONFIG_PAGE_PATH))), + ) +} + +/// The value at a concrete path, such as `proxy.asset_routes[0].origin_url`. +fn at<'v>(value: &'v Value, path: &str) -> Option<&'v Value> { + let pattern = PathPattern::parse(path).expect("a concrete path"); + let mut node = value; + for step in &pattern.steps { + node = match (step, node) { + (PatternStep::Key(key), Value::Object(map)) => map.get(key)?, + (PatternStep::Index(index), Value::Array(items)) => items.get(*index)?, + _ => return None, + }; + } + Some(node) +} + +fn is_masked(view: &ConfigView, path: &str) -> bool { + view.masked.iter().any(|masked| masked == path) +} + +/// `value` with every sensitive marker replaced by what it wraps. +fn unmarked(value: &Value) -> Value { + if let Some(inner) = unmark(value) { + return unmarked(inner); + } + match value { + Value::Object(map) => Value::Object( + map.iter() + .map(|(key, value)| (key.clone(), unmarked(value))) + .collect(), + ), + Value::Array(items) => Value::Array(items.iter().map(unmarked).collect()), + other => other.clone(), + } +} + +#[test] +fn a_pattern_names_keys_every_element_or_one_element() { + let every = PathPattern::parse("proxy.asset_routes[].origin_url").expect("well formed"); + let one = PathPattern::parse("proxy.asset_routes[1].origin_url").expect("well formed"); + let route = |index| { + vec![ + PathStep::Key("proxy".to_owned()), + PathStep::Key("asset_routes".to_owned()), + PathStep::Index(index), + PathStep::Key("origin_url".to_owned()), + ] + }; + + assert!(every.matches(&route(0)) && every.matches(&route(7))); + assert!(one.matches(&route(1))); + assert!(!one.matches(&route(0)), "should name only its own index"); + assert!( + !every.matches(&route(0)[..3]), + "should name only values at its own depth" + ); + assert_eq!(render_path(&route(1)), "proxy.asset_routes[1].origin_url"); +} + +#[test] +fn a_malformed_pattern_is_refused_when_the_document_is_read() { + for pattern in [ + "", + "a..b", + ".a", + "a.", + "a[x]", + "a[", + "a]", + "[0]", + "a b", + "a[0]b", + "a[99999999999999999999999]", + ] { + let toml = format!( + "{}\n[inspect]\nhide = [{pattern:?}]\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("a malformed pattern is refused"); + + assert!( + format!("{error:?}").contains(&format!("`{pattern}` is not a path pattern")), + "should name the pattern {pattern:?}: {error:?}" + ); + } +} + +#[test] +fn a_key_the_inspect_section_does_not_have_is_refused_naming_it() { + let toml = format!("{}\n[inspect]\nshows = []\n", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("a misspelt key is refused"); + + assert!( + format!("{error:?}").contains("unknown field `shows`"), + "should name the key: {error:?}" + ); +} + +/// Documents that between them hold a secret at every path core declares +/// one, each a key name the canary store resolves to a value of its own. +fn documents_holding_every_declared_secret() -> Vec<(&'static str, Value)> { + let mut main = document(""); + main["publisher"]["proxy_secret"] = json!("proxy"); + main["ec"]["hmac"]["passphrase"] = json!("hmac"); + main["ec"]["partners"] = json!([{ + "name": "Example Partner", + "source_domain": "partner.example.com", + "api_token": "partner-api", + "pull_sync_enabled": true, + "pull_sync_url": "https://partner.example.com/sync", + "pull_sync_allowed_domains": ["partner.example.com"], + "ts_pull_token": "partner-pull", + }]); + main["trusted_client_ip"] = json!({ + "ip_header": "x-ts-client-ip", + "auth_header": "x-ts-client-ip-auth", + "shared_secret": "client-ip", + }); + main["proxy"]["asset_routes"] = json!([{ + "prefix": "/assets/", + "origin_url": "https://assets.example.com", + "auth": { + "type": "s3_sigv4", + "region": "us-east-1", + "access_key_id": "route-access", + "secret_access_key": "route-secret", + "session_token": "route-session", + }, + }]); + main["analytics"] = json!({ + "module": "tinybird", + "tinybird": { + "api_host": "api.tinybird.example.com", + "auction_token_secret": "telemetry", + }, + }); + + let mut host_signals = document(""); + let ec = host_signals["ec"].as_object_mut().expect("an ec table"); + ec.remove("hmac"); + ec.insert("module".to_owned(), json!("host_signals")); + ec.insert("host_signals".to_owned(), json!({ "passphrase": "host" })); + + let mut legacy = document(""); + let ec = legacy["ec"].as_object_mut().expect("an ec table"); + ec.remove("hmac"); + ec.remove("module"); + ec.insert("passphrase".to_owned(), json!("legacy")); + + let mut labeled = document(""); + let ec = labeled["ec"].as_object_mut().expect("an ec table"); + ec.remove("hmac"); + ec.insert("module".to_owned(), json!("primary")); + ec.insert( + "primary".to_owned(), + json!({ "implementation": "hmac", "passphrase": "labeled" }), + ); + + vec![ + ("main", main), + ("host signals", host_signals), + ("legacy", legacy), + ("labeled", labeled), + ] +} + +/// A concrete path with every index written as `[]`, as a declaration +/// writes it. +fn generalized(path: &[PathStep]) -> String { + let mut out = String::new(); + for step in path { + match step { + PathStep::Key(key) => { + if !out.is_empty() { + out.push('.'); + } + out.push_str(key); + } + PathStep::Index(_) => out.push_str("[]"), + } + } + out +} + +/// Every secret core declares, loaded the way a deployment loads it, is +/// live in the settings and absent from both forms the endpoint serves, +/// and wherever the view still has its path, that path is masked. +#[test] +fn every_declared_core_secret_is_masked_and_never_served() { + let mut covered = BTreeSet::new(); + for (name, data) in documents_holding_every_declared_secret() { + let settings = load(data).unwrap_or_else(|error| panic!("{name}: {error:?}")); + let record = settings.resolved_secrets(); + let normal = serde_json::to_string(&settings).expect("should serialize"); + assert!( + !record.values().is_empty(), + "{name}: the load should record the secrets it wrote" + ); + for value in record.values() { + assert!( + normal.contains(value.as_str()), + "{name}: each resolved secret should be live in the settings" + ); + } + let view = view(&settings); + for path in record.paths() { + let rendered = render_path(path); + if let Some(value) = at(&view.settings, &rendered) { + assert_eq!(value, MASK, "{name}: {rendered}"); + assert!(is_masked(&view, &rendered), "{name}: {rendered}"); + } + covered.insert(generalized(path)); + } + let (json, page) = served(&settings); + assert!(!json.contains(CANARY), "{name}: the JSON carries a secret"); + assert!(!page.contains(CANARY), "{name}: the page carries a secret"); + } + + let declared: BTreeSet = TrustedServerAppConfig::secret_fields() + .iter() + .filter_map(|field| PathPattern::from_segments(&field.path)) + .map(|pattern| pattern.as_str().to_owned()) + .collect(); + let missing: Vec<&String> = declared.difference(&covered).collect(); + assert!( + missing.is_empty(), + "every declared secret should be exercised, missing {missing:?}" + ); +} + +/// Test settings holding a value of its own in every field marked +/// sensitive, with those paths and values. +fn settings_with_every_sensitive_field() -> (Settings, Vec<(&'static str, &'static str)>) { + let expected = vec![ + ("publisher.origin_url", "https://origin-canary.example.com"), + ( + "publisher.origin_host_header_override", + "host-canary.example.com", + ), + ( + "proxy.asset_routes[0].origin_url", + "https://bucket-canary.example.com", + ), + ("ec.ec_store", "identity-store-canary"), + ("auction.creative_store", "creative-store-canary"), + ]; + let value = |path: &str| { + expected + .iter() + .find(|(at, _)| *at == path) + .map(|(_, value)| (*value).to_owned()) + .expect("an expected path") + }; + let mut settings = settings(); + settings.publisher.origin_url = value("publisher.origin_url"); + settings.publisher.origin_host_header_override = + Some(value("publisher.origin_host_header_override")); + settings.proxy.asset_routes = vec![ProxyAssetRoute::new( + "/assets/", + value("proxy.asset_routes[0].origin_url"), + )]; + settings.ec.ec_store = Some(value("ec.ec_store")); + settings.auction.creative_store = value("auction.creative_store"); + (settings, expected) +} + +#[test] +fn every_value_sensitive_by_default_is_masked() { + let (settings, expected) = settings_with_every_sensitive_field(); + + let view = view(&settings); + let (json, page) = served(&settings); + + for (path, value) in expected { + assert_eq!(at(&view.settings, path), Some(&json!(MASK)), "{path}"); + assert!(is_masked(&view, path), "{path}"); + assert!(!json.contains(value), "the JSON carries {path}"); + assert!(!page.contains(value), "the page carries {path}"); + } + assert!( + is_masked(&view, "publisher.proxy_secret"), + "a Redacted value is masked too: {:?}", + view.masked + ); +} + +#[test] +fn an_ordinary_serialization_is_unchanged() { + let (settings, expected) = settings_with_every_sensitive_field(); + + let normal = serde_json::to_value(&settings).expect("should serialize"); + let marked = to_marked_value(&settings).expect("should serialize"); + + assert_eq!( + normal, + unmarked(&marked), + "should differ from a marked serialization only by the markers" + ); + assert!(!normal.to_string().contains(SENSITIVE_MARKER)); + for (path, value) in expected { + assert_eq!(at(&normal, path), Some(&json!(value)), "{path}"); + } + assert_eq!( + at(&normal, "publisher.proxy_secret"), + Some(&json!("unit-test-proxy-secret")) + ); +} + +#[test] +fn a_stored_document_carries_the_section_only_when_the_publisher_wrote_one() { + let unwritten = serde_json::to_value(settings()).expect("should serialize"); + let written = document("[inspect]\nhide = [\"publisher.domain\"]"); + + assert_eq!( + unwritten.get("inspect"), + None, + "should leave an unwritten section out, so the document is the bytes it was" + ); + assert_eq!( + written.get("inspect"), + Some(&json!({ "config": true, "hide": ["publisher.domain"] })), + "should store a section the publisher wrote" + ); + assert!( + !unwritten.to_string().contains("resolved_secrets"), + "should never store where the loader wrote secrets" + ); +} + +#[test] +fn a_recorded_secret_path_is_masked_without_the_value_scan() { + let mut settings = settings(); + settings.set_resolved_secrets(ResolvedSecrets::recorded( + vec![vec![ + PathStep::Key("publisher".to_owned()), + PathStep::Key("domain".to_owned()), + ]], + Vec::new(), + )); + + let view = view(&settings); + + assert_eq!(at(&view.settings, "publisher.domain"), Some(&json!(MASK))); + assert!(is_masked(&view, "publisher.domain")); +} + +#[test] +fn a_secret_copied_into_an_undeclared_field_is_masked_by_the_value_scan() { + let secret = "canary-copied-secret-value"; + let mut settings = settings(); + settings + .response_headers + .insert("x-debug".to_owned(), format!("prefix {secret} suffix")); + settings + .response_headers + .insert(format!("x-{secret}"), "open".to_owned()); + settings.set_resolved_secrets(ResolvedSecrets::recorded( + Vec::new(), + vec![secret.to_owned()], + )); + + let view = view(&settings); + let (json, page) = served(&settings); + + assert_eq!( + at(&view.settings, "response_headers.x-debug"), + Some(&json!(MASK)) + ); + assert!(is_masked(&view, "response_headers.x-debug")); + assert!( + is_masked(&view, "response_headers.XXXX"), + "a key holding a secret is masked too: {:?}", + view.masked + ); + assert!(!json.contains(secret) && !page.contains(secret)); +} + +#[test] +fn a_path_masked_beneath_a_key_holding_a_secret_lists_the_renamed_key() { + struct Case { + name: &'static str, + real_mask_key: bool, + renamed: [&'static str; 2], + } + let secret = "canary-key-secret-value"; + // Upper case names sort ahead of a real `XXXX` key, so they are + // renamed before it is reached. + let keys = [format!("A-{secret}"), format!("B-{secret}")]; + let path_to = |key: &str| { + vec![ + PathStep::Key("response_headers".to_owned()), + PathStep::Key(key.to_owned()), + ] + }; + for case in [ + Case { + name: "two keys holding the secret", + real_mask_key: false, + renamed: ["response_headers.XXXX", "response_headers.XXXX-2"], + }, + Case { + name: "beside a real key named XXXX", + real_mask_key: true, + renamed: ["response_headers.XXXX-2", "response_headers.XXXX-3"], + }, + ] { + let mut settings = settings(); + for key in &keys { + settings + .response_headers + .insert(key.clone(), "open".to_owned()); + } + if case.real_mask_key { + settings + .response_headers + .insert(MASK.to_owned(), "real".to_owned()); + } + settings.set_resolved_secrets(ResolvedSecrets::recorded( + keys.iter().map(|key| path_to(key)).collect(), + vec![secret.to_owned()], + )); + + let view = view(&settings); + let (json, page) = served(&settings); + + assert!( + !view.masked.iter().any(|path| path.contains(secret)), + "{}: no masked path should carry the secret: {:?}", + case.name, + view.masked + ); + for renamed in case.renamed { + assert!( + is_masked(&view, renamed), + "{}: should list {renamed}: {:?}", + case.name, + view.masked + ); + assert_eq!( + at(&view.settings, renamed), + Some(&json!(MASK)), + "{}", + case.name + ); + } + if case.real_mask_key { + assert_eq!( + at(&view.settings, "response_headers.XXXX"), + Some(&json!("real")), + "{}: should keep the real key's value", + case.name + ); + assert!( + !is_masked(&view, "response_headers.XXXX"), + "{}: should not list the real key as masked: {:?}", + case.name, + view.masked + ); + } + assert!( + !json.contains(secret) && !page.contains(secret), + "{}: should publish no secret", + case.name + ); + } +} + +/// Two asset routes, for a list holding values masked by default and a +/// secret. +const ASSET_ROUTES: &str = r#" + [[proxy.asset_routes]] + prefix = "/assets/" + origin_url = "https://assets.example.com" + + [proxy.asset_routes.auth] + type = "s3_sigv4" + region = "us-east-1" + access_key_id = "route-access" + secret_access_key = "route-secret" + + [[proxy.asset_routes]] + prefix = "/media/" + origin_url = "https://media.example.com" +"#; + +#[test] +fn show_reveals_a_value_masked_by_default() { + let settings = load(document(&format!( + "{ASSET_ROUTES}\n[inspect]\nshow = [\"publisher.origin_url\", \ + \"proxy.asset_routes[].origin_url\"]" + ))) + .expect("should load"); + + let view = view(&settings); + + assert_eq!( + at(&view.settings, "publisher.origin_url"), + Some(&json!("https://origin.test-publisher.com")) + ); + assert_eq!( + at(&view.settings, "proxy.asset_routes[1].origin_url"), + Some(&json!("https://media.example.com")) + ); + assert!(!view.masked.iter().any(|path| path.ends_with(".origin_url"))); + assert!(!is_masked(&view, "publisher.origin_url")); +} + +#[test] +fn hide_masks_a_value_shown_by_default() { + let settings = load(document("[inspect]\nhide = [\"publisher.domain\"]")).expect("should load"); + + let view = view(&settings); + + assert_eq!(at(&view.settings, "publisher.domain"), Some(&json!(MASK))); + assert!(is_masked(&view, "publisher.domain")); +} + +/// A pattern that would not be honored as written is refused when the +/// document loads. Each row's fragment carries the whole reason, so a +/// pattern refused for another reason fails its row. +#[test] +fn a_pattern_not_honored_as_written_is_refused_saying_why() { + for (section, reason) in [ + ( + r#"show = ["publisher.proxy_secret"]"#, + "show names `publisher.proxy_secret`, which is a secret", + ), + ( + r#"show = ["proxy.asset_routes[].auth.secret_access_key"]"#, + "show names `proxy.asset_routes[].auth.secret_access_key`, which is a secret", + ), + ( + r#"hide = ["publisher.no_such_setting"]"#, + "hide names `publisher.no_such_setting`, which matches no value", + ), + ( + r#"show = ["proxy.asset_routes[9].origin_url"]"#, + "show names `proxy.asset_routes[9].origin_url`, which matches no value", + ), + ( + r#"show = ["publisher.domain"]"#, + "show names `publisher.domain`, which changes nothing, because nothing \ + it names is masked by default", + ), + ( + r#"hide = ["publisher.origin_url"]"#, + "hide names `publisher.origin_url`, which changes nothing, because \ + everything it names is masked or empty already", + ), + ( + r#"hide = ["publisher.proxy_secret"]"#, + "hide names `publisher.proxy_secret`, which changes nothing, because \ + everything it names is masked or empty already", + ), + ( + r#"show = ["publisher.origin_url", "publisher.origin_url"]"#, + "show names `publisher.origin_url`, which changes nothing, because \ + `publisher.origin_url` in the same list already names everything it does", + ), + ( + r#"show = ["proxy.asset_routes[].origin_url", "proxy.asset_routes[0].origin_url"]"#, + "show names `proxy.asset_routes[0].origin_url`, which changes nothing, because \ + `proxy.asset_routes[].origin_url` in the same list already names everything \ + it does", + ), + ( + "show = [\"publisher.origin_url\"]\nhide = [\"publisher.origin_url\"]", + "show names `publisher.origin_url`, which changes nothing, because hide \ + names it as `publisher.origin_url` and hide wins over show", + ), + ( + "config = false\nhide = [\"publisher.domain\"]", + "config = false publishes no page, so they change nothing", + ), + ] { + let message = format!( + "{:?}", + load(document(&format!("{ASSET_ROUTES}\n[inspect]\n{section}"))) + .expect_err("the document should be refused") + ); + + assert!(message.contains(reason), "{section}: {message}"); + } +} + +/// The check a deployment's settings pass before they are pushed refuses +/// the same patterns, so a bad one is caught before it is live. +#[test] +fn the_deploy_check_refuses_a_pattern_that_matches_nothing() { + let toml = format!( + "{}\n[inspect]\nhide = [\"publisher.no_such_setting\"]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("should parse the document"); + + let message = format!( + "{:?}", + validate_patterns(&settings).expect_err("the pattern should be refused") + ); + + assert!( + message.contains("hide names `publisher.no_such_setting`, which matches no value"), + "{message}" + ); +} + +/// A secret only the configuration or a deployment's own module names is +/// one the load resolves, so the deploy check refuses showing it as the +/// load does. +#[test] +fn the_deploy_check_refuses_showing_a_secret_only_the_load_resolves() { + let builder = IntegrationBuilder::new( + "probe", + "example-crate", + crate::integrations::registry_test_support::probe_registration, + crate::integrations::registry_test_support::validate_nothing, + ) + .with_module_name("testing.probe") + .with_secret_settings(MODULE_SECRETS); + let labeled = crate::test_support::tests::crate_test_settings_str_with_ec_section( + "[ec] +module = \"primary\" + +[ec.primary] +implementation = \"hmac\" + passphrase = \"labeled-passphrase-key-0123456789\"", + ); + let mut labeled = Settings::from_toml(&format!( + "{labeled} +[inspect] +show = [\"ec.primary.passphrase\"] +" + )) + .expect("should parse the document"); + labeled.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + let mut module = settings(); + module.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + module + .insert_module_config( + "testing", + "testing.probe", + &json!({ "lock": true, "key_name": "module-key" }), + ) + .expect("should insert the module's table"); + module.inspect.show = vec![PathPattern::parse("testing.probe.key_name").expect("a pattern")]; + + for (settings, pattern) in [ + (labeled, "ec.primary.passphrase"), + (module, "testing.probe.key_name"), + ] { + let reason = format!("show names `{pattern}`, which is a secret"); + let data = serde_json::to_value(&settings).expect("should serialize the document"); + let loaded = format!( + "{:?}", + load_with(data, &[builder]).expect_err("the load should refuse the pattern") + ); + let deployed = format!( + "{:?}", + crate::config::validate_settings_for_deploy_with(&settings, &[builder]) + .expect_err("the deploy check should refuse the pattern") + ); + + assert!(loaded.contains(&reason), "load: {loaded}"); + assert!(deployed.contains(&reason), "deploy: {deployed}"); + } +} + +#[test] +fn the_page_answers_not_found_when_the_publisher_does_not_publish() { + let settings = load(document("[inspect]\nconfig = false")).expect("should load"); + + for (path, content_type) in [ + (CONFIG_PAGE_PATH, "text/html; charset=utf-8"), + (CONFIG_JSON_PATH, "application/json"), + ] { + let response = handle_config(&settings, &request(path)); + + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + content_type, + "{path}" + ); + assert!(body_text(response).contains(NOT_PUBLISHED), "{path}"); + } +} + +#[test] +fn the_page_and_the_json_carry_the_view() { + let settings = load(document("")).expect("should load"); + + for (path, content_type) in [ + (CONFIG_PAGE_PATH, "text/html; charset=utf-8"), + (CONFIG_JSON_PATH, "application/json"), + ] { + let response = handle_config(&settings, &request(path)); + + assert_eq!(response.status(), StatusCode::OK, "{path}"); + assert_eq!(response.headers()[header::CONTENT_TYPE], content_type); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(response.headers()[header::ACCESS_CONTROL_ALLOW_ORIGIN], "*"); + } + let json: Value = serde_json::from_str(&body_text(handle_config( + &settings, + &request(CONFIG_JSON_PATH), + ))) + .expect("a JSON body"); + let keys: Vec<&String> = json.as_object().expect("an object").keys().collect(); + assert_eq!(keys, ["masked", "settings", "version"]); + assert_eq!(json["version"], env!("CARGO_PKG_VERSION")); + assert_eq!( + json["settings"]["publisher"]["domain"], + "test-publisher.com" + ); + assert!( + json["masked"] + .as_array() + .expect("a list") + .contains(&json!("publisher.proxy_secret")), + "{json}" + ); +} + +fn assert_keys_sorted(value: &Value) { + match value { + Value::Object(map) => { + let keys: Vec<&String> = map.keys().collect(); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "keys should be sorted"); + map.values().for_each(assert_keys_sorted); + } + Value::Array(items) => items.iter().for_each(assert_keys_sorted), + _ => {} + } +} + +/// Two loads of one document build hash maps of their own, each iterating +/// in its own order, and the view has to come out the same. +#[test] +fn the_same_settings_give_the_same_view() { + let headers: String = (0..12) + .map(|index| format!("x-header-{index} = \"value-{index}\"\n")) + .collect(); + let data = document_from(&format!( + "{}\n[response_headers]\n{headers}", + crate_test_settings_str() + )); + + let first = load(data.clone()).expect("should load"); + let second = load(data).expect("should load"); + let first_json = view_payload(&view(&first)).to_string(); + let second_json = view_payload(&view(&second)).to_string(); + + assert_eq!(first_json, second_json); + let parsed: Value = serde_json::from_str(&first_json).expect("JSON"); + assert_keys_sorted(&parsed); + assert_eq!( + parsed["settings"]["response_headers"] + .as_object() + .expect("the headers") + .len(), + 12, + "should show every header" + ); + let masked = view(&first).masked; + let mut sorted_masked = masked.clone(); + sorted_masked.sort_unstable(); + assert_eq!(masked, sorted_masked); +} + +fn module_lock_is_on(table: &serde_json::Map) -> bool { + table.get("lock").and_then(Value::as_bool) == Some(true) +} + +const MODULE_SECRETS: &[ModuleSecretSetting] = &[ModuleSecretSetting { + path: &["key_name"], + in_use: module_lock_is_on, +}]; + +/// A secret a module a deployment added declares in its own table is one +/// the loader wrote, so it is masked like any of core's. +#[test] +fn a_secret_a_module_declares_is_masked() { + let builder = IntegrationBuilder::new( + "probe", + "example-crate", + crate::integrations::registry_test_support::probe_registration, + crate::integrations::registry_test_support::validate_nothing, + ) + .with_module_name("testing.probe") + .with_secret_settings(MODULE_SECRETS); + let mut settings = settings(); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + settings + .insert_module_config( + "testing", + "testing.probe", + &json!({ "lock": true, "key_name": "module-key", "label": "open" }), + ) + .expect("should insert the module's table"); + let data = serde_json::to_value(&settings).expect("should serialize the document"); + + let loaded = load_with(data, &[builder]).expect("should load"); + let view = view(&loaded); + let (json, page) = served(&loaded); + + assert!( + is_masked(&view, "testing.probe.key_name"), + "should mask the module's secret: {:?}", + view.masked + ); + assert_eq!( + at(&view.settings, "testing.probe.label"), + Some(&json!("open")), + "should show the module's other settings" + ); + assert!(!json.contains(CANARY) && !page.contains(CANARY)); +} diff --git a/crates/trusted-server-core/src/inspect/data.rs b/crates/trusted-server-core/src/inspect/data.rs new file mode 100644 index 0000000000..1850c1ae7a --- /dev/null +++ b/crates/trusted-server-core/src/inspect/data.rs @@ -0,0 +1,683 @@ +//! What a deployment holds against the reader's own Edge Cookie, at +//! [`DATA_PAGE_PATH`]. +//! +//! 1. Only a browser opening the address as a page is answered, which the +//! browser marks with `Sec-Fetch-Mode: navigate` and +//! `Sec-Fetch-Dest: document`. +//! 2. The page is sent with `Content-Security-Policy: sandbox`, which gives +//! the document an origin of its own, so a page that opened it cannot +//! read it. +//! 3. No identifier is shown. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderValue, Request, Response, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::ec::kv::KvIdentityGraph; +use crate::ec::kv_types::KvEntry; +use crate::ec::module::{AcceptedModules, EdgeCookieModule}; +use crate::error::TrustedServerError; +use crate::response_privacy::RequiredHeaders; + +use super::{MASK, render_page_with_lede}; + +/// The page showing what is held against the reader's own Edge Cookie. +pub const DATA_PAGE_PATH: &str = "/_ts/data"; + +const NOT_OPENED_AS_A_PAGE: &str = "This address shows a reader what is held against their own \ + browser, so it answers only a browser opening it as a page. \ + Open it from the address bar.\n"; + +/// `sandbox` gives the document an origin of its own, so a page that opened +/// it cannot read it, and nothing in it runs or loads but its own style. +const CONTENT_SECURITY_POLICY: &str = "sandbox; default-src 'none'; style-src 'unsafe-inline'; \ + frame-ancestors 'none'; base-uri 'none'; form-action 'none'"; + +/// Answers [`DATA_PAGE_PATH`] for `req`. +/// +/// `kv` is the identity graph the deployment keeps, or `None` when it keeps +/// none, and `module` is the Edge Cookie module it selects. The record is the +/// one stored against the identifier in the request's own Edge Cookie, named +/// after `public`, the request's public prefix. +/// Nothing is created or written for the reader. +/// +/// # Errors +/// +/// When the request's `Cookie` header cannot be read, or the store cannot be +/// read, or what it holds is not a record. +pub fn handle_data( + kv: Option<&KvIdentityGraph>, + module: Option<&dyn EdgeCookieModule>, + public: &crate::prefix::PublicPrefix, + req: &Request, +) -> Result, Report> { + if !is_opened_as_a_page(req) { + return Ok(protected( + StatusCode::FORBIDDEN, + "text/plain; charset=utf-8", + NOT_OPENED_AS_A_PAGE.to_owned(), + )); + } + let Some(kv) = kv else { + return Ok(page(¬hing_held( + "This deployment keeps no record against an Edge Cookie.", + ))); + }; + let Some(ec_id) = crate::ec::request_cookie_ec(req, public)? else { + return Ok(page(¬hing_held( + "The request carried no Edge Cookie, so there is nothing to look up.", + ))); + }; + let Some(kv_key) = AcceptedModules::active(module).canonical_kv_key(&ec_id) else { + return Ok(page(¬hing_held( + "The request's Edge Cookie is not one this deployment issued.", + ))); + }; + let Some((entry, _generation)) = kv.get(&kv_key)? else { + return Ok(page(¬hing_held( + "This deployment holds no record against the request's Edge Cookie. A record \ + made in the last few seconds may not be readable yet.", + ))); + }; + Ok(page(&held_payload(&entry))) +} + +/// Whether `req` is a browser opening the address as a page of its own. +/// +/// A browser sends `Sec-Fetch-*` only to a trustworthy origin, so every +/// browser on a plain `http://` host other than localhost fails this. +fn is_opened_as_a_page(req: &Request) -> bool { + let is = |name: &str, expected: &str| { + req.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case(expected)) + }; + is("sec-fetch-mode", "navigate") && is("sec-fetch-dest", "document") +} + +fn nothing_held(why: &str) -> Value { + json!({ + "held": false, + "why": why, + "version": env!("CARGO_PKG_VERSION"), + }) +} + +/// What the page shows for `entry`, being the record as it is stored with +/// each partner's identifier masked, and the two times in it written out. +fn held_payload(entry: &KvEntry) -> Value { + let mut record = entry.clone(); + for id in record.ids.values_mut() { + MASK.clone_into(&mut id.uid); + } + json!({ + "held": true, + "identifier": MASK, + "created": readable_time(entry.created), + "consent_updated": readable_time(entry.consent.updated), + "withdrawn": !entry.consent.ok, + "record": record, + "version": env!("CARGO_PKG_VERSION"), + }) +} + +/// `unix_seconds` as RFC 3339 in UTC, or `None` beyond what a date can hold. +fn readable_time(unix_seconds: u64) -> Option { + let unix_seconds = i64::try_from(unix_seconds).ok()?; + chrono::DateTime::from_timestamp(unix_seconds, 0) + .map(|time| time.format("%Y-%m-%dT%H:%M:%SZ").to_string()) +} + +fn page(payload: &Value) -> Response { + let lede = format!( + "What this deployment holds against the Edge Cookie your browser sent. Each \ + identifier shows as {MASK}." + ); + protected( + StatusCode::OK, + "text/html; charset=utf-8", + render_page_with_lede("Your data", &lede, payload), + ) +} + +/// A response only the person looking at it can read, being never stored, +/// never framed, in an origin of its own, and offered to no other origin. +/// Its headers are [`RequiredHeaders`], so an operator's own headers of the +/// same names cannot replace them. +fn protected(status: StatusCode, content_type: &'static str, body: String) -> Response { + let mut response = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header( + header::CACHE_CONTROL, + HeaderValue::from_static("no-store, private"), + ) + .header( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(CONTENT_SECURITY_POLICY), + ) + .header(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY")) + .header( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ) + .header( + header::REFERRER_POLICY, + HeaderValue::from_static("no-referrer"), + ) + .header( + "cross-origin-resource-policy", + HeaderValue::from_static("same-origin"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build the data response"); + let required = response.headers().clone(); + response.extensions_mut().insert(RequiredHeaders(required)); + response +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + + use super::*; + use crate::consent::ConsentContext; + use crate::ec::kv_types::{ + KvConsent, KvDevice, KvGeo, KvNetwork, KvPartnerId, KvPubProperties, + }; + use crate::ec::module::HmacModule; + use crate::redacted::Redacted; + + /// An identifier in the form the built-in HMAC module issues. + const EC_ID: &str = + "hmac~0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef.aB3dE9"; + /// The part of [`EC_ID`] that no page may carry. + const EC_HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + const PARTNER_UID: &str = "partner-uid-canary-1234"; + /// 14 March 2025 at midnight UTC. + const CREATED: u64 = 1_741_910_400; + + fn module() -> HmacModule { + HmacModule::new(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())) + } + + fn request(cookie: Option<&str>, mode: Option<&str>, dest: Option<&str>) -> Request { + let mut builder = Request::builder() + .method("GET") + .uri(format!("https://publisher.example{DATA_PAGE_PATH}")); + for (name, value) in [ + ("cookie", cookie), + ("sec-fetch-mode", mode), + ("sec-fetch-dest", dest), + ] { + if let Some(value) = value { + builder = builder.header(name, value); + } + } + builder + .body(EdgeBody::empty()) + .expect("should build the request") + } + + /// A request as a browser sends it when a person opens the address. + fn opened_as_a_page(cookie: Option<&str>) -> Request { + request(cookie, Some("navigate"), Some("document")) + } + + fn body_text(response: Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("a buffered body") + .to_vec(); + String::from_utf8(bytes).expect("a text body") + } + + /// A graph holding one record against [`EC_ID`], with one partner's + /// identifier in it. + fn graph_with_a_record() -> KvIdentityGraph { + let graph = KvIdentityGraph::in_memory("test-ec-store"); + let module = module(); + let key = AcceptedModules::active(Some(&module)) + .canonical_kv_key(EC_ID) + .expect("the test identifier should be one the module issues"); + let mut entry = KvEntry::new( + &ConsentContext::default(), + None, + CREATED, + "publisher.example", + ); + entry.ids.insert( + "partner.example.com".to_owned(), + KvPartnerId { + uid: PARTNER_UID.to_owned(), + }, + ); + graph.create(&key, &entry).expect("should store the record"); + graph + } + + #[test] + fn only_a_browser_opening_the_page_is_answered() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + + for (mode, dest, who) in [ + (None, None, "a tool that sends neither header"), + (Some("cors"), Some("empty"), "a script's fetch"), + (Some("same-origin"), Some("empty"), "a same-origin fetch"), + (Some("navigate"), Some("iframe"), "a frame"), + (Some("navigate"), Some("object"), "an embedded object"), + ( + Some("navigate"), + None, + "a navigation that names no destination", + ), + ( + Some("no-cors"), + Some("document"), + "a request that is not a navigation", + ), + ] { + let response = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &request(Some(&cookie), mode, dest), + ) + .expect("should answer"); + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "should refuse {who}" + ); + assert_eq!( + body_text(response), + NOT_OPENED_AS_A_PAGE, + "should tell {who} nothing about the record" + ); + } + + let response = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"); + assert_eq!( + response.status(), + StatusCode::OK, + "should answer a browser opening the page" + ); + } + + #[test] + fn the_record_is_shown_with_every_identifier_masked() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + + let response = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + let html = body_text(response); + assert!( + html.contains("\"held\": true"), + "should say a record is held" + ); + assert!( + html.contains("partner.example.com"), + "should name the partner that holds an identifier" + ); + assert!( + html.contains("\"created\": \"2025-03-14T00:00:00Z\""), + "should say when the record was made in a form a person reads" + ); + assert!( + html.contains("publisher.example"), + "should show the site the record was made on" + ); + assert!( + !html.contains(PARTNER_UID), + "should not show the partner's identifier" + ); + assert!( + !html.contains(EC_HASH), + "should not show the Edge Cookie identifier" + ); + assert!( + html.contains("\"uid\": \"XXXX\""), + "should show that the partner's identifier is masked" + ); + assert!( + html.contains("Each identifier shows as XXXX."), + "should tell the reader what the mask means" + ); + } + + #[test] + fn an_identifier_sent_in_another_case_reaches_its_own_record() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec=hmac~{}.aB3dE9", EC_HASH.to_ascii_uppercase()); + + let html = body_text( + handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"), + ); + + assert!( + html.contains("\"held\": true"), + "should read the record under the key it is stored under: {html}" + ); + } + + /// Every field the record has is set here and listed below, so a field + /// added to the record fails this test until it is decided whether the + /// field is an identifier. + #[test] + fn every_field_of_the_record_is_shown_or_masked_by_decision() { + fn leaves(value: &Value, path: &str, out: &mut BTreeSet) { + match value { + Value::Object(map) => { + for (key, child) in map { + let key = if path == "ids" { "" } else { key }; + let path = if path.is_empty() { + key.to_owned() + } else { + format!("{path}.{key}") + }; + leaves(child, &path, out); + } + } + _ => { + out.insert(path.to_owned()); + } + } + } + let entry = KvEntry { + v: 1, + created: CREATED, + consent: KvConsent { + tcf: Some("tcf-string".to_owned()), + gpp: Some("gpp-string".to_owned()), + ok: true, + updated: CREATED + 60, + }, + geo: KvGeo { + country: "GB".to_owned(), + region: Some("ENG".to_owned()), + asn: Some(64_496), + dma: Some(807), + }, + pub_properties: Some(KvPubProperties { + origin_domain: "publisher.example".to_owned(), + seen_domains: BTreeSet::from(["publisher.example".to_owned()]), + }), + device: Some(KvDevice { + is_mobile: 0, + ja4_class: Some("t13d1516h2".to_owned()), + platform_class: Some("windows".to_owned()), + h2_fp_hash: Some("0123456789ab".to_owned()), + known_browser: Some(true), + }), + network: Some(KvNetwork { + cluster_size: Some(3), + }), + ids: BTreeMap::from([( + "partner.example.com".to_owned(), + KvPartnerId { + uid: PARTNER_UID.to_owned(), + }, + )]), + }; + + let payload = held_payload(&entry); + + let mut shown = BTreeSet::new(); + leaves(&payload["record"], "", &mut shown); + assert_eq!( + shown.iter().map(String::as_str).collect::>(), + [ + "consent.gpp", + "consent.ok", + "consent.tcf", + "consent.updated", + "created", + "device.h2_fp_hash", + "device.is_mobile", + "device.ja4_class", + "device.known_browser", + "device.platform_class", + "geo.asn", + "geo.country", + "geo.dma", + "geo.region", + "ids..uid", + "network.cluster_size", + "pub_properties.origin_domain", + "pub_properties.seen_domains", + "v", + ], + "a field the record has gained should be shown or masked by decision" + ); + assert_eq!( + payload["record"]["ids"]["partner.example.com"]["uid"], + json!(MASK), + "the one identifier the record holds should be masked" + ); + assert_eq!(payload["identifier"], json!(MASK)); + assert_eq!(payload["consent_updated"], json!("2025-03-14T00:01:00Z")); + assert_eq!(payload["withdrawn"], json!(false)); + assert!( + !payload.to_string().contains(PARTNER_UID), + "should carry no identifier anywhere" + ); + } + + /// The record and the answer the API reference shows. + #[test] + fn a_known_record_gives_the_documented_answer() { + let made = 1_773_480_600; + let entry = KvEntry { + v: 1, + created: made, + consent: KvConsent { + tcf: Some("CP...".to_owned()), + gpp: None, + ok: true, + updated: made, + }, + geo: KvGeo { + country: "GB".to_owned(), + region: Some("ENG".to_owned()), + asn: None, + dma: None, + }, + pub_properties: Some(KvPubProperties { + origin_domain: "example.com".to_owned(), + seen_domains: BTreeSet::from(["example.com".to_owned()]), + }), + device: None, + network: None, + ids: BTreeMap::from([( + "partner.example.com".to_owned(), + KvPartnerId { + uid: PARTNER_UID.to_owned(), + }, + )]), + }; + + assert_eq!( + held_payload(&entry), + json!({ + "consent_updated": "2026-03-14T09:30:00Z", + "created": "2026-03-14T09:30:00Z", + "held": true, + "identifier": "XXXX", + "record": { + "consent": { "ok": true, "tcf": "CP...", "updated": 1_773_480_600 }, + "created": 1_773_480_600, + "geo": { "country": "GB", "region": "ENG" }, + "ids": { "partner.example.com": { "uid": "XXXX" } }, + "pub_properties": { + "origin_domain": "example.com", + "seen_domains": ["example.com"] + }, + "v": 1 + }, + "version": env!("CARGO_PKG_VERSION"), + "withdrawn": false + }) + ); + } + + #[test] + fn a_withdrawal_is_shown_as_what_is_held() { + let payload = held_payload(&KvEntry::tombstone(CREATED)); + + assert_eq!(payload["held"], json!(true)); + assert_eq!( + payload["withdrawn"], + json!(true), + "should say the record is one of a withdrawal" + ); + } + + #[test] + fn a_reader_with_nothing_held_is_told_why() { + let graph = graph_with_a_record(); + let module = module(); + let another = format!("ts-ec=hmac~{}.zZ9yY8", "f".repeat(64)); + + for (kv, cookie, why) in [ + (None, Some("ts-ec=anything"), "keeps no record"), + (Some(&graph), None, "carried no Edge Cookie"), + ( + Some(&graph), + Some("ts-ec=t0op~device-1.issued-monday"), + "is not one this deployment issued", + ), + ( + Some(&graph), + Some(another.as_str()), + "holds no record against the request", + ), + ] { + let response = handle_data( + kv, + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(cookie), + ) + .expect("should answer"); + + assert_eq!(response.status(), StatusCode::OK, "{why}"); + let html = body_text(response); + assert!(html.contains("\"held\": false"), "{why}: {html}"); + assert!(html.contains(why), "should say `{why}`: {html}"); + assert!( + !html.contains("partner.example.com"), + "should show nothing of another reader's record" + ); + } + } + + #[test] + fn a_store_that_cannot_be_read_is_an_error_and_not_an_empty_answer() { + let graph = KvIdentityGraph::failing("test-ec-store"); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + + let result = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ); + + assert!( + result.is_err(), + "should not tell the reader nothing is held when the store was not read" + ); + } + + #[test] + fn every_answer_is_for_the_reader_alone() { + let graph = graph_with_a_record(); + let module = module(); + let cookie = format!("ts-ec={EC_ID}"); + let answered = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &opened_as_a_page(Some(&cookie)), + ) + .expect("should answer"); + let refused = handle_data( + Some(&graph), + Some(&module), + &crate::prefix::PublicPrefix::default(), + &request(Some(&cookie), None, None), + ) + .expect("should answer"); + + for (name, response) in [("the page", answered), ("the refusal", refused)] { + let headers = response.headers(); + assert_eq!( + headers[header::CACHE_CONTROL], + "no-store, private", + "{name} should never be stored" + ); + assert!( + headers[header::CONTENT_SECURITY_POLICY] + .to_str() + .expect("a text policy") + .starts_with("sandbox;"), + "{name} should be in an origin of its own" + ); + assert_eq!(headers[header::X_FRAME_OPTIONS], "DENY", "{name}"); + assert_eq!(headers[header::X_CONTENT_TYPE_OPTIONS], "nosniff", "{name}"); + assert_eq!(headers[header::REFERRER_POLICY], "no-referrer", "{name}"); + assert_eq!( + headers["cross-origin-resource-policy"], "same-origin", + "{name}" + ); + assert!( + headers.get(header::ACCESS_CONTROL_ALLOW_ORIGIN).is_none(), + "{name} should be offered to no other origin" + ); + } + } + + #[test] + fn the_page_does_not_offer_a_data_form() { + let html = body_text(page(¬hing_held("nothing"))); + + assert!( + !html.contains(".json"), + "should not point at a form the address does not have" + ); + assert!(html.contains("

Your data

")); + } +} diff --git a/crates/trusted-server-core/src/inspect/permissions.rs b/crates/trusted-server-core/src/inspect/permissions.rs new file mode 100644 index 0000000000..9aaf4925ee --- /dev/null +++ b/crates/trusted-server-core/src/inspect/permissions.rs @@ -0,0 +1,331 @@ +//! The permissions resolved for the request being served, at +//! [`PERMISSIONS_PAGE_PATH`] and [`PERMISSIONS_JSON_PATH`]. +//! +//! The answer is the evaluation the page receives as +//! `window.tsjs.permissions`, offered at an address so client code can fetch +//! it, another front end can render it, and a person can read it. It is +//! resolved for the request that asks, exactly as a page's is, and nothing is +//! written for the reader, because the Edge Cookie lifecycle does not run +//! here. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{HeaderValue, Request, Response, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::ec::EcContext; +use crate::error::TrustedServerError; +use crate::permissions::PermissionState; +use crate::platform::RuntimeServices; +use crate::settings::Settings; + +use super::render_page; + +/// The permissions page. +pub const PERMISSIONS_PAGE_PATH: &str = "/_ts/permissions"; + +/// The same resolution as data. +pub const PERMISSIONS_JSON_PATH: &str = "/_ts/permissions.json"; + +/// Both addresses, the page first. +pub const PERMISSIONS_PATHS: [&str; 2] = [PERMISSIONS_PAGE_PATH, PERMISSIONS_JSON_PATH]; + +/// The permission resolution for a request, as the endpoint shows it, being +/// the page shape of [`PermissionState::page_value`] with these fields added. +/// +/// - `storageWithdrawn`: whether the request explicitly withdrew device storage. +/// - `modules.contributed`: the signal modules that produced a signal used here. +/// - `modules.configured`: the signal modules the publisher named, or `null`. +/// - `version`: the version of the core crate that answered. +#[must_use] +pub fn permissions_payload(state: &PermissionState, settings: &Settings) -> Value { + let mut contributed: Vec<&str> = state.signals().iter().map(|signal| signal.module).collect(); + contributed.sort_unstable(); + contributed.dedup(); + let configured = settings + .permission_signal + .modules + .as_ref() + .map(|names| json!(names)); + + let mut payload = state.page_value(); + if let Value::Object(fields) = &mut payload { + fields.insert("version".to_owned(), json!(env!("CARGO_PKG_VERSION"))); + fields.insert( + "storageWithdrawn".to_owned(), + json!(state.storage_withdrawn()), + ); + fields.insert( + "modules".to_owned(), + json!({ + "contributed": contributed, + "configured": configured, + }), + ); + } + payload +} + +/// The permissions page, or its JSON when `path` is +/// [`PERMISSIONS_JSON_PATH`], for a request whose permissions have been +/// resolved. +/// +/// Never stored, because each answer is the asking request's own, and +/// readable from any origin, so a page on another site can render it. A +/// request from another origin carries no cookies, so what that page reads is +/// the resolution of a request with no stored signal. +/// +/// # Panics +/// +/// Never in practice, because the status and every header are fixed, valid +/// values. +#[must_use] +pub fn permissions_response( + state: &PermissionState, + settings: &Settings, + path: &str, +) -> Response { + let payload = permissions_payload(state, settings); + let (content_type, body) = if path == PERMISSIONS_JSON_PATH { + ("application/json", payload.to_string()) + } else { + ( + "text/html; charset=utf-8", + render_page("Permissions", &payload), + ) + }; + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, HeaderValue::from_static(content_type)) + .header(header::CACHE_CONTROL, HeaderValue::from_static("no-store")) + .header( + header::ACCESS_CONTROL_ALLOW_ORIGIN, + HeaderValue::from_static("*"), + ) + .body(EdgeBody::from(body.into_bytes())) + .expect("should build permissions response") +} + +/// Answers [`PERMISSIONS_PAGE_PATH`] or [`PERMISSIONS_JSON_PATH`] for `req`. +/// +/// The permissions are resolved for this request as a page's would be, from +/// its signals and its location, and then shown. Nothing is created or +/// written for the reader. +/// +/// # Errors +/// +/// When the request's state cannot be read, which is when the selected Edge +/// Cookie module cannot be built or the `Cookie` header is not valid UTF-8, +/// the same as for a page. +pub async fn handle_permissions( + settings: &Settings, + services: &RuntimeServices, + req: &Request, +) -> Result, Report> { + let state = EcContext::read_from_request_resolving_geo(settings, req, services).await?; + Ok(permissions_response( + state.permissions(), + settings, + req.uri().path(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::permissions::{Permission, PermissionSet, ValidSignal}; + use crate::platform::test_support::noop_services; + use crate::tdl::Tdl; + use crate::test_support::tests::create_test_settings; + + fn state_with_signal() -> PermissionState { + PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)) + .with_signals(vec![ValidSignal::new("tcf", "tcf", "CPabc123")].into()) + .with_tdls( + vec![ + Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept the test locator"), + ] + .into(), + ) + } + + fn body_of(response: Response) -> String { + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("the body should be text") + } + + /// A reader comes to see which signal produced the answer, so the signal + /// and the module that read it must both be there. + #[test] + fn the_payload_names_the_signal_and_the_module_that_read_it() { + let payload = permissions_payload(&state_with_signal(), &create_test_settings()); + + assert_eq!(payload["signals"][0]["module"], "tcf"); + assert_eq!(payload["signals"][0]["scheme"], "tcf"); + assert_eq!(payload["signals"][0]["value"], "CPabc123"); + assert_eq!(payload["modules"]["contributed"], json!(["tcf"])); + assert_eq!( + payload["tdls"], + json!(["https://terms.example.com/marketing/2.txt"]) + ); + assert_eq!(payload["set"], json!(["necessary.operations.storage"])); + assert_eq!(payload["storageWithdrawn"], json!(false)); + assert_eq!(payload["version"], json!(env!("CARGO_PKG_VERSION"))); + } + + /// The page shape is spelled once, on the state, so the endpoint cannot + /// drift from what a page is given. + #[test] + fn the_payload_holds_everything_the_page_is_given() { + let state = state_with_signal(); + let payload = permissions_payload(&state, &create_test_settings()); + + let page = state.page_value(); + for (key, value) in page.as_object().expect("the page shape is an object") { + assert_eq!( + &payload[key], value, + "`{key}` should be what the page is given" + ); + } + } + + /// A module that ran and found nothing and a module that was never + /// configured are different facts, and somebody asking why a permission + /// is not set needs to tell them apart. + #[test] + fn configured_and_contributed_modules_are_reported_separately() { + let mut settings = create_test_settings(); + settings.permission_signal.modules = Some(vec!["tcf".to_owned(), "gpc".to_owned()]); + + let payload = permissions_payload(&state_with_signal(), &settings); + + assert_eq!(payload["modules"]["contributed"], json!(["tcf"])); + assert_eq!(payload["modules"]["configured"], json!(["tcf", "gpc"])); + } + + /// With no modules named the selection is the adapter's, which core + /// cannot see, so it says nothing and does not invent a list. + #[test] + fn configured_is_null_when_the_publisher_named_no_modules() { + let mut settings = create_test_settings(); + settings.permission_signal.modules = None; + + let payload = permissions_payload(&state_with_signal(), &settings); + + assert!( + payload["modules"]["configured"].is_null(), + "should not guess which modules the adapter offers" + ); + } + + /// "No signal was found" is itself the answer somebody came for. + #[test] + fn a_resolution_with_no_signals_still_answers() { + let payload = permissions_payload(&PermissionState::default(), &create_test_settings()); + + assert_eq!(payload["signals"], json!([])); + assert_eq!(payload["set"], json!([])); + assert_eq!(payload["awaiting"], json!([])); + assert_eq!(payload["modules"]["contributed"], json!([])); + } + + #[test] + fn a_withdrawal_of_storage_is_shown_as_one() { + let state = PermissionState::default().with_storage_withdrawn(true); + + let payload = permissions_payload(&state, &create_test_settings()); + + assert_eq!(payload["storageWithdrawn"], json!(true)); + } + + #[test] + fn the_json_address_answers_data_and_the_page_address_a_page() { + let settings = create_test_settings(); + let state = state_with_signal(); + + let data = permissions_response(&state, &settings, PERMISSIONS_JSON_PATH); + assert_eq!(data.status(), StatusCode::OK); + assert_eq!(data.headers()[header::CONTENT_TYPE], "application/json"); + assert_eq!(data.headers()[header::CACHE_CONTROL], "no-store"); + assert_eq!(data.headers()[header::ACCESS_CONTROL_ALLOW_ORIGIN], "*"); + let parsed: Value = + serde_json::from_str(&body_of(data)).expect("the data form should be JSON"); + assert_eq!(parsed, permissions_payload(&state, &settings)); + + let page = permissions_response(&state, &settings, PERMISSIONS_PAGE_PATH); + assert_eq!( + page.headers()[header::CONTENT_TYPE], + "text/html; charset=utf-8" + ); + assert_eq!(page.headers()[header::CACHE_CONTROL], "no-store"); + let html = body_of(page); + assert!(html.contains("

Permissions

"), "should be the page"); + assert!(html.contains("CPabc123"), "should show the signal"); + } + + /// The page carries a signal value straight from the request, so + /// anything that could close the surrounding tag is escaped. + #[test] + fn the_page_escapes_a_hostile_signal_value() { + let state = PermissionState::new(PermissionSet::none()) + .with_signals(vec![ValidSignal::new("tcf", "tcf", " + + + + + +An article on the origin + +A link to the same address + + diff --git a/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.recorded.html b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.recorded.html new file mode 100644 index 0000000000..4168885df0 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.recorded.html @@ -0,0 +1,17 @@ + + + + +JS asset proxy + + + + + + + +An article on the origin + +A link to the same address + + diff --git a/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.settings.toml b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.settings.toml new file mode 100644 index 0000000000..2cedf868f5 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/fixtures/js-asset-proxy.settings.toml @@ -0,0 +1,13 @@ +[proxy] +modules = ["js_asset_proxy"] + +[proxy.js_asset_proxy] + +[[proxy.js_asset_proxy.assets]] +path = "/assets/example-vendor-loader.js" +origin_url = "https://cdn.example.com/vendor-loader.js" +proxy = "enabled" + +[[fetch]] +media_type = "text/html" +middleware = ["js_asset_proxy"] diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 6b301bf935..183288a568 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -6,6 +6,7 @@ //! proxying, or block matching script tags from publisher HTML. use std::collections::HashSet; +use std::rc::Rc; use std::sync::Arc; use async_trait::async_trait; @@ -22,14 +23,31 @@ use crate::constants::{ }; use crate::error::TrustedServerError; use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, + AttributeRewriteAction, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, +}; +use crate::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, MiddlewareContext, + MiddlewarePhase, }; use crate::platform::RuntimeServices; use crate::proxy::{ProxyRequestConfig, proxy_request}; use crate::settings::{IntegrationConfig, Settings}; pub(crate) const JS_ASSET_PROXY_INTEGRATION_ID: &str = "js_asset_proxy"; + +/// The name this module is selected by, in `[proxy]`. +pub const MODULE: &str = "js_asset_proxy"; + +/// The builder the registry runs when a section selects [`MODULE`]. +pub(crate) const BUILDER: crate::integrations::IntegrationBuilder = + crate::integrations::IntegrationBuilder::new( + JS_ASSET_PROXY_INTEGRATION_ID, + crate::integrations::CORE_SOURCE, + register, + validate, + ) + .with_module_name(MODULE) + .selected_in("proxy"); const JS_ASSET_CONTENT_TYPE: &str = "application/javascript; charset=utf-8"; const X_CONTENT_TYPE_OPTIONS_NOSNIFF: &str = "nosniff"; const ERROR_ORIGIN_UNREACHABLE: &str = "js-asset-origin-unreachable"; @@ -37,10 +55,8 @@ const ERROR_ORIGIN_STATUS: &str = "js-asset-origin-status"; /// Configuration for the JavaScript asset proxy integration. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct JsAssetProxyConfig { - /// Enables or disables the integration. - #[serde(default)] - pub enabled: bool, /// Optional downstream cache TTL override for every asset. #[serde(default)] pub cache_ttl_seconds: Option, @@ -88,18 +104,16 @@ impl JsAssetProxyConfig { } } -impl IntegrationConfig for JsAssetProxyConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} +impl IntegrationConfig for JsAssetProxyConfig {} impl Validate for JsAssetProxyConfig { fn validate(&self) -> Result<(), ValidationErrors> { let mut errors = ValidationErrors::new(); errors.merge_self("assets", self.assets.validate()); - if self.enabled && self.assets.is_empty() { + // A selected integration with nothing to serve is a configuration + // mistake, so the block has to list at least one asset. + if self.assets.is_empty() { errors.add("assets", ValidationError::new("empty_assets")); } @@ -268,14 +282,32 @@ impl JsAssetProxyIntegration { fn asset_for_script_src( &self, script_src: &str, - ctx: &IntegrationAttributeContext<'_>, + request_scheme: &str, ) -> Option<&JsAssetProxyAsset> { self.asset_for_origin_url(script_src).or_else(|| { - let normalized_src = normalize_script_src(script_src, ctx.request_scheme)?; + let normalized_src = normalize_script_src(script_src, request_scheme)?; self.asset_for_origin_url(&normalized_src) }) } + /// What happens to a script whose address is `script_src`, on a page + /// served over `request_scheme`. + fn rewrite_script_address( + &self, + script_src: &str, + request_scheme: &str, + ) -> AttributeRewriteAction { + let Some(asset) = self.asset_for_script_src(script_src, request_scheme) else { + return AttributeRewriteAction::keep(); + }; + + match asset.proxy { + JsAssetProxyMode::Enabled => AttributeRewriteAction::replace(asset.path.clone()), + JsAssetProxyMode::Disabled => AttributeRewriteAction::keep(), + JsAssetProxyMode::Blocked => AttributeRewriteAction::remove_element(), + } + } + fn build_proxy_config<'a>( origin_url: &'a str, req: &Request, @@ -446,9 +478,7 @@ impl JsAssetProxyIntegration { fn build( settings: &Settings, ) -> Result>, Report> { - let Some(mut config) = - settings.integration_config::(JS_ASSET_PROXY_INTEGRATION_ID)? - else { + let Some(mut config) = settings.module_config::(MODULE)? else { return Ok(None); }; config.normalize_origin_urls(); @@ -456,11 +486,24 @@ fn build( Ok(Some(JsAssetProxyIntegration::new(config))) } +/// Validates the JavaScript asset proxy configuration for deployment and +/// reports whether a section selects the integration's module. +/// +/// # Errors +/// +/// Returns an error when the configuration cannot be parsed or fails +/// validation. +pub(crate) fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) +} + /// Register the JavaScript asset proxy integration. /// /// # Errors /// -/// Returns an error when the integration is enabled with invalid configuration. +/// Returns an error when the integration runs with invalid configuration. pub fn register( settings: &Settings, ) -> Result, Report> { @@ -471,31 +514,17 @@ pub fn register( Ok(Some( IntegrationRegistration::builder(JS_ASSET_PROXY_INTEGRATION_ID) .with_proxy(integration.clone()) - .with_attribute_rewriter(integration) + .with_middleware(Arc::new(ScriptAddress(integration))) .build(), )) } -#[async_trait(?Send)] -impl IntegrationProxy for JsAssetProxyIntegration { - fn integration_name(&self) -> &'static str { - JS_ASSET_PROXY_INTEGRATION_ID - } - - fn routes(&self) -> Vec { - self.config - .assets - .iter() - .filter(|asset| asset.proxy == JsAssetProxyMode::Enabled) - .map(|asset| IntegrationEndpoint::new(Method::GET, asset.path.clone())) - .collect() - } - - async fn handle( +impl JsAssetProxyIntegration { + async fn route( &self, + req: Request, settings: &Settings, services: &RuntimeServices, - req: Request, ) -> Result, Report> { let request_path = req.uri().path().to_string(); let asset = self.enabled_asset_for_path(&request_path).ok_or_else(|| { @@ -533,33 +562,60 @@ impl IntegrationProxy for JsAssetProxyIntegration { } } -impl IntegrationAttributeRewriter for JsAssetProxyIntegration { - fn integration_id(&self) -> &'static str { +#[async_trait(?Send)] +impl IntegrationProxy for JsAssetProxyIntegration { + fn integration_name(&self) -> &'static str { JS_ASSET_PROXY_INTEGRATION_ID } - fn handles_attribute(&self, attribute: &str) -> bool { - attribute == "src" + fn routes(&self) -> Vec { + self.config + .assets + .iter() + .filter(|asset| asset.proxy == JsAssetProxyMode::Enabled) + .map(|asset| IntegrationEndpoint::new(Method::GET, asset.path.clone())) + .collect() } - fn rewrite( + async fn handle( &self, - attr_name: &str, - attr_value: &str, - ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - if attr_name != "src" || !ctx.element_name.eq_ignore_ascii_case("script") { - return AttributeRewriteAction::keep(); - } + call: crate::module_context::ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::route)?.await + } +} - let Some(asset) = self.asset_for_script_src(attr_value, ctx) else { - return AttributeRewriteAction::keep(); - }; +/// Points a configured script at its first-party path, or removes one that +/// is blocked, on the pages a `[[fetch]]` entry names [`MODULE`] for. +/// +/// It is asked about a script's address as the middleware named before it +/// left it, so it is named first where it has to see the address the page +/// wrote. +struct ScriptAddress(Arc); + +impl Middleware for ScriptAddress { + fn middleware_id(&self) -> &'static str { + MODULE + } - match asset.proxy { - JsAssetProxyMode::Enabled => AttributeRewriteAction::replace(asset.path.clone()), - JsAssetProxyMode::Disabled => AttributeRewriteAction::keep(), - JsAssetProxyMode::Blocked => AttributeRewriteAction::remove_element(), + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let integration = Arc::clone(&self.0); + let request_scheme = context.request_scheme.to_owned(); + let decide: Rc = Rc::new(move |matched| { + integration.rewrite_script_address(matched.value, &request_scheme) + }); + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + "script[src]", + "src", + decide, + ))], + ..MiddlewareAction::pass() } } } @@ -571,10 +627,13 @@ mod tests { use std::sync::Arc; use crate::constants::{HEADER_REFERER, HEADER_X_FORWARDED_FOR, HEADER_X_TS_EC}; - use crate::html_processor::{BodyCloseInjection, HtmlProcessorConfig, create_html_processor}; - use crate::integrations::{ - AttributeRewriteAction, IntegrationAttributeRewriter, IntegrationRegistry, + use crate::html_processor::test_support::{place_on_every_page, processed_page}; + use crate::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor_with_middleware, }; + use crate::integrations::registry_test_support::tag_fixture as tag; + use crate::integrations::{AttributeRewriteAction, IntegrationRegistry}; + use crate::middleware::MiddlewareChain; use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; @@ -600,47 +659,39 @@ mod tests { fn config_with_assets(assets: Vec) -> JsAssetProxyConfig { JsAssetProxyConfig { - enabled: true, cache_ttl_seconds: None, assets, } } - fn rewrite_context() -> IntegrationAttributeContext<'static> { - IntegrationAttributeContext { - attribute_name: "src", - element_name: "script", - request_host: "publisher.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - } - } - + /// The page a reader of `html` receives when `integration`'s middleware + /// alone runs on it. fn process_html_with_integration( html: &str, integration: Arc, ) -> String { - let rewriter: Arc = integration; - process_html_with_registry( - html, - IntegrationRegistry::from_rewriters(vec![rewriter], Vec::new()), - ) - } - - fn process_html_with_registry(html: &str, integrations: IntegrationRegistry) -> String { - let processor = create_html_processor(HtmlProcessorConfig { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, + vec![Arc::new(ScriptAddress(integration))], + ); + let config = HtmlProcessorConfig { csp_nonce_observed: None, + public_prefix: None, + prepared: std::sync::Arc::new(crate::middleware::Prepared::default()), body_close: BodyCloseInjection::None, origin_host: "origin.example.com".to_string(), request_host: "publisher.example.com".to_string(), request_scheme: "https".to_string(), - integrations, + integrations: IntegrationRegistry::default(), ad_slots_script: None, ad_bids_state: Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, - }); + request_state: crate::integrations::IntegrationRequestState::default(), + permissions_script: None, + }; + let (processor, _) = create_html_processor_with_middleware(config, &chain, None) + .expect("should plan the asset proxy's middleware"); let pipeline_config = PipelineConfig { input_compression: Compression::None, output_compression: Compression::None, @@ -655,35 +706,50 @@ mod tests { String::from_utf8(output).expect("should produce UTF-8 HTML") } - #[test] - fn disabled_config_does_not_register_routes() { + /// The page a reader of `html` receives from a deployment running + /// `settings`. + fn process_html_with_settings(html: &str, settings: &Settings) -> String { + let registry = IntegrationRegistry::new(settings).expect("should build registry"); + processed_page(settings, ®istry, html, 8192) + } + + /// Settings that run the tag stand-in and proxy its script in `mode`, + /// with the two middleware named in the order given. + fn settings_proxying_the_tag(mode: &str, middleware: &[&str]) -> Settings { let mut settings = create_test_settings(); + settings.select_module("testing", tag::MODULE); settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, + .insert_module_config( + "proxy", + "js_asset_proxy", &json!({ - "enabled": false, "assets": [{ - "path": "/assets/vendor.js", - "origin_url": "https://cdn.example.com/vendor.js" + "path": "/assets/sdk.js", + "origin_url": tag::SCRIPT_URL, + "proxy": mode }] }), ) - .expect("should insert integration config"); + .expect("should insert JS asset proxy config"); + place_on_every_page(&mut settings, MiddlewarePhase::Fetch, middleware); + settings + } + + #[test] + fn an_integration_that_is_not_named_registers_no_routes() { + let settings = create_test_settings(); let registry = IntegrationRegistry::new(&settings).expect("should build registry"); assert!( !registry.has_route(&Method::GET, "/assets/vendor.js"), - "disabled integration should not register asset route" + "an integration that is not named should register no asset route" ); } #[test] fn enabled_config_requires_at_least_one_asset() { let config = JsAssetProxyConfig { - enabled: true, cache_ttl_seconds: None, assets: Vec::new(), }; @@ -723,17 +789,16 @@ mod tests { assert_eq!(routes[0].method, Method::GET); assert_eq!(routes[0].path, "/assets/enabled.js"); - let ctx = rewrite_context(); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/enabled.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/enabled.js", "https"), AttributeRewriteAction::Replace(ref value) if value == "/assets/enabled.js" )); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/disabled.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/disabled.js", "https"), AttributeRewriteAction::Keep )); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/blocked.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/blocked.js", "https"), AttributeRewriteAction::RemoveElement )); } @@ -745,46 +810,13 @@ mod tests { "https://cdn.example.com/vendor.js", JsAssetProxyMode::Enabled, )])); - let ctx = rewrite_context(); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/vendor.js?v=1", &ctx), + integration.rewrite_script_address("https://cdn.example.com/vendor.js?v=1", "https"), AttributeRewriteAction::Keep )); assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/other.js", &ctx), - AttributeRewriteAction::Keep - )); - } - - #[test] - fn non_script_src_matches_are_not_rewritten_or_blocked() { - let integration = JsAssetProxyIntegration::new(config_with_assets(vec![ - asset( - "/assets/enabled.js", - "https://cdn.example.com/enabled.js", - JsAssetProxyMode::Enabled, - ), - asset( - "/assets/blocked.js", - "https://cdn.example.com/blocked.js", - JsAssetProxyMode::Blocked, - ), - ])); - let ctx = IntegrationAttributeContext { - attribute_name: "src", - element_name: "img", - request_host: "publisher.example.com", - request_scheme: "https", - origin_host: "origin.example.com", - }; - - assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/enabled.js", &ctx), - AttributeRewriteAction::Keep - )); - assert!(matches!( - integration.rewrite("src", "https://cdn.example.com/blocked.js", &ctx), + integration.rewrite_script_address("https://cdn.example.com/other.js", "https"), AttributeRewriteAction::Keep )); } @@ -818,6 +850,25 @@ mod tests { assert!(processed.contains(r#""#)); } + #[test] + fn a_script_tag_written_in_upper_case_is_a_script() { + let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( + "/assets/enabled.js", + "https://cdn.example.com/enabled.js", + JsAssetProxyMode::Enabled, + )])); + + let processed = process_html_with_integration( + r#""#, + integration, + ); + + assert!( + processed.contains(r#"SRC="/assets/enabled.js""#), + "should rewrite a script whatever case its tag is written in: {processed}" + ); + } + #[test] fn script_src_matching_normalizes_common_browser_url_forms() { let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( @@ -825,7 +876,6 @@ mod tests { "https://cdn.example.com/vendor.js", JsAssetProxyMode::Enabled, )])); - let ctx = rewrite_context(); for script_src in [ "//cdn.example.com/vendor.js", @@ -834,7 +884,7 @@ mod tests { ] { assert!( matches!( - integration.rewrite("src", script_src, &ctx), + integration.rewrite_script_address(script_src, "https"), AttributeRewriteAction::Replace(ref value) if value == "/assets/vendor.js" ), "script src {script_src} should normalize to the configured origin URL" @@ -843,106 +893,84 @@ mod tests { } #[test] - fn js_asset_proxy_rewriter_takes_precedence_over_native_rewriters() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt", &json!({ "enabled": true })) - .expect("should insert GPT config"); - settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, - &json!({ - "enabled": true, - "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - "proxy": "enabled" - }] - }), - ) - .expect("should insert JS asset proxy config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + fn the_asset_proxy_named_first_rewrites_a_script_ahead_of_a_module() { + let settings = settings_proxying_the_tag("enabled", &[MODULE, tag::MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_settings(&html, &settings); assert!( - processed.contains(r#""#), - "JS asset proxy should rewrite before GPT native rewriter: {processed}" + processed.contains(r#""#), + "JS asset proxy should rewrite before the module named after it: {processed}" ); assert!( - !processed.contains("/integrations/gpt/script"), - "GPT native rewrite should not override JS asset proxy" + !processed.contains(tag::FIRST_PARTY_SCRIPT), + "the module's own rewrite should not override JS asset proxy" ); } #[test] - fn js_asset_proxy_blocking_takes_precedence_over_native_rewriters() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt", &json!({ "enabled": true })) - .expect("should insert GPT config"); - settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, - &json!({ - "enabled": true, - "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - "proxy": "blocked" - }] - }), - ) - .expect("should insert JS asset proxy config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + fn a_module_named_ahead_of_the_asset_proxy_rewrites_the_script_first() { + let settings = settings_proxying_the_tag("enabled", &[tag::MODULE, MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); + + let processed = process_html_with_settings(&html, &settings); + + assert!( + processed.contains(&format!( + r#""#, + tag::FIRST_PARTY_SCRIPT + )), + "the module named first should rewrite the script: {processed}" + ); + assert!( + !processed.contains("/assets/sdk.js"), + "JS asset proxy should not know an address the module has already moved" + ); + } + + #[test] + fn the_asset_proxy_named_first_blocks_a_script_ahead_of_a_module() { + let settings = settings_proxying_the_tag("blocked", &[MODULE, tag::MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_settings(&html, &settings); assert!( - !processed.contains("googletag.cmd"), - "blocked JS asset should remove the script element before GPT can rewrite it" + !processed.contains("vendor.cmd"), + "blocked JS asset should remove the script element before the module can rewrite it" ); assert!( - !processed.contains("/integrations/gpt/script"), - "GPT native rewrite should not keep a blocked script" + !processed.contains(tag::FIRST_PARTY_SCRIPT), + "the module's own rewrite should not keep a blocked script" ); } #[test] fn disabled_js_asset_proxy_candidate_allows_native_rewriters() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt", &json!({ "enabled": true })) - .expect("should insert GPT config"); - settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, - &json!({ - "enabled": true, - "assets": [{ - "path": "/assets/gpt.js", - "origin_url": "https://securepubads.g.doubleclick.net/tag/js/gpt.js", - "proxy": "disabled" - }] - }), - ) - .expect("should insert JS asset proxy config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let html = r#""#; + let settings = settings_proxying_the_tag("disabled", &[MODULE, tag::MODULE]); + let html = format!( + r#""#, + tag::SCRIPT_URL + ); - let processed = process_html_with_registry(html, registry); + let processed = process_html_with_settings(&html, &settings); assert!( - processed.contains(r#""#), - "disabled JS asset proxy entries should not suppress native integration rewrites" + processed.contains(&format!( + r#""#, + tag::FIRST_PARTY_SCRIPT + )), + "disabled JS asset proxy entries should not suppress a module's own rewrite" ); } @@ -1032,16 +1060,6 @@ mod tests { #[test] fn rejects_unknown_proxy_mode() { let toml = r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" @@ -1051,14 +1069,16 @@ mod tests { [ec] passphrase = "test-secret-key-32-bytes-minimum" + [geo] + assume_single_jurisdiction = true + [request_signing] - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" + enabled = false - [integrations.js_asset_proxy] - enabled = true + [proxy] + modules = ["js_asset_proxy"] - [[integrations.js_asset_proxy.assets]] + [[proxy.js_asset_proxy.assets]] path = "/assets/vendor.js" origin_url = "https://cdn.example.com/vendor.js" proxy = "passthrough" @@ -1067,7 +1087,7 @@ mod tests { assert!( settings - .integration_config::(JS_ASSET_PROXY_INTEGRATION_ID) + .module_config::(MODULE) .is_err(), "unknown proxy mode should fail deserialization" ); @@ -1077,11 +1097,10 @@ mod tests { fn exact_configured_routes_are_registered() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, + .insert_module_config( + "proxy", + "js_asset_proxy", &json!({ - "enabled": true, "assets": [ { "path": "/assets/vendor.js", @@ -1278,11 +1297,10 @@ mod tests { fn configured_origin_urls_are_canonicalized_for_matching_and_duplicates() { let mut settings = create_test_settings(); settings - .integrations - .insert_config( - JS_ASSET_PROXY_INTEGRATION_ID, + .insert_module_config( + "proxy", + "js_asset_proxy", &json!({ - "enabled": true, "assets": [{ "path": "/assets/vendor.js", "origin_url": "HTTPS://CDN.EXAMPLE.COM:443/vendor.js" @@ -1290,10 +1308,10 @@ mod tests { }), ) .expect("should insert integration config"); - let registry = IntegrationRegistry::new(&settings).expect("should build registry"); - let processed = process_html_with_registry( + place_on_every_page(&mut settings, MiddlewarePhase::Fetch, &[MODULE]); + let processed = process_html_with_settings( r#""#, - registry, + &settings, ); assert!(processed.contains(r#""#)); @@ -1385,7 +1403,7 @@ mod tests { ); let response = integration - .handle(&settings, &services, request) + .route(request, &settings, &services) .await .expect("should proxy conditional asset response"); @@ -1435,13 +1453,13 @@ mod tests { Arc::clone(&buffered_stub) as Arc ); let success = integration - .handle( - &settings, - &buffered_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &buffered_services, ) .await .expect("should proxy buffered asset response"); @@ -1455,13 +1473,13 @@ mod tests { Arc::clone(&streaming_stub) as Arc ); let success = integration - .handle( - &settings, - &streaming_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &streaming_services, ) .await .expect("should proxy streaming asset response"); @@ -1483,13 +1501,13 @@ mod tests { Arc::clone(&unavailable_stub) as Arc, ); let unavailable = integration - .handle( - &settings, - &unavailable_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &unavailable_services, ) .await .expect("should map unavailable origin response"); @@ -1508,13 +1526,13 @@ mod tests { Arc::clone(&status_stub) as Arc ); let status = integration - .handle( - &settings, - &status_services, + .route( build_http_request( Method::GET, "https://publisher.example.com/assets/vendor.js", ), + &settings, + &status_services, ) .await .expect("should map non-success origin response"); @@ -1664,4 +1682,21 @@ mod tests { "should identify the unknown asset field" ); } + + /// The page a reader receives with the script proxy running, kept as a + /// file so that changing how the page change is made can be shown to + /// leave the page as it was. + #[test] + fn the_page_a_reader_receives_is_the_recorded_one() { + crate::html_processor::test_support::assert_page_is_recorded( + include_str!("fixtures/js-asset-proxy.settings.toml"), + &[], + include_str!("fixtures/js-asset-proxy.input.html"), + include_str!("fixtures/js-asset-proxy.recorded.html"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/integrations/fixtures/js-asset-proxy.recorded.html" + ), + ); + } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 820e2c24b8..cf07cee5d8 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -1,43 +1,32 @@ //! Integration module registry and sample implementations. +use std::sync::Arc; use std::time::Duration; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use futures::StreamExt as _; +use http::{Request, Response}; use url::Url; +use crate::auction::AuctionPlan; +use crate::auction::demand::{AdServerImplementation, DemandImplementation}; use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; -pub mod adserver_mock; -pub mod aps; -pub mod datadome; -pub mod didomi; -pub mod google_tag_manager; -pub mod gpt; -pub mod gpt_diagnostics; pub mod js_asset_proxy; -pub mod lockr; -pub mod nextjs; -pub mod osano; -pub mod permutive; -pub mod prebid; mod registry; -pub mod sourcepoint; -pub mod testlight; +#[cfg(test)] +pub(crate) use registry::test_support as registry_test_support; pub use registry::{ - AttributeRewriteAction, AttributeRewriteOutcome, HeaderMutation, HeaderMutationMode, - IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationDocumentState, - IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, - IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, IntegrationMetadata, - IntegrationProxy, IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, - IntegrationRequestFilter, IntegrationScriptContext, IntegrationScriptRewriter, - ProxyDispatchInput, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, - RequestFilterRegistryInput, RequestFilterRegistryOutcome, ScriptRewriteAction, - ScriptTextAccumulator, + AttributeRewriteAction, CarriedJsModule, HeaderMutation, HeaderMutationMode, + IntegrationDocumentState, IntegrationEndpoint, IntegrationMetadata, IntegrationProxy, + IntegrationRegistration, IntegrationRegistrationBuilder, IntegrationRegistry, + IntegrationRequestFilter, IntegrationRequestState, ProxyDispatchInput, RequestFilterDecision, + RequestFilterEffects, RequestFilterInput, RequestFilterRegistryInput, + RequestFilterRegistryOutcome, ScriptRewriteAction, }; /// Registers or retrieves a platform backend for the given URL. @@ -46,11 +35,13 @@ pub use registry::{ /// 15-second first-byte timeout, and delegates to /// [`crate::platform::PlatformBackend::ensure`]. /// +/// Public for the same reason as [`ensure_integration_backend_with_timeout`]. +/// /// # Errors /// /// Returns an error when `url` cannot be parsed, is missing a host, or the /// backend registration fails. -pub(crate) fn ensure_integration_backend( +pub fn ensure_integration_backend( services: &RuntimeServices, url: &str, integration: &'static str, @@ -77,11 +68,15 @@ pub(crate) fn ensure_integration_backend( /// given `first_byte_timeout`, and delegates to /// [`crate::platform::PlatformBackend::ensure`]. /// +/// Public because an integration crate outside this one cannot reach an +/// upstream without registering a backend first, and a crate rolling its own +/// registration would produce a different backend name for the same URL. +/// /// # Errors /// /// Returns an error when `url` cannot be parsed, is missing a host, or the /// backend registration fails. -pub(crate) fn ensure_integration_backend_with_timeout( +pub fn ensure_integration_backend_with_timeout( services: &RuntimeServices, url: &str, integration: &'static str, @@ -106,11 +101,15 @@ pub(crate) fn ensure_integration_backend_with_timeout( /// Parses `url`, builds a [`PlatformBackendSpec`], and delegates to /// [`crate::platform::PlatformBackend::predict_name`]. /// +/// Public for the same reason as [`ensure_integration_backend_with_timeout`], +/// and the two have to agree, so a crate outside this one uses this function +/// and does not derive a name of its own. +/// /// # Errors /// /// Returns an error when the URL cannot be parsed, is missing a host, or the /// platform backend cannot predict a name for the spec. -pub(crate) fn predict_integration_backend_name( +pub fn predict_integration_backend_name( services: &RuntimeServices, url: &str, integration: &'static str, @@ -164,22 +163,34 @@ fn integration_backend_spec( } /// Maximum body size accepted by integration proxy endpoints (256 KiB). -pub(crate) const INTEGRATION_MAX_BODY_BYTES: usize = 256 * 1024; +/// +/// Public so every integration crate bounds a request body at one size. +pub const INTEGRATION_MAX_BODY_BYTES: usize = 256 * 1024; -/// Maximum response body size from RTB providers (prebid, aps, mediator). -pub(crate) const UPSTREAM_RTB_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; +/// Maximum response body size from RTB providers (prebid, aps, ad server). +/// +/// Public so an integration crate outside this one bounds an upstream +/// response at the size the built-in ones do. +pub const UPSTREAM_RTB_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; /// Maximum response body size from SDK/proxy integrations. -pub(crate) const UPSTREAM_SDK_MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +/// +/// Public so every integration crate bounds an upstream script or proxied +/// response at one size. +pub const UPSTREAM_SDK_MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; /// Drains an [`EdgeBody`] into a byte vector, rejecting bodies larger than /// `max_bytes` with [`TrustedServerError::RequestTooLarge`]. /// +/// Public because an integration crate outside this one reads request bodies +/// too, and an unbounded read of a client's body is not a fault each crate +/// should solve again. +/// /// # Errors /// /// Returns an error when: /// - The body exceeds `max_bytes`. /// - A streaming body chunk cannot be read (mapped to an `Integration` error). -pub(crate) async fn collect_body_bounded( +pub async fn collect_body_bounded( body: EdgeBody, max_bytes: usize, integration: &'static str, @@ -226,7 +237,7 @@ pub(crate) async fn collect_body_bounded( /// /// Use this for upstream (provider/integration) response bodies to bound /// memory usage when a third-party server misbehaves. Unlike -/// [`collect_body_bounded`], oversized bodies are classified as +/// `collect_body_bounded`, oversized bodies are classified as /// [`TrustedServerError::Integration`] (502 `BAD_GATEWAY`) rather than /// [`TrustedServerError::RequestTooLarge`] (413). /// @@ -234,12 +245,16 @@ pub(crate) async fn collect_body_bounded( /// because the size check runs after each chunk is materialized. Fastly /// H2/H3 chunks are ≤ 16 KiB in practice, making the overshoot negligible. /// +/// Public because an integration crate outside this one reads upstream +/// responses too, and an unbounded read of a misbehaving upstream is not a +/// fault each crate should solve again. +/// /// # Errors /// /// Returns an error when: /// - The body exceeds `max_bytes` (mapped to [`TrustedServerError::Integration`]). /// - A streaming body chunk cannot be read (same error type). -pub(crate) async fn collect_response_bounded( +pub async fn collect_response_bounded( body: EdgeBody, max_bytes: usize, integration: &'static str, @@ -281,69 +296,737 @@ pub(crate) async fn collect_response_bounded( } } -type IntegrationBuilderFn = +/// Builds an integration's registration from settings, or `None` when the +/// settings give it nothing to register. +/// +/// The registry calls this only for an integration a section selects, so an integration runs exactly when an operator names it. +pub type IntegrationBuilderFn = fn(&Settings) -> Result, Report>; -pub(crate) struct IntegrationBuilder { +/// Validates an integration's configuration for deployment and reports +/// whether a section selects it. +/// +/// Runs for every builder, named or not, so one builder's rules cannot be +/// skipped by the order the builders happen to be in. At deploy time secret +/// fields hold secret-store key names rather than values, so a validator must +/// not depend on a resolved secret. +pub type IntegrationValidateFn = fn(&Settings) -> Result>; + +/// Prepares a request before routing, for every routed request except the +/// health check. +/// +/// Runs whether or not a section selects the integration, so one +/// can strip its own reserved query or cookie in a deployment that does not +/// run it. +pub type IntegrationPrepareRequestFn = + fn(&Settings, &mut Request) -> Result<(), Report>; + +/// Finishes the response the page path returns for one request, from what +/// the module's request hooks left for it in the request's +/// [`IntegrationRequestState`]. +/// +/// Runs whether or not a section selects the integration, as the preparer +/// does, and has nothing to do for a request its module left nothing on. +pub type IntegrationFinalizeResponseFn = fn(&IntegrationRequestState, &mut Response); + +/// Builds a module's registration from the settings and the compiled auction +/// plan, or `None` when the two give it nothing to register. +/// +/// Runs for every builder that has one, whether or not a section selects the +/// module, because a module registered this way follows what the plan +/// selects and decides for itself whether it runs. +pub type IntegrationPlanRegistrationFn = + fn( + &Settings, + &AuctionPlan, + ) -> Result, Report>; + +/// Checks a module's configuration against the compiled auction plan, for a +/// rule that depends on what the plan selects. +/// +/// Runs for every builder that has one, when a deployment is validated and +/// as the settings load. +pub type IntegrationPlanValidateFn = + fn(&Settings, &AuctionPlan) -> Result<(), Report>; + +/// A setting in a module's own table that holds the name of a secret. +/// +/// The name is looked up in the default secret store as the settings load, +/// and the setting holds the secret from then on. +#[derive(Clone, Copy, Debug)] +pub struct ModuleSecretSetting { + /// Where the setting sits inside the module's table, one name for each + /// level. + pub path: &'static [&'static str], + /// Whether the module's table, as written, puts the setting to use. A + /// setting not in use is cleared as the settings load and is not looked + /// up, and one in use has to name a key before a deployment is accepted. + pub in_use: fn(&serde_json::Map) -> bool, +} + +/// Source label for the built-in integrations. +pub const CORE_SOURCE: &str = "trusted-server-core"; + +/// A named factory for one integration, the unit an adapter or a vendor crate +/// hands to [`IntegrationRegistry::with_plan_and_registrations`]. +/// +/// # Examples +/// +/// ``` +/// use error_stack::Report; +/// use trusted_server_core::error::TrustedServerError; +/// use std::sync::Arc; +/// +/// use trusted_server_core::auction::compile_auction_plan; +/// use trusted_server_core::integrations::{ +/// IntegrationBuilder, IntegrationRegistration, IntegrationRegistry, +/// }; +/// use trusted_server_core::settings::Settings; +/// +/// fn build( +/// _settings: &Settings, +/// ) -> Result, Report> { +/// Ok(Some(IntegrationRegistration::builder("example").build())) +/// } +/// +/// fn validate(_settings: &Settings) -> Result> { +/// Ok(true) +/// } +/// +/// # fn demo(settings: &Settings) -> Result<(), Report> { +/// let builder = IntegrationBuilder::new("example", "example-crate", build, validate) +/// .with_module_name("testing.example"); +/// // The registry builds a module a section selects, so a deployment that +/// // wants this one writes `[testing] modules = ["example"]`. +/// let mut settings = settings.clone(); +/// settings.select_module("testing", "testing.example"); +/// let plan = Arc::new(compile_auction_plan(&settings)?); +/// let registry = IntegrationRegistry::with_plan_and_registrations(&settings, plan, &[builder])?; +/// assert!(registry.integration_runs("example")); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Copy, Debug)] +pub struct IntegrationBuilder { id: &'static str, + source: &'static str, build: IntegrationBuilderFn, + validate: IntegrationValidateFn, + prepare_request: Option, + finalize_response: Option, + plan_registration: Option, + plan_validator: Option, + reads_auction_token: bool, + secret_settings: &'static [ModuleSecretSetting], + supplies_integration: bool, + demand: Option<&'static DemandImplementation>, + adserver: Option<&'static AdServerImplementation>, + module: Option<&'static str>, + section: Option<&'static str>, + loading_cache: Option, + shared: Option<&'static str>, +} + +/// Gives the loading cache a module supplies, see +/// [`IntegrationBuilder::with_loading_cache`]. +pub type LoadingCacheFactory = fn() -> Arc; + +/// The build function of a builder that supplies no page integration. +fn no_registration( + _settings: &Settings, +) -> Result, Report> { + Ok(None) +} + +/// The validate function of a builder that supplies no page integration. +fn nothing_to_validate(_settings: &Settings) -> Result> { + Ok(false) +} + +impl IntegrationBuilder { + /// Creates a builder for the integration `id`, attributed to `source` + /// (a crate or package name used in duplicate-id errors). + #[must_use] + pub const fn new( + id: &'static str, + source: &'static str, + build: IntegrationBuilderFn, + validate: IntegrationValidateFn, + ) -> Self { + Self { + id, + source, + build, + validate, + prepare_request: None, + finalize_response: None, + plan_registration: None, + plan_validator: None, + reads_auction_token: false, + secret_settings: &[], + supplies_integration: true, + demand: None, + adserver: None, + module: None, + section: None, + loading_cache: None, + shared: None, + } + } + + /// Creates a builder that supplies only implementations, such as a demand + /// or ad server implementation, and no page integration. + /// + /// Its `id` still claims a place among builder ids, so two crates cannot + /// register under one id, but it is not an integration a deployment can + /// name. + #[must_use] + pub const fn implementations(id: &'static str, source: &'static str) -> Self { + Self { + id, + source, + build: no_registration, + validate: nothing_to_validate, + prepare_request: None, + finalize_response: None, + plan_registration: None, + plan_validator: None, + reads_auction_token: false, + secret_settings: &[], + supplies_integration: false, + demand: None, + adserver: None, + module: None, + section: None, + loading_cache: None, + shared: None, + } + } + + /// Registers a demand implementation, which `[demand] modules` or an + /// `implementation` line can name. + #[must_use] + pub const fn with_demand(mut self, demand: &'static DemandImplementation) -> Self { + self.demand = Some(demand); + self + } + + /// Registers an ad server implementation, which `[ad-server] module` or + /// an `implementation` line can name. + #[must_use] + pub const fn with_adserver(mut self, adserver: &'static AdServerImplementation) -> Self { + self.adserver = Some(adserver); + self + } + + /// Whether this builder supplies a page integration, as opposed to only + /// implementations. + #[must_use] + pub const fn supplies_integration(&self) -> bool { + self.supplies_integration + } + + /// The demand implementation this builder registers, when it registers + /// one. + #[must_use] + pub const fn demand(&self) -> Option<&'static DemandImplementation> { + self.demand + } + + /// The ad server implementation this builder registers, when it registers + /// one. + #[must_use] + pub const fn adserver(&self) -> Option<&'static AdServerImplementation> { + self.adserver + } + + /// Attaches a request preparation function that runs before routing on + /// every request, whether or not the integration runs. + #[must_use] + pub const fn with_request_preparer(mut self, prepare: IntegrationPrepareRequestFn) -> Self { + self.prepare_request = Some(prepare); + self + } + + /// Attaches a function that finishes the response the page path returns, + /// from what the module's request hooks left for the request. + #[must_use] + pub const fn with_response_finalizer( + mut self, + finalize: IntegrationFinalizeResponseFn, + ) -> Self { + self.finalize_response = Some(finalize); + self + } + + /// Registers the module from the compiled auction plan as well as the + /// settings, for a module whose page support follows what the plan + /// selects. + /// + /// The function runs whether or not a section selects the module, and + /// what it registers has its hooks run ahead of the modules the sections + /// select. + #[must_use] + pub const fn with_plan_registration(mut self, register: IntegrationPlanRegistrationFn) -> Self { + self.plan_registration = Some(register); + self + } + + /// Attaches a check of the module's configuration against the compiled + /// auction plan. + #[must_use] + pub const fn with_plan_validator(mut self, validate: IntegrationPlanValidateFn) -> Self { + self.plan_validator = Some(validate); + self + } + + /// Declares that the module's browser script reads the token an auction + /// publishes with its winning bids. + /// + /// A token is made for each auction when a section selects such a + /// module, and none is made in a deployment where nothing reads it. + #[must_use] + pub const fn with_auction_token(mut self) -> Self { + self.reads_auction_token = true; + self + } + + /// Whether the module's browser script reads the auction token. + #[must_use] + pub const fn reads_auction_token(&self) -> bool { + self.reads_auction_token + } + + /// Declares the settings in the module's own table that hold the name of + /// a secret, so each is looked up as the settings load and checked when + /// a deployment is validated. + #[must_use] + pub const fn with_secret_settings(mut self, secrets: &'static [ModuleSecretSetting]) -> Self { + self.secret_settings = secrets; + self + } + + /// The settings in the module's own table that hold the name of a + /// secret. + #[must_use] + pub const fn secret_settings(&self) -> &'static [ModuleSecretSetting] { + self.secret_settings + } + + /// Runs this builder when a section selects the module `name`, which is a + /// crate's path under `crates` with `.` between the parts, such as + /// `cmp.example`. An integration still in core holds the name its crate + /// will have as a constant. + #[must_use] + pub const fn with_module_name(mut self, name: &'static str) -> Self { + self.module = Some(name); + self + } + + /// The name a section selects this builder's module by, when it has one. + #[must_use] + pub const fn module_name(&self) -> Option<&'static str> { + self.module + } + + /// Offers the loading cache this crate supplies, see + /// [`crate::platform::PlatformLoadingCache`]. + /// + /// The offer is not a selection. A cache is selected by naming the module + /// in its own `[cache.]` table, and an adapter resolves every table's + /// selection with [`loading_caches_from`] when it builds its state. A cache + /// whose table selects none asks each source itself. + #[must_use] + pub const fn with_loading_cache(mut self, factory: LoadingCacheFactory) -> Self { + self.loading_cache = Some(factory); + self + } + + /// Runs this builder when `[shared.]` is present, for a vendor + /// whose modules all read that one table and cannot be built without it. + /// A builder the table runs is selected by no section, so the registry + /// refuses one that also has a module name. + #[must_use] + pub const fn with_shared_settings(mut self, vendor: &'static str) -> Self { + self.shared = Some(vendor); + self + } + + /// The vendor whose `[shared.]` table runs this builder, when one + /// does. + #[must_use] + pub const fn shared_settings(&self) -> Option<&'static str> { + self.shared + } + + /// The loading cache this builder offers, when it offers one. + #[must_use] + pub const fn loading_cache(&self) -> Option { + self.loading_cache + } + + /// Names the section that selects this module, for one selected outside + /// the section of its type, such as a module of core's own in a section + /// of core's. + #[must_use] + pub const fn selected_in(mut self, section: &'static str) -> Self { + self.section = Some(section); + self + } + + /// The section that selects this builder's module: the one + /// [`selected_in`](Self::selected_in) named, or else its type's. + #[must_use] + pub fn section(&self) -> Option<&'static str> { + self.section.or_else(|| { + self.module + .and_then(|name| name.split_once('.').map(|(folder, _)| folder)) + }) + } + + /// The integration id this builder produces. + #[must_use] + pub const fn id(&self) -> &'static str { + self.id + } + + /// The source label used in diagnostics. + #[must_use] + pub const fn source(&self) -> &'static str { + self.source + } + + /// Builds the registration, or `None` when the settings give the + /// integration nothing to register. + /// + /// # Errors + /// + /// Returns an error when the integration runs with invalid configuration. + pub(crate) fn build( + &self, + settings: &Settings, + ) -> Result, Report> { + (self.build)(settings) + } + + /// Validates the integration's configuration for deployment and reports + /// whether a section selects the integration's module. + /// + /// # Errors + /// + /// Returns an error when the configuration cannot be parsed or fails + /// validation. + pub(crate) fn validate(&self, settings: &Settings) -> Result> { + (self.validate)(settings) + } + + /// The request preparation function, when one is attached. + pub(crate) fn prepare_request(&self) -> Option { + self.prepare_request + } + + /// The response finalizer, when one is attached. + pub(crate) fn finalize_response(&self) -> Option { + self.finalize_response + } + + /// The registration from the auction plan, when one is attached. + pub(crate) fn plan_registration(&self) -> Option { + self.plan_registration + } + + /// The check against the auction plan, when one is attached. + pub(crate) fn plan_validator(&self) -> Option { + self.plan_validator + } } +/// The built-in integrations, in the order `trusted-server-modules` lists them. +const BUILT_IN_BUILDERS: &[IntegrationBuilder] = &[ + // This stays the first module a section selects, so its middleware is + // the first of theirs in the order the registry lists them. + js_asset_proxy::BUILDER, + // A stand-in for an integration that streams, which core's own tests + // select where they need one. + #[cfg(test)] + registry_test_support::payload_fixture::BUILDER, + // A stand-in for an integration that tags a page, for the same tests. + #[cfg(test)] + registry_test_support::tag_fixture::BUILDER, + // A stand-in for an integration that acts on one request, for the same + // tests. + #[cfg(test)] + registry_test_support::request_fixture::BUILDER, + // A stand-in for an integration whose browser module loads deferred, for + // the same tests. + #[cfg(test)] + registry_test_support::deferred_fixture::BUILDER, + // A stand-in for a module that changes a page through middleware, for + // the same tests. + #[cfg(test)] + registry_test_support::middleware_fixture::BUILDER, + // A stand-in for the plainest demand implementation there can be, which + // core's own tests name where they need a source. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::plain_fixture::MODULE, + CORE_SOURCE, + ) + .with_demand(&crate::auction::test_support::plain_fixture::DEMAND), + // A stand-in demand implementation, which core's own tests name where + // they need one that departs from the plain one. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::demand_fixture::MODULE, + CORE_SOURCE, + ) + .with_demand(&crate::auction::test_support::demand_fixture::DEMAND), + // A stand-in ad server, which core's own tests select where they need + // one. + #[cfg(test)] + IntegrationBuilder::implementations( + crate::auction::test_support::adserver_fixture::MODULE, + CORE_SOURCE, + ) + .with_adserver(&crate::auction::test_support::adserver_fixture::ADSERVER), +]; + +/// The built-in integration builders, in the order `trusted-server-modules` +/// lists them. pub(crate) fn builders() -> &'static [IntegrationBuilder] { - &[ - // This must remain first: attribute rewriters chain replacements and short-circuit removals. - IntegrationBuilder { - id: js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, - build: js_asset_proxy::register, - }, - IntegrationBuilder { - id: "testlight", - build: testlight::register, - }, - IntegrationBuilder { - id: "nextjs", - build: nextjs::register, - }, - IntegrationBuilder { - id: "permutive", - build: permutive::register, - }, - IntegrationBuilder { - id: "lockr", - build: lockr::register, - }, - IntegrationBuilder { - id: "didomi", - build: didomi::register, - }, - IntegrationBuilder { - id: "sourcepoint", - build: sourcepoint::register, - }, - IntegrationBuilder { - id: "osano", - build: osano::register, - }, - IntegrationBuilder { - id: "google_tag_manager", - build: google_tag_manager::register, - }, - IntegrationBuilder { - id: "datadome", - build: datadome::register, - }, - IntegrationBuilder { - id: "gpt", - build: gpt::register, - }, - IntegrationBuilder { - id: "gpt_diagnostics", - build: gpt_diagnostics::register, - }, - ] + BUILT_IN_BUILDERS +} + +/// Every builder the registry will consider: the built-in set followed by +/// `extra`, in that order, so the built-ins keep their order. +pub(crate) fn all_builders( + extra: &[IntegrationBuilder], +) -> impl Iterator + '_ { + builders().iter().copied().chain(extra.iter().copied()) +} + +/// The loading cache each `[cache.]` table selects, from the caches the +/// builders a deployment links offer. A table that names no module keeps no +/// copy, and a module's cache is made once however many tables select it. +/// +/// # Errors +/// +/// When a table names a module no linked builder offers a cache from, naming +/// the table and the modules that do. +pub fn loading_caches_from( + settings: &Settings, + builders: &[IntegrationBuilder], +) -> Result, Report> { + let offered: Vec<(&'static str, LoadingCacheFactory)> = builders + .iter() + .filter_map(|builder| { + builder + .loading_cache() + .map(|factory| (builder.id(), factory)) + }) + .collect(); + let mut made: Vec<(&'static str, Arc)> = Vec::new(); + let mut caches = crate::platform::LoadingCaches::default(); + for (table, config) in settings.cache.loading_caches() { + // A table naming no module takes the one loading cache module the + // build links, so answers are held without a line in the document. + // With none linked, or several, it holds nothing. + let written = match config.module.as_deref() { + Some(written) => written, + None if offered.len() == 1 => offered[0].0, + None => continue, + }; + let Some((name, factory)) = offered.iter().find(|(name, _)| { + *name == written || crate::module_name::short_form(LOADING_CACHE_TYPE, name) == written + }) else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[cache.{table}] module names `{written}`, which no linked crate offers \ + a loading cache from. The loading cache modules this build has are: {}", + offered_cache_names(&offered) + ), + })); + }; + let cache = match made.iter().find(|(made_name, _)| made_name == name) { + Some((_, cache)) => Arc::clone(cache), + None => { + let cache = factory(); + made.push((*name, Arc::clone(&cache))); + cache + } + }; + caches.insert(table, cache); + } + Ok(Arc::new(caches)) +} + +/// The type folder loading cache modules live under, which their short +/// names leave out. +const LOADING_CACHE_TYPE: &str = "loading-cache"; + +/// The short names of the modules offering a cache, for a refusal. +fn offered_cache_names(offered: &[(&'static str, LoadingCacheFactory)]) -> String { + if offered.is_empty() { + return "none".to_owned(); + } + offered + .iter() + .map(|(name, _)| crate::module_name::short_form(LOADING_CACHE_TYPE, name)) + .collect::>() + .join(", ") } #[cfg(test)] -pub(crate) fn registered_builder_ids() -> impl Iterator { - builders().iter().map(|builder| builder.id) +mod loading_caches_from_tests { + use bytes::Bytes; + + use super::*; + use crate::platform::{CacheSpec, LoadError, Loader, PlatformLoadingCache, StoreKey}; + use crate::settings::ROBOTS_TXT_CACHE; + + fn no_build( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + fn no_validate(_settings: &Settings) -> Result> { + Ok(true) + } + + /// A cache that says it keeps values, so a test can tell it from the + /// default, which keeps none. + struct Keeping; + + #[async_trait::async_trait(?Send)] + impl PlatformLoadingCache for Keeping { + async fn get_or_load( + &self, + _services: &crate::platform::RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + load: Loader, + ) -> Result { + load().await.map(|loaded| loaded.bytes) + } + + async fn remove( + &self, + _services: &crate::platform::RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + ) { + } + } + + fn keeping() -> Arc { + Arc::new(Keeping) + } + + fn nothing() -> Arc { + Arc::new(crate::platform::NoLoadingCache) + } + + fn builder(source: &'static str) -> IntegrationBuilder { + IntegrationBuilder::new("an_id", source, no_build, no_validate) + } + + /// A module `id` offering a cache that keeps values. + fn offering(id: &'static str) -> IntegrationBuilder { + IntegrationBuilder::implementations(id, "a-crate").with_loading_cache(keeping) + } + + /// A module `id` offering a cache that keeps nothing. + fn offering_nothing(id: &'static str) -> IntegrationBuilder { + IntegrationBuilder::implementations(id, "a-crate").with_loading_cache(nothing) + } + + fn settings_selecting(module: Option<&str>) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.cache.robots_txt.module = module.map(str::to_owned); + settings + } + + #[test] + fn a_feature_selecting_nothing_takes_the_one_module_offered() { + let caches = loading_caches_from( + &settings_selecting(None), + &[builder("first"), offering("loading-cache.one")], + ) + .expect("no selection is not an error"); + + assert!(caches.for_feature(ROBOTS_TXT_CACHE).keeps_values()); + assert!(!caches.is_empty()); + } + + #[test] + fn a_feature_selecting_nothing_keeps_nothing_when_two_are_offered() { + let caches = loading_caches_from( + &settings_selecting(None), + &[offering("loading-cache.one"), offering("loading-cache.two")], + ) + .expect("no selection is not an error"); + + assert!(!caches.for_feature(ROBOTS_TXT_CACHE).keeps_values()); + assert!(caches.is_empty()); + } + + #[test] + fn a_feature_selecting_nothing_keeps_nothing_when_none_is_offered() { + let caches = loading_caches_from(&settings_selecting(None), &[builder("first")]) + .expect("no selection is not an error"); + + assert!(!caches.for_feature(ROBOTS_TXT_CACHE).keeps_values()); + assert!(caches.is_empty()); + } + + #[test] + fn a_feature_selects_a_module_by_its_short_name_or_in_full() { + for written in ["one", "loading-cache.one"] { + let caches = loading_caches_from( + &settings_selecting(Some(written)), + &[ + offering_nothing("loading-cache.other"), + offering("loading-cache.one"), + ], + ) + .expect("a selected module should be found"); + + assert!( + caches.for_feature(ROBOTS_TXT_CACHE).keeps_values(), + "{written} should select the cache that keeps values, not the first offered" + ); + assert!(!caches.is_empty()); + } + } + + #[test] + fn a_feature_naming_a_module_nothing_offers_is_refused_naming_what_is_offered() { + let error = loading_caches_from( + &settings_selecting(Some("none")), + &[offering("loading-cache.one"), offering("loading-cache.two")], + ) + .expect_err("a module nothing offers should be refused"); + + let message = format!("{error:?}"); + assert!( + message.contains( + "[cache.robots-txt] module names `none`, which no linked crate offers a \ + loading cache from. The loading cache modules this build has are: one, two" + ), + "{message}" + ); + } + + #[test] + fn with_nothing_offered_the_refusal_says_so() { + let error = loading_caches_from(&settings_selecting(Some("one")), &[builder("first")]) + .expect_err("a selection with nothing offered should be refused"); + + let message = format!("{error:?}"); + assert!( + message.contains("The loading cache modules this build has are: none"), + "{message}" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/osano.rs b/crates/trusted-server-core/src/integrations/osano.rs deleted file mode 100644 index 9fc3aa66c1..0000000000 --- a/crates/trusted-server-core/src/integrations/osano.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Osano integration for client-side consent mirroring. -//! -//! The Rust side of this integration intentionally only provides explicit -//! enablement for the `tsjs-osano` browser module. Osano consent extraction runs -//! in JavaScript because the relevant CMP APIs (`__uspapi`, `__gpp`, and -//! `__tcfapi`) are browser-only. - -use error_stack::Report; -use serde::Deserialize; -use validator::Validate; - -use crate::error::TrustedServerError; -use crate::settings::{IntegrationConfig, Settings}; - -use super::IntegrationRegistration; - -const OSANO_INTEGRATION_ID: &str = "osano"; - -/// Configuration for the Osano consent mirror integration. -#[derive(Debug, Clone, Deserialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct OsanoConfig { - /// Whether the Osano browser consent mirror is enabled. - #[serde(default)] - pub enabled: bool, -} - -impl IntegrationConfig for OsanoConfig { - fn is_enabled(&self) -> bool { - self.enabled - } -} - -/// Register the Osano JS integration when enabled. -/// -/// # Errors -/// -/// Returns an error when the Osano integration configuration cannot be parsed or -/// fails validation. -pub fn register( - settings: &Settings, -) -> Result, Report> { - let Some(_config) = settings.integration_config::(OSANO_INTEGRATION_ID)? else { - return Ok(None); - }; - - Ok(Some( - IntegrationRegistration::builder(OSANO_INTEGRATION_ID).build(), - )) -} - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::{OsanoConfig, register}; - use crate::test_support::tests::create_test_settings; - - #[test] - fn register_returns_none_when_disabled() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("osano", &json!({ "enabled": false })) - .expect("should insert osano config"); - - let registration = register(&settings).expect("should parse disabled osano config"); - - assert!( - registration.is_none(), - "disabled Osano integration should not register" - ); - } - - #[test] - fn register_returns_js_module_registration_when_enabled() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("osano", &json!({ "enabled": true })) - .expect("should insert osano config"); - - let registration = register(&settings) - .expect("should parse enabled osano config") - .expect("enabled Osano integration should register"); - - assert_eq!(registration.integration_id, "osano"); - assert!( - registration.proxies.is_empty(), - "Osano v1 should not register Rust proxy routes" - ); - assert!( - registration.head_injectors.is_empty(), - "Osano v1 should not inject HTML from Rust" - ); - } - - #[test] - fn config_rejects_unknown_fields() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("osano", &json!({ "enabled": true, "typo": true })) - .expect("should insert osano config"); - - let err = settings - .integration_config::("osano") - .expect_err("should reject unknown Osano config fields"); - let error_text = format!("{err:?}"); - - assert!( - error_text.contains("typo") || error_text.contains("unknown field"), - "error should mention the unknown field: {err:?}" - ); - } -} diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d3c09cd7d7..3083aa0871 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1,25 +1,33 @@ use std::any::{Any, TypeId}; use std::collections::BTreeMap; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +use sha2::{Digest as _, Sha256}; +use crate::attestation::PlatformIdentity; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; +use crate::ec::device::DeviceModule; use crate::ec::kv::KvIdentityGraph; +use crate::ec::module::{EcModuleSelection, EdgeCookieModule}; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::http_util::is_navigation_request; -use crate::platform::RuntimeServices; +use crate::middleware::{Middleware, MiddlewareChain, MiddlewarePhase, PhaseEntries}; +use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, ResolvedRequest}; +use crate::platform::{DisabledGeo, PlatformGeo, RuntimeServices}; +use crate::prefix::{PrefixModule, Prefixes, SelectedPrefixModule}; +use crate::robots_txt::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor}; use crate::settings::Settings; -use crate::streaming_processor::StreamProcessor; -/// Action returned by attribute rewriters to describe how the runtime should mutate the element. +/// What a middleware's element handler decides about one attribute of an +/// element. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AttributeRewriteAction { /// Leave the attribute and element untouched. @@ -47,15 +55,8 @@ impl AttributeRewriteAction { } } -/// Outcome returned by the registry after running every matching attribute rewriter. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AttributeRewriteOutcome { - Unchanged, - Replaced(String), - RemoveElement, -} - -/// Action returned by inline script rewriters to describe how to mutate the node. +/// What a middleware's text handler decides about one chunk of the text +/// inside an element. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ScriptRewriteAction { Keep, @@ -80,33 +81,13 @@ impl ScriptRewriteAction { } } -/// Context provided to integration HTML attribute rewriters. -#[derive(Debug)] -pub struct IntegrationAttributeContext<'a> { - pub attribute_name: &'a str, - pub element_name: &'a str, - pub request_host: &'a str, - pub request_scheme: &'a str, - pub origin_host: &'a str, -} - -/// Context passed to script/text rewriters for inline HTML handling. -#[derive(Debug)] -pub struct IntegrationScriptContext<'a> { - pub selector: &'a str, - pub request_host: &'a str, - pub request_scheme: &'a str, - pub origin_host: &'a str, - pub is_last_in_text_node: bool, - pub max_buffered_script_bytes: usize, - pub document_state: &'a IntegrationDocumentState, -} - type IntegrationDocumentStateMap = BTreeMap<(&'static str, TypeId), Arc>; -/// Per-document state shared between HTML/script rewriters and post-processors. +/// State shared by the middleware working on one document in one phase. /// -/// This exists to support multi-phase HTML processing without requiring a second HTML parse. +/// What one handler leaves here a later handler or a stream processor reads, +/// so the document is parsed once. In the serve phase it starts with what +/// the request's hooks left, see [`IntegrationRequestState`]. #[derive(Clone, Default)] pub struct IntegrationDocumentState { inner: Arc>, @@ -191,35 +172,101 @@ impl IntegrationDocumentState { } } -/// Per-document buffer for script text fragments split across chunks. +/// Values a module's request hooks leave for its own serve middleware, for +/// one request. /// -/// `lol_html` can deliver one text node as several chunks, so a rewriter that -/// needs the whole script must accumulate until `is_last_in_text_node`. +/// A request preparer or a request filter leaves a value under its +/// integration id with [`IntegrationRequestState::insert`]. Where the +/// request's reader is served an HTML document, every value is copied into +/// the [`IntegrationDocumentState`] the `[[serve]]` entry's middleware share, +/// so the module's serve middleware reads it there, and the same values are +/// handed to the module's response finalizer. /// -/// This lives in [`IntegrationDocumentState`] rather than on the rewriter. -/// Rewriters are registered once as `Arc` and -/// live as long as the [`IntegrationRegistry`], so a buffer owned by a -/// rewriter is shared by every document that registry serves. A document whose -/// stream ends before the final fragment — client disconnect, origin error, -/// truncated body — leaves its partial script in that buffer, and the next -/// document prepends the residue to its own accumulation. That corrupts the -/// response and can disclose the previous document's content. -/// -/// Keyed per integration id, so each integration gets its own buffer, and -/// dropped with the document state at end of document. -#[derive(Debug, Default)] -pub struct ScriptTextAccumulator { - buffer: Mutex, +/// A request that carries any value keeps to the origin path. Its document is +/// never read from a shared template and never stored as one, and an HTML +/// response with a body is sent `private, no-store`. A fetch middleware is +/// handed none of these values. +#[derive(Clone, Default)] +pub struct IntegrationRequestState { + values: IntegrationDocumentStateMap, +} + +impl std::fmt::Debug for IntegrationRequestState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IntegrationRequestState") + .field("keys", &self.values.keys().collect::>()) + .finish() + } } -impl ScriptTextAccumulator { - /// Locks the buffer. +impl IntegrationRequestState { + /// Leaves `value` on `request` for the serve middleware of + /// `integration_id`, in place of a value of the same type left earlier. + pub fn insert(request: &mut Request, integration_id: &'static str, value: T) + where + T: Any + Send + Sync + 'static, + { + let mut state = request + .extensions_mut() + .remove::() + .unwrap_or_default(); + state.set(integration_id, value); + request.extensions_mut().insert(state); + } + + /// Holds `value` for the serve middleware of `integration_id`, in place + /// of a value of the same type held earlier. + pub fn set(&mut self, integration_id: &'static str, value: T) + where + T: Any + Send + Sync + 'static, + { + self.values + .insert((integration_id, TypeId::of::()), Arc::new(value)); + } + + /// The values left on `request`, which is none for a request no module + /// left one on. + #[must_use] + pub fn of(request: &Request) -> Self { + request + .extensions() + .get::() + .cloned() + .unwrap_or_default() + } + + /// The value of type `T` left for `integration_id`. + #[must_use] + pub fn get(&self, integration_id: &'static str) -> Option> + where + T: Any + Send + Sync + 'static, + { + let value = self.values.get(&(integration_id, TypeId::of::()))?; + Arc::clone(value).downcast::().ok() + } + + /// Whether no module left a value. + #[must_use] + pub fn is_empty(&self) -> bool { + self.values.is_empty() + } + + /// Copies every value into the state of a document that is starting. + /// + /// # Panics /// - /// Recovers from poisoning rather than panicking: a poisoned buffer holds - /// at worst a partial script, and the caller's `is_last_in_text_node` - /// handling already tolerates unexpected contents. - pub fn buffer(&self) -> MutexGuard<'_, String> { - self.buffer.lock().unwrap_or_else(PoisonError::into_inner) + /// Panics if the document state's lock is poisoned. + pub(crate) fn seed(&self, document: &IntegrationDocumentState) { + if self.values.is_empty() { + return; + } + let mut guard = document + .inner + .lock() + .expect("should lock integration document state"); + for (key, value) in &self.values { + guard.insert(*key, Arc::clone(value)); + } } } @@ -315,11 +362,21 @@ pub trait IntegrationProxy: Send + Sync { /// or define routes manually for backwards compatibility. fn routes(&self) -> Vec; + /// The permissions this route declares, which decide whether a gated + /// value, such as the Edge Cookie identifier, is passed to it. None by + /// default. + fn required_permissions(&self) -> crate::permissions::PermissionSet { + crate::permissions::PermissionSet::none() + } + /// Handle the proxied request. + /// + /// Hand a function to [`ModuleCall::inject_with`] with `req` as its + /// argument. A route whose use needs a permission declares it and checks + /// it is granted, because nothing gates the route itself. async fn handle( &self, - settings: &Settings, - services: &RuntimeServices, + call: ModuleCall<'_>, req: Request, ) -> Result, Report>; @@ -365,6 +422,11 @@ pub struct RequestFilterInput<'a> { pub services: &'a RuntimeServices, pub request: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, + /// The permission state resolved for this request at the start of the + /// request cycle, so a filter reads the same permissions the rest of the + /// request uses rather than resolving its own. `None` only on paths that + /// build no EC context, such as batch sync. + pub permissions: Option<&'a crate::permissions::PermissionState>, /// Whether the request matches a registered integration proxy route. pub is_integration_route: bool, } @@ -445,6 +507,10 @@ pub struct RequestFilterRegistryInput<'a> { pub services: &'a RuntimeServices, pub req: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, + /// The permission state resolved for this request at the start of the + /// request cycle, passed on to every filter. `None` only on paths that + /// build no EC context, such as batch sync. + pub permissions: Option<&'a crate::permissions::PermissionState>, } /// Outcome returned by [`IntegrationRegistry::filter_request`]. @@ -548,74 +614,16 @@ fn apply_header_mutation_to_response(response: &mut Response, mutation } } -/// Trait for integration-provided HTML attribute rewrite hooks. -pub trait IntegrationAttributeRewriter: Send + Sync { - /// Identifier for logging/diagnostics. - fn integration_id(&self) -> &'static str; - /// Return true when this rewriter wants to inspect a given attribute. - fn handles_attribute(&self, attribute: &str) -> bool; - /// Attempt to rewrite the attribute value. Return `AttributeRewriteAction::Replace` - /// to update the attribute, `Keep` to leave it untouched, or `RemoveElement` to drop the node. - fn rewrite( - &self, - attr_name: &str, - attr_value: &str, - ctx: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction; -} - -/// Trait for integration-provided inline script/text rewrite hooks. -pub trait IntegrationScriptRewriter: Send + Sync { - /// Identifier for logging/diagnostics. - fn integration_id(&self) -> &'static str; - /// CSS selector (e.g. `script#__NEXT_DATA__`) that should trigger this rewriter. - fn selector(&self) -> &'static str; - /// Attempt to rewrite the inline text content for the selector. - fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction; -} - -/// Context for HTML post-processors. -#[derive(Debug)] -pub struct IntegrationHtmlContext<'a> { - pub request_host: &'a str, - pub request_scheme: &'a str, - pub origin_host: &'a str, - pub document_state: &'a IntegrationDocumentState, -} - -/// Owned request data supplied when an integration creates an HTML stream processor. -#[derive(Clone)] -pub struct IntegrationHtmlStreamContext { - /// Publisher-facing host used for rewritten URLs. - pub request_host: String, - /// Publisher-facing scheme used for rewritten URLs. - pub request_scheme: String, - /// Origin host whose URLs may be rewritten. - pub origin_host: String, - /// Request-local state shared with the document's integration rewriters. - pub document_state: IntegrationDocumentState, -} - -/// Creates one mutable HTML output processor for each document. -pub trait IntegrationHtmlStreamProcessorFactory: Send + Sync { - /// Identifier for logging and diagnostics. - fn integration_id(&self) -> &'static str; - - /// Create a request-local streaming processor. - fn create(&self, context: IntegrationHtmlStreamContext) -> Box; -} - -/// Trait for integration-provided HTML head injections. -pub trait IntegrationHeadInjector: Send + Sync { - /// Identifier for logging/diagnostics. - fn integration_id(&self) -> &'static str; - /// Return HTML snippets to insert at the start of ``. - fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec; - - /// Return attributes to add to the publisher TSJS bundle tag. - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - Vec::new() - } +/// A browser module a registration carries, for a module built outside +/// `trusted-server-js`. The crate embeds its built IIFE with `include_str!` +/// and states its SHA-256 as a literal next to it; the registry verifies the +/// two agree when it is built, and the served `?v=` hash is derived from it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CarriedJsModule { + /// The built IIFE. + pub source: &'static str, + /// SHA-256 of `source`, hex encoded, lower case. + pub sha256: &'static str, } /// Registration payload returned by integration builders. @@ -623,12 +631,56 @@ pub struct IntegrationRegistration { pub integration_id: &'static str, pub js_deferred: bool, pub js_disabled: bool, + /// Browser module carried by the registration, when the module is not + /// compiled into `trusted-server-js`. + pub js_module: Option, + /// Serve the module only on its own `/static/tsjs=tsjs-.min.js` path, + /// never in the unified bundle and never as a deferred tag; the + /// integration injects the tag itself when it decides to. + pub js_standalone: bool, pub proxies: Vec>, - pub attribute_rewriters: Vec>, - pub script_rewriters: Vec>, - pub html_stream_processors: Vec>, - pub head_injectors: Vec>, pub request_filters: Vec>, + /// The page changes this registration supplies, see [`crate::middleware`]. + /// + /// Declaring one does not make it run, because a middleware changes a + /// page only where an entry of the settings names it. + pub middleware: Vec>, + /// Attributes this registration puts on the script bundle's tag, each a + /// name and a value, for a browser module that reads a setting from the + /// tag it was loaded by. + pub bundle_tag_attributes: Vec<(&'static str, &'static str)>, + /// Geo module this registration supplies, with the name `[geo] module` + /// selects it by. + /// + /// Declaring one does not make it active, because the module is only asked + /// to resolve location when `[geo] module` names it. + pub geo_module: Option<(&'static str, Arc)>, + /// Edge Cookie module this registration supplies, with the name + /// `[ec] module` selects it by. + /// + /// Declaring one does not make it active, because the module is only asked + /// to create an identifier when `[ec] module` names it. This is the same + /// route geo takes, so identity is not a second extension mechanism + /// sitting beside the integration system. + pub ec_module: Option<(&'static str, Arc)>, + /// Device module this registration supplies, with the name + /// `[device] module` selects it by. + /// + /// Declaring one does not make it active, because the module is only asked + /// to classify a request when `[device] module` names it. + pub device_module: Option<(&'static str, Arc)>, + /// Prefix module this registration supplies, with the name + /// `[prefix] module` selects it by. + /// + /// Declaring one does not make it active, because the module only gives + /// the deployment's prefixes when `[prefix] module` names it. + pub prefix_module: Option<(&'static str, Arc)>, + /// The rules this registration adds to `/robots.txt`, with the name + /// `[robots-txt] modules` selects them by. + /// + /// Declaring them does not put them in the file, because the file is only + /// built from the contributors `[robots-txt] modules` names. + pub robots_txt_contributor: Option<(&'static str, Arc)>, } impl IntegrationRegistration { @@ -649,12 +701,17 @@ impl IntegrationRegistrationBuilder { integration_id, js_deferred: false, js_disabled: false, + js_module: None, + js_standalone: false, proxies: Vec::new(), - attribute_rewriters: Vec::new(), - script_rewriters: Vec::new(), - html_stream_processors: Vec::new(), - head_injectors: Vec::new(), request_filters: Vec::new(), + middleware: Vec::new(), + bundle_tag_attributes: Vec::new(), + geo_module: None, + ec_module: None, + device_module: None, + prefix_module: None, + robots_txt_contributor: None, }, } } @@ -666,38 +723,100 @@ impl IntegrationRegistrationBuilder { } #[must_use] - pub fn with_attribute_rewriter( - mut self, - rewriter: Arc, - ) -> Self { - self.registration.attribute_rewriters.push(rewriter); + pub fn with_request_filter(mut self, filter: Arc) -> Self { + self.registration.request_filters.push(filter); self } + /// Declare a page change this registration supplies, see + /// [`crate::middleware`]. Called once for each one. + /// + /// The middleware changes a page only where an entry of the settings + /// names it, and one no entry names is logged as a warning when the + /// registry is built. #[must_use] - pub fn with_script_rewriter(mut self, rewriter: Arc) -> Self { - self.registration.script_rewriters.push(rewriter); + pub fn with_middleware(mut self, middleware: Arc) -> Self { + self.registration.middleware.push(middleware); self } + /// Put an attribute on the script bundle's tag, on every page the bundle + /// is written into. + /// + /// The name is lower case letters, digits and hyphens, and the value + /// holds none of `"`, `&`, `<` and `>`, or startup is refused. Where two + /// registrations give one name different values the first is kept, and + /// the other is logged as a warning. #[must_use] - pub fn with_html_stream_processor( - mut self, - processor: Arc, - ) -> Self { - self.registration.html_stream_processors.push(processor); + pub fn with_bundle_tag_attribute(mut self, name: &'static str, value: &'static str) -> Self { + self.registration.bundle_tag_attributes.push((name, value)); self } + /// Declare the geo module this registration supplies, under its `name`. + /// + /// The name is the path under `crates` of the crate the module lives in, + /// as [`crate::module_name!`] gives it, so a module from + /// `crates/geo/example` is selected by `[geo] module = "example"`. One + /// registration can therefore supply a module of each type, each under + /// the name of its own crate. The module only resolves location when + /// `[geo] module` names it, and a module the selector does not choose is + /// logged as a warning when the registry is built. #[must_use] - pub fn with_head_injector(mut self, injector: Arc) -> Self { - self.registration.head_injectors.push(injector); + pub fn with_geo_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.geo_module = Some((name, module)); self } + /// Declare the Edge Cookie module this registration supplies, under its + /// `name`, which is also what the module's + /// [`id`](EdgeCookieModule::id) returns. + /// + /// The module only creates identifiers when `[ec] module` names it, and + /// a module the selector does not choose is logged as a warning when the + /// registry is built, the same as geo. #[must_use] - pub fn with_request_filter(mut self, filter: Arc) -> Self { - self.registration.request_filters.push(filter); + pub fn with_ec_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.ec_module = Some((name, module)); + self + } + + /// Declare the device module this registration supplies, under its + /// `name`. + /// + /// The module only classifies requests when `[device] module` names it, + /// and a module the selector does not choose is logged as a warning when + /// the registry is built, the same as geo. + #[must_use] + pub fn with_device_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.device_module = Some((name, module)); + self + } + + /// Declare the prefix module this registration supplies, with the name + /// `[prefix] module` selects it by. + /// + /// It only gives the deployment's prefixes when `[prefix] module` names + /// it, see [`crate::prefix`], and a module the selector does not choose is + /// logged as a warning when the registry is built, the same as geo. + #[must_use] + pub fn with_prefix_module(mut self, name: &'static str, module: Arc) -> Self { + self.registration.prefix_module = Some((name, module)); + self + } + + /// Declare the rules this registration adds to `/robots.txt`, under their + /// `name`. + /// + /// They are only asked for when `[robots-txt] modules` names them. Core + /// holds what they return and writes the file, see [`crate::robots_txt`]. + #[must_use] + pub fn with_robots_txt_contributor( + mut self, + name: &'static str, + contributor: Arc, + ) -> Self { + self.registration.robots_txt_contributor = Some((name, contributor)); self } @@ -706,6 +825,8 @@ impl IntegrationRegistrationBuilder { #[must_use] pub fn with_deferred_js(mut self) -> Self { self.registration.js_deferred = true; + self.registration.js_disabled = false; + self.registration.js_standalone = false; self } @@ -714,6 +835,28 @@ impl IntegrationRegistrationBuilder { pub fn without_js(mut self) -> Self { self.registration.js_disabled = true; self.registration.js_deferred = false; + self.registration.js_standalone = false; + self + } + + /// Carry a browser module built outside `trusted-server-js`. + #[must_use] + pub fn with_js_module(mut self, module: CarriedJsModule) -> Self { + self.registration.js_module = Some(module); + self + } + + /// Serve the module standalone only; see + /// [`IntegrationRegistration::js_standalone`]. + /// + /// The three delivery flags are exclusive and the last builder call wins, + /// so this clears the disabled and deferred flags as those methods clear + /// this one. + #[must_use] + pub fn with_standalone_js(mut self) -> Self { + self.registration.js_standalone = true; + self.registration.js_disabled = false; + self.registration.js_deferred = false; self } @@ -725,6 +868,10 @@ impl IntegrationRegistrationBuilder { type RouteValue = (Arc, &'static str); +/// Marks a request the preparers have run for. +#[derive(Clone, Copy)] +struct RequestPrepared; + struct IntegrationRegistryInner { // Method-specific routers for O(log n) lookups get_router: Router, @@ -737,14 +884,63 @@ struct IntegrationRegistryInner { // Metadata for introspection routes: Vec<(IntegrationEndpoint, &'static str)>, - enabled_integration_ids: Vec<&'static str>, + // Every builder considered at construction, named or not, in order. + builder_ids: Vec<(&'static str, &'static str)>, + running_integration_ids: Vec<&'static str>, deferred_js_ids: Vec<&'static str>, disabled_js_ids: Vec<&'static str>, - html_rewriters: Vec>, - script_rewriters: Vec>, - html_stream_processors: Vec>, - head_injectors: Vec>, + // Modules that run, served only on their own path, never in the bundle. + standalone_js_ids: Vec<&'static str>, + // Modules carried by their registrations, verified against their + // declared hash at construction. + carried_js: Vec<(&'static str, CarriedJsModule)>, request_filters: Vec>, + // The middleware the running registrations supply, each against the + // integration that supplied it, in registration order. + middleware: Vec<(&'static str, Arc)>, + // The attributes the running registrations put on the script bundle's + // tag, each against the integration that asked for it, in registration + // order. + bundle_tag_attributes: Vec<(&'static str, (&'static str, &'static str))>, + /// JS module IDs to include in the bundle that come from a source other than + /// a registered integration, for example a module tied to the selected Edge + /// Cookie module. Populated in [`IntegrationRegistry::new`] from settings. + extra_js_module_ids: Vec<&'static str>, + // Preparers from every builder, named or not, in registration order. + request_preparers: Vec, + // Finalizers from every builder, named or not, in registration order. + response_finalizers: Vec, + // Geo modules the running registrations supply, each with the name + // `[geo] module` selects it by, in registration order. Declaring one does + // not activate it. + geo_modules: Vec<(&'static str, Arc)>, + // Edge Cookie modules the running registrations supply, each with its + // name, in registration order. `[ec] module` picks at most one of them. + ec_modules: Vec<(&'static str, Arc)>, + // Device modules the running registrations supply, each with its name, in + // registration order. `[device] module` picks at most one of them. + device_modules: Vec<(&'static str, Arc)>, + // The robots.txt contributors the running registrations supply, each with + // its name, in registration order. `[robots-txt] modules` picks any of + // them, in its own order. + robots_txt_contributors: Vec<(&'static str, Arc)>, + // Every builder's module name, with whether a section selects it, so the + // refusal of a name a selector wrote can say what that name is. + builder_modules: Vec<(&'static str, bool)>, + // Vendors whose shared table is absent, for the refusal note. + absent_shared: Vec<&'static str>, + // The module `[geo] module` resolved to, or `None` when the selector + // is `platform` and the adapter's own host lookup stands. Unset and `none` + // both resolve the disabled module. + geo_module: Option>, + ec_module: Option>, + device_module: Option>, + // The prefix modules the running registrations supply, each with its + // name, in registration order. `[prefix] module` picks at most one. + prefix_modules: Vec<(&'static str, Arc)>, + // The prefix module `[prefix] module` resolved to, core's own when + // nothing is selected. + prefix_module: SelectedPrefixModule, } impl Default for IntegrationRegistryInner { @@ -758,16 +954,442 @@ impl Default for IntegrationRegistryInner { head_router: Router::new(), options_router: Router::new(), routes: Vec::new(), - enabled_integration_ids: Vec::new(), + builder_ids: Vec::new(), + running_integration_ids: Vec::new(), deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), - html_rewriters: Vec::new(), - script_rewriters: Vec::new(), - html_stream_processors: Vec::new(), - head_injectors: Vec::new(), + standalone_js_ids: Vec::new(), + carried_js: Vec::new(), request_filters: Vec::new(), + middleware: Vec::new(), + bundle_tag_attributes: Vec::new(), + extra_js_module_ids: Vec::new(), + request_preparers: Vec::new(), + response_finalizers: Vec::new(), + geo_modules: Vec::new(), + ec_modules: Vec::new(), + device_modules: Vec::new(), + robots_txt_contributors: Vec::new(), + builder_modules: Vec::new(), + absent_shared: Vec::new(), + geo_module: None, + ec_module: None, + device_module: None, + prefix_modules: Vec::new(), + prefix_module: SelectedPrefixModule::default(), + } + } +} + +/// Reserved value of `[geo] module` that resolves no location at all. +const GEO_MODULE_NONE: &str = "none"; + +/// `[geo] module` value opting in to the adapter's own host geo lookup. +const GEO_MODULE_PLATFORM: &str = "platform"; + +/// `[device] module` value naming the User-Agent-only module core supplies. +/// +/// The same choice as leaving the selector unset, spelled explicitly. +const DEVICE_MODULE_BUILTIN: &str = "builtin"; + +/// `[device] module` value opting in to the host module the adapter builds. +/// +/// Resolved by `build_device_module` in [`device`](crate::ec::device) rather +/// than by a module, so the registry supplies nothing for it. +const DEVICE_MODULE_FASTLY: &str = "fastly"; + +/// The type folder each of these kinds of module is selected in, which a name +/// written in its section may leave off. +const EDGECOOKIE_TYPE: &str = crate::ec::module::MODULE_TYPE; +const DEVICE_TYPE: &str = "device"; +const GEO_TYPE: &str = "geo"; +const PREFIX_TYPE: &str = crate::prefix::MODULE_TYPE; +const ROBOTS_TXT_TYPE: &str = crate::robots_txt::MODULE_TYPE; + +impl IntegrationRegistryInner { + /// The registered middleware an entry means by `name`. + fn middleware_named(&self, name: &str) -> Option<&Arc> { + self.middleware + .iter() + .map(|(_, middleware)| middleware) + .find(|middleware| middleware.middleware_id() == name) + } + + /// A sentence to follow the refusal of a middleware name an entry wrote, + /// when the name is that of a module no section selects, or begins with + /// one. Empty otherwise. + fn unselected_module_note(&self, name: &str) -> String { + self.builder_modules + .iter() + .find(|(module, selected)| { + !*selected + && name + .strip_prefix(*module) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('.')) + }) + .map(|(module, _)| { + format!( + ". `{module}` is a module no section selects, so nothing it supplies is \ + running" + ) + }) + .unwrap_or_default() + } + + /// A sentence to follow the refusal of a name a selector wrote, when the + /// name is a builder's module: one no section selects, which is why + /// nothing it supplies is running, or one that is selected and supplies no + /// module of this kind. Empty when no builder's module answers to the + /// name. + fn builder_note(&self, type_folder: &str, kind: &str, written: &str) -> String { + let names: Vec<&str> = self.builder_modules.iter().map(|(name, _)| *name).collect(); + let Some(name) = crate::module_name::resolve(type_folder, written, &names) else { + return String::new(); + }; + let selected = self + .builder_modules + .iter() + .any(|(module, selected)| *module == name && *selected); + if selected { + format!(". `{written}` is selected and supplies no {kind} module") + } else { + format!( + ". `{written}` is a module no section selects, so nothing it supplies is running" + ) + } + } + + /// A sentence naming the shared tables whose absence kept their vendors' + /// modules from running, to follow a refusal of a module nothing supplies, + /// or nothing when every builder a shared table runs is running. + fn absent_shared_note(&self) -> String { + if self.absent_shared.is_empty() { + return String::new(); + } + let tables = self + .absent_shared + .iter() + .map(|vendor| format!("[shared.{vendor}]")) + .collect::>() + .join(" or "); + let those = if self.absent_shared.len() == 1 { + "that table runs" + } else { + "those tables run" + }; + format!(". There is no {tables}, so the modules {those} are not running") + } +} + +/// The module a name written in a section means, among those the running +/// registrations supply, written in full or with the section's type folder +/// left off. +fn find_named<'a, T: ?Sized>( + type_folder: &str, + written: &str, + offered: &'a [(&'static str, Arc)], +) -> Option<&'a Arc> { + let names: Vec<&str> = offered.iter().map(|(name, _)| *name).collect(); + let name = crate::module_name::resolve(type_folder, written, &names)?; + offered + .iter() + .find(|(offered_name, _)| *offered_name == name) + .map(|(_, module)| module) +} + +/// Warns about each module on offer that the section does not select, so an +/// operator can see a capability the deployment never uses. +fn warn_unselected( + section: &str, + type_folder: &str, + selected: Option<&Arc>, + offered: &[(&'static str, Arc)], +) { + for (name, module) in offered { + if !selected.is_some_and(|chosen| Arc::ptr_eq(chosen, module)) { + log::warn!( + "the module `{}` is offered, and `[{section}] module` does not select it", + crate::module_name::short_form(type_folder, name) + ); + } + } +} + +/// The registered module a section names. +/// +/// # Errors +/// +/// A configuration error naming the section and the modules on offer when no +/// registered module answers to the name, because otherwise a mistyped name +/// would fall back to core's own module with nothing said. +fn named_module( + type_folder: &str, + section: &str, + written: &str, + offered: &[(&'static str, Arc)], + inner: &IntegrationRegistryInner, +) -> Result, Report> { + if let Some(module) = find_named(type_folder, written, offered) { + return Ok(Arc::clone(module)); + } + let names: Vec<&str> = offered + .iter() + .map(|(name, _)| crate::module_name::short_form(type_folder, name)) + .collect(); + let runs = if names.is_empty() { + format!("It runs no {section} module") + } else { + format!("The {section} modules it runs are [{}]", names.join(", ")) + }; + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`[{section}] module` names `{written}`, which no module this deployment runs \ + supplies. {runs}{}{}", + inner.builder_note(type_folder, section, written), + inner.absent_shared_note() + ), + })) +} + +/// Records that `integration` supplies the `kind` module `name`. +/// +/// # Errors +/// +/// When another registration already supplies a module of that kind under the +/// name, naming both, because a selector could then mean either. +fn claim_module_name( + claimed: &mut Vec<(&'static str, &'static str, &'static str)>, + kind: &'static str, + name: &'static str, + integration: &'static str, +) -> Result<(), Report> { + if let Some((_, _, first)) = claimed + .iter() + .find(|(claimed_kind, claimed_name, _)| *claimed_kind == kind && *claimed_name == name) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "the {kind} module `{name}` is declared twice, by integration `{first}` and by \ + integration `{integration}`" + ), + })); + } + claimed.push((kind, name, integration)); + Ok(()) +} + +/// Resolves the Edge Cookie module `[ec] module` names, when a registered +/// module supplies it. +/// +/// Core has Edge Cookie modules of its own, so a name no registered module +/// answers to is not an error here. This returns `Some` only for a registered +/// module, and core resolves the rest. The name looked for is the +/// implementation the selection's `[ec.]` block names, or the selection +/// itself, which is the name core checks the module's id against. +fn resolve_ec_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Option> { + // `None` spells statelessness and never names a module, so only a named + // selection can match one. + let implementation = match settings.ec.module.as_ref() { + Some(EcModuleSelection::Named(key)) => Some(settings.ec.module_blocks.implementation(key)), + Some(EcModuleSelection::None) | None => None, + }; + let resolved = implementation + .and_then(|name| find_named(EDGECOOKIE_TYPE, name, &inner.ec_modules)) + .map(Arc::clone); + warn_unselected("ec", EDGECOOKIE_TYPE, resolved.as_ref(), &inner.ec_modules); + resolved +} + +/// Resolves the device module `[device] module` names, when a registered +/// module supplies it. +/// +/// As with identity, core has a device module of its own, so a selector naming +/// it is not an error here. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the selector names no +/// registered module, or names one that declares a permission nothing can +/// enforce. +fn resolve_device_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result>, Report> { + let selector = settings.device.module.as_deref(); + let resolved = match selector { + // Unset and `builtin` both name the module core supplies itself, and + // `fastly` names the host module the adapter builds, so the registry + // supplies nothing for any of the three. + None | Some(DEVICE_MODULE_BUILTIN) | Some(DEVICE_MODULE_FASTLY) => None, + Some(written) => Some(named_module( + DEVICE_TYPE, + "device", + written, + &inner.device_modules, + inner, + )?), + }; + + // A module-supplied device module may declare the permissions its data + // use requires, but nothing enforces that declaration yet. Device + // classification runs before the permission set for the request is + // assembled, so there is no per-request gate to check it against. Selecting + // a module is an operator decision and not a per-request permission + // decision, so honoring the declaration by silently ignoring it would let a + // vendor state a requirement that never binds. Refuse the selection instead, + // loudly and at startup, until a real per-request device gate exists. + if let Some(module) = resolved.as_ref() + && module.required_permissions() != crate::permissions::PermissionSet::none() + { + let declared = module + .required_permissions() + .iter() + .map(crate::permissions::Permission::as_str) + .collect::>() + .join(", "); + let message = format!( + "`[device] module` selects `{}`, a device module requiring `{declared}`, and \ + Trusted Server has no per-request gate that can enforce that yet, so the \ + selection is refused rather than silently ignored", + selector.unwrap_or_default(), + ); + return Err(Report::new(TrustedServerError::Configuration { message })); + } + + warn_unselected( + "device", + DEVICE_TYPE, + resolved.as_ref(), + &inner.device_modules, + ); + + Ok(resolved) +} + +/// Resolves `[prefix] module` against the registered prefix modules, or to +/// core's own when nothing or `ts` is selected, and asks the module for the prefixes +/// of the publisher the settings describe, so a module that names something +/// its call never carries is refused here rather than on every request. +/// +/// # Errors +/// +/// A configuration error when the selector names no registered module, or +/// when the module names a value other than the request and the settings. +fn resolve_prefix_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result> { + let selected = match settings.prefix.module.as_deref() { + None | Some(crate::prefix::TRUSTED_SERVER_MODULE) => SelectedPrefixModule::trusted_server(), + Some(written) => { + let module = + named_module(PREFIX_TYPE, "prefix", written, &inner.prefix_modules, inner)?; + let name = inner + .prefix_modules + .iter() + .find(|(_, offered)| Arc::ptr_eq(offered, &module)) + .map_or(PREFIX_TYPE, |(name, _)| *name); + SelectedPrefixModule::new(name, module) + } + }; + let chosen = inner + .prefix_modules + .iter() + .find(|(name, _)| *name == selected.name()) + .map(|(_, module)| module); + warn_unselected("prefix", PREFIX_TYPE, chosen, &inner.prefix_modules); + let trial = ModuleRequest::new(&Method::GET, &settings.publisher.domain, "https", "/"); + selected.prefixes(settings, trial).map_err(|withheld| { + Report::new(TrustedServerError::Configuration { + message: format!( + "`[prefix] module` selects `{}`, which cannot give its prefixes: {withheld}. A \ + prefix module is handed the request and the settings, and nothing else", + selected.name() + ), + }) + })?; + Ok(selected) +} + +/// Refuses a `[robots-txt] modules` that names something nothing this +/// deployment runs supplies. +/// +/// Only knowable here, where every registration has been handed over, so it +/// is checked when the registry is built rather than when the settings load. +/// The file would otherwise fail on the first request for it rather than at +/// deploy. +fn check_robots_txt_selection( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result<(), Report> { + let Some(robots_txt) = settings.robots_txt.as_ref() else { + return Ok(()); + }; + for written in &robots_txt.modules { + if written == REFUSE_ALL + || written == ALLOW_ALL + || find_named(ROBOTS_TXT_TYPE, written, &inner.robots_txt_contributors).is_some() + { + continue; } + let offered: Vec<&str> = [REFUSE_ALL, ALLOW_ALL] + .into_iter() + .chain( + inner + .robots_txt_contributors + .iter() + .map(|(name, _)| crate::module_name::short_form(ROBOTS_TXT_TYPE, name)), + ) + .collect(); + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`[robots-txt] modules` names `{written}`, which nothing this deployment runs \ + supplies. What it can name is [{}]{}", + offered.join(", "), + inner.builder_note(ROBOTS_TXT_TYPE, "robots.txt", written) + ), + })); } + Ok(()) +} + +/// Resolves `[geo] module` against the registered geo modules. +/// +/// Returns the disabled module when the selector is unset or `none`, `None` +/// when it is `platform` so the adapter's own host lookup stands, and the +/// registered module otherwise. A module on offer that the selector does not +/// choose is logged as a warning when the registry is built. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the selector names no +/// registered module. +fn resolve_geo_module( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result>, Report> { + let selector = settings.geo.module.as_deref(); + let resolved = match selector { + // Unset resolves nothing and makes no host geo call, so a default + // deployment is not tied to any host geo service. `none` spells the + // same choice explicitly. + None | Some(GEO_MODULE_NONE) => Some(Arc::new(DisabledGeo) as Arc), + // `platform` opts in to the adapter's own host lookup, so the registry + // supplies nothing and the adapter's module stands. + Some(GEO_MODULE_PLATFORM) => None, + Some(written) => Some(named_module( + GEO_TYPE, + "geo", + written, + &inner.geo_modules, + inner, + )?), + }; + warn_unselected("geo", GEO_TYPE, resolved.as_ref(), &inner.geo_modules); + Ok(resolved) } /// Summary of registered integration capabilities. @@ -775,9 +1397,8 @@ impl Default for IntegrationRegistryInner { pub struct IntegrationMetadata { pub id: &'static str, pub routes: Vec, - pub attribute_rewriters: usize, - pub script_selectors: Vec<&'static str>, - pub head_injectors: usize, + /// The names of the middleware the integration supplies. + pub middleware: Vec<&'static str>, pub request_filters: usize, } @@ -786,9 +1407,7 @@ impl IntegrationMetadata { Self { id, routes: Vec::new(), - attribute_rewriters: 0, - script_selectors: Vec::new(), - head_injectors: 0, + middleware: Vec::new(), request_filters: 0, } } @@ -809,19 +1428,203 @@ pub struct ProxyDispatchInput<'a> { pub req: Request, } +/// Refuses a registered middleware no entry could name, and an entry this +/// deployment cannot run. +/// +/// A middleware no entry names is left alone and logged. +/// +/// # Errors +/// +/// Naming the module or the entry at fault. +fn check_middleware( + settings: &Settings, + inner: &IntegrationRegistryInner, +) -> Result<(), Report> { + let refuse = |message: String| Report::new(TrustedServerError::Configuration { message }); + let phases_of = |middleware: &Arc| -> Vec { + middleware + .phases() + .iter() + .map(|phase| format!("[[{phase}]]")) + .collect() + }; + for (position, (integration, middleware)) in inner.middleware.iter().enumerate() { + let id = middleware.middleware_id(); + if !crate::module_name::is_valid(id) { + return Err(refuse(format!( + "module `{integration}` supplies a middleware named `{id}`, which is not a \ + name an entry can write. A name is parts joined by `.`, each in lower case \ + letters, digits, `_` or `-`" + ))); + } + if middleware.phases().is_empty() { + return Err(refuse(format!( + "module `{integration}` supplies the middleware `{id}`, which says it runs \ + in no phase, so no entry could name it" + ))); + } + if let Some((earlier, _)) = inner.middleware[..position] + .iter() + .find(|(_, earlier)| earlier.middleware_id() == id) + { + return Err(refuse(format!( + "modules `{earlier}` and `{integration}` both supply a middleware named \ + `{id}`, so an entry could not say which one it means" + ))); + } + } + for phase in MiddlewarePhase::ALL { + for (index, entry) in settings.phase_entries(phase).entries().iter().enumerate() { + let at = format!("[[{phase}]] entry {}", index + 1); + for name in &entry.middleware { + let Some(middleware) = inner.middleware_named(name) else { + let supplied: Vec<&str> = inner + .middleware + .iter() + .map(|(_, middleware)| middleware.middleware_id()) + .collect(); + return Err(refuse(format!( + "{at} names `{name}`, which no module that runs supplies. The \ + middleware the running modules supply is [{}]{}", + supplied.join(", "), + inner.unselected_module_note(name) + ))); + }; + if !middleware.phases().contains(&phase) { + return Err(refuse(format!( + "{at} names `{name}`, which does not run in that phase. It runs in {}", + phases_of(middleware).join(" and ") + ))); + } + if !middleware.handles_media_type(&entry.media_type) { + return Err(refuse(format!( + "{at} covers `{}` and names `{name}`, which does not work on that media \ + type", + entry.media_type + ))); + } + } + } + } + for (integration, middleware) in unnamed_middleware(settings, inner) { + log::warn!( + "Module `{integration}` supplies the middleware `{}` and no {} entry names \ + it, so it changes no page", + middleware.middleware_id(), + phases_of(middleware).join(" or ") + ); + } + Ok(()) +} + +/// The middleware no entry names in a phase it runs in, each with the +/// integration that supplies it, in registration order. Core's own +/// middleware is offered to every deployment, so one that no entry names +/// is not a module running for nothing, and is left out. +fn unnamed_middleware<'a>( + settings: &Settings, + inner: &'a IntegrationRegistryInner, +) -> Vec<(&'static str, &'a Arc)> { + inner + .middleware + .iter() + .filter(|(integration, _)| *integration != crate::service_version::MODULE) + .filter(|(_, middleware)| { + let id = middleware.middleware_id(); + !middleware + .phases() + .iter() + .any(|phase| settings.phase_entries(*phase).names().contains(&id)) + }) + .map(|(integration, middleware)| (*integration, middleware)) + .collect() +} + /// In-memory registry of integrations discovered from settings. #[derive(Clone, Default)] pub struct IntegrationRegistry { inner: Arc, plan: Option>, + /// What the adapter reports about the platform the deployment runs on, + /// which a middleware is told about every document. + platform: Arc, +} + +/// Refuses a name a section selects that no module in this deployment +/// supplies, listing what the section could select instead. +/// +/// # Errors +/// +/// Naming the first section and name at fault. +fn check_section_selections( + settings: &Settings, + extra: &[crate::integrations::IntegrationBuilder], + inner: &IntegrationRegistryInner, +) -> Result<(), Report> { + let offered: Vec<&'static str> = crate::integrations::all_builders(extra) + .filter_map(|builder| builder.module_name()) + .collect(); + for (section, modules) in settings.module_sections() { + for written in modules.selected() { + if crate::module_name::resolve(section, written, &offered).is_some() { + continue; + } + let of_type: Vec<&str> = offered + .iter() + .map(|name| crate::module_name::short_form(section, name)) + .filter(|short| !offered.contains(short)) + .collect(); + let supplied = if of_type.is_empty() { + let mut types: Vec<&str> = offered + .iter() + .filter_map(|name| name.split_once('.').map(|(folder, _)| folder)) + .collect(); + types.sort_unstable(); + types.dedup(); + format!( + "It supplies no {section} module. The types it supplies modules of are [{}]", + types.join(", ") + ) + } else { + format!( + "The {section} modules it supplies are [{}]", + of_type.join(", ") + ) + }; + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{section}] selects `{written}`, which no module this deployment \ + supplies. {supplied}{}", + inner.absent_shared_note() + ), + })); + } + } + Ok(()) } impl IntegrationRegistry { - /// Build a registry and auction plan from the provided settings for tests. - /// - /// Runtime adapters should compile one plan and pass it to [`Self::with_plan`]. - /// - /// # Errors + /// Tells the registry what the adapter reports about the platform, which + /// a middleware is told about every document. + #[must_use] + pub fn with_platform(mut self, platform: PlatformIdentity) -> Self { + self.platform = Arc::new(platform); + self + } + + /// What the adapter reports about the platform the deployment runs on. + #[must_use] + pub fn platform(&self) -> &PlatformIdentity { + &self.platform + } + + /// Build a registry and auction plan from the provided settings for tests. + /// + /// Runtime adapters should compile one plan and pass it to + /// [`Self::with_plan`], or to [`Self::with_plan_and_registrations`] when + /// they also supply integrations of their own. + /// + /// # Errors /// /// Returns an error if the auction plan or integration registry is invalid. #[cfg(test)] @@ -830,41 +1633,196 @@ impl IntegrationRegistry { Self::with_plan(settings, plan) } - /// Build a registry from the provided settings. + /// Build a registry from the built-in integrations followed by `extra`, + /// compiling the auction plan from `settings`, for tests. + /// + /// Runtime adapters compile one plan and pass it to + /// [`Self::with_plan_and_registrations`], so the plan is shared rather than + /// compiled twice. + /// + /// # Errors + /// + /// Returns an error if the auction plan or the integration registry is + /// invalid. + #[cfg(any(test, feature = "test-utils"))] + pub fn with_registrations( + settings: &Settings, + extra: &[crate::integrations::IntegrationBuilder], + ) -> Result> { + let plan = Arc::new(crate::auction::compile_auction_plan_with(settings, extra)?); + Self::with_plan_and_registrations(settings, plan, extra) + } + + /// Build a registry from the built-in integrations, against one shared + /// compiled auction plan. + /// + /// # Errors + /// + /// Returns an error if route registration fails due to duplicate routes or + /// invalid paths, or when a registration carries a browser module whose + /// declared SHA-256 does not match its source. + pub fn with_plan( + settings: &Settings, + plan: Arc, + ) -> Result> { + Self::with_plan_and_registrations(settings, plan, &[]) + } + + /// Build a registry from the built-in integrations followed by `extra`, + /// the builders an adapter or a vendor crate supplies, against one shared + /// compiled auction plan. + /// + /// The plan-backed auction providers, Prebid then APS, register before the + /// builders, so opening the builder table changes no existing hook order. + /// Their ids are reserved for core whether or not they run, so an + /// outside builder claiming either is refused the way two builders claiming + /// one id are. /// /// # Errors /// - /// Returns an error if route registration fails due to duplicate routes or invalid paths. + /// Returns an error when two builders claim the same integration id, when + /// route registration fails due to duplicate routes or invalid paths, when + /// a builder fails, or when a registration carries a browser module whose + /// declared SHA-256 does not match its source. /// /// # Panics /// /// Panics if a route path ends with `/*` but `strip_suffix` unexpectedly fails (invariant violation). - pub fn with_plan( + pub fn with_plan_and_registrations( settings: &Settings, plan: Arc, + extra: &[crate::integrations::IntegrationBuilder], ) -> Result> { let mut inner = IntegrationRegistryInner::default(); - let mut registrations = Vec::new(); - if let Some(registration) = crate::integrations::prebid::register_for_plan(settings, &plan)? - { - registrations.push(registration); - } - if let Some(registration) = crate::integrations::aps::register_for_plan(settings, &plan)? { - registrations.push(registration); + // What a builder registers from the auction plan goes first, so its + // hooks run ahead of every module a section selects. + let mut plan_registrations: Vec = Vec::new(); + let mut registrations: Vec = Vec::new(); + + // A vendor's shared table is read by that vendor's builder alone, so + // one no builder reads is a table the deployment cannot use. + let unread: Vec<&str> = settings + .shared + .vendors() + .filter(|vendor| { + !crate::integrations::all_builders(extra) + .any(|builder| builder.shared_settings() == Some(*vendor)) + }) + .collect(); + if !unread.is_empty() { + let read: Vec<&str> = crate::integrations::all_builders(extra) + .filter_map(|builder| builder.shared_settings()) + .collect(); + let reads = if read.is_empty() { + "It reads none".to_owned() + } else { + format!("The shared tables it reads are [{}]", read.join(", ")) + }; + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[shared.{}] is read by no module this deployment supplies. {reads}", + unread.join("] and [shared."), + ), + })); } - for builder in crate::integrations::builders() { - if let Some(registration) = (builder.build)(settings)? { + + for builder in crate::integrations::all_builders(extra) { + if let Some((_, first_source)) = + inner.builder_ids.iter().find(|(id, _)| *id == builder.id()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration id `{}` is registered twice, by `{first_source}` and by `{}`", + builder.id(), + builder.source() + ), + })); + } + inner.builder_ids.push((builder.id(), builder.source())); + // A builder its shared table runs answers to no section, so a + // module name would let a section select what it does not run. + if let (Some(vendor), Some(name)) = (builder.shared_settings(), builder.module_name()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` from `{}` is run by [shared.{vendor}] and also names \ + the module `{name}`. A builder its shared table runs is selected by \ + no section, so it takes no module name", + builder.id(), + builder.source() + ), + })); + } + // Preparers are collected before the selection check, so an + // integration can sanitize its own reserved query or cookie in a + // deployment that does not run it. + if let Some(prepare) = builder.prepare_request() { + inner.request_preparers.push(prepare); + } + if let Some(finalize) = builder.finalize_response() { + inner.response_finalizers.push(finalize); + } + // A registration from the plan is made whatever the sections + // select, because the function decides from the plan and the + // settings whether the module runs. + if let Some(register) = builder.plan_registration() + && let Some(registration) = register(settings, &plan)? + { + debug_assert_eq!( + registration.integration_id, + builder.id(), + "integration builder ID should match registration ID" + ); + plan_registrations.push(registration); + } + + // Only a builder whose module a section selects is built, so an + // integration runs exactly when an operator selects it, whatever + // its builder would otherwise make of the settings. + let selected = builder + .module_name() + .is_some_and(|name| settings.selects_module(name)); + if let Some(name) = builder.module_name() { + inner.builder_modules.push((name, selected)); + } + // A builder a vendor's shared table runs is built when the table + // is present, whatever any section selects. + let runs = match builder.shared_settings() { + Some(vendor) => { + let present = settings.shared.contains(vendor); + if !present && !inner.absent_shared.contains(&vendor) { + inner.absent_shared.push(vendor); + } + present + } + None => selected, + }; + if !runs { + continue; + } + + if let Some(registration) = builder.build(settings)? { debug_assert_eq!( - registration.integration_id, builder.id, + registration.integration_id, + builder.id(), "integration builder ID should match registration ID" ); registrations.push(registration); } } - for registration in registrations { + // Which names a section can select is only knowable here, where the + // adapter's and a vendor crate's builders have been handed over, so + // the selections are checked against them rather than in the + // settings. Deploy validation deliberately does not make this check, + // because a vendor crate the CLI never links may supply the name. + check_section_selections(settings, extra, &inner)?; + + // The geo, Edge Cookie and device module names taken so far, with the + // integration that supplies each. + let mut claimed = Vec::new(); + for registration in plan_registrations.into_iter().chain(registrations) { inner - .enabled_integration_ids + .running_integration_ids .push(registration.integration_id); for proxy in registration.proxies { @@ -915,25 +1873,154 @@ impl IntegrationRegistry { inner.routes.push((route, registration.integration_id)); } } - inner - .html_rewriters - .extend(registration.attribute_rewriters); - inner.script_rewriters.extend(registration.script_rewriters); - inner - .html_stream_processors - .extend(registration.html_stream_processors); - inner.head_injectors.extend(registration.head_injectors); inner.request_filters.extend(registration.request_filters); + for middleware in registration.middleware { + inner + .middleware + .push((registration.integration_id, middleware)); + } + for (name, value) in registration.bundle_tag_attributes { + // The tag is written as markup with no escaping, so a name + // or a value that could end the attribute is refused. + let name_is_plain = !name.is_empty() + && name.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-' + }); + let value_is_plain = !value + .bytes() + .any(|byte| matches!(byte, b'"' | b'&' | b'<' | b'>')); + if !name_is_plain || !value_is_plain { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` puts the attribute `{name}` on the script \ + bundle's tag with the value `{value}`. A name is lower case \ + letters, digits and hyphens, and a value holds none of `\"`, `&`, \ + `<` and `>`", + registration.integration_id + ), + })); + } + inner + .bundle_tag_attributes + .push((registration.integration_id, (name, value))); + } + if let Some((name, module)) = registration.geo_module { + claim_module_name(&mut claimed, "geo", name, registration.integration_id)?; + inner.geo_modules.push((name, module)); + } + if let Some((name, module)) = registration.ec_module { + claim_module_name( + &mut claimed, + "Edge Cookie", + name, + registration.integration_id, + )?; + // Core checks the selected module by its own id, so a module + // declared under another name would be selected here and + // refused there. + if module.id() != name { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "integration `{}` declares an Edge Cookie module under `{name}`, \ + and the module's own id is `{}`. The two must be the same name", + registration.integration_id, + module.id() + ), + })); + } + inner.ec_modules.push((name, module)); + } + if let Some((name, module)) = registration.device_module { + claim_module_name(&mut claimed, "device", name, registration.integration_id)?; + inner.device_modules.push((name, module)); + } + if let Some((name, module)) = registration.prefix_module { + claim_module_name(&mut claimed, "prefix", name, registration.integration_id)?; + inner.prefix_modules.push((name, module)); + } + if let Some((name, contributor)) = registration.robots_txt_contributor { + claim_module_name( + &mut claimed, + "robots.txt", + name, + registration.integration_id, + )?; + inner.robots_txt_contributors.push((name, contributor)); + } if registration.js_disabled { inner.disabled_js_ids.push(registration.integration_id); } else if registration.js_deferred { inner.deferred_js_ids.push(registration.integration_id); } + if registration.js_standalone { + inner.standalone_js_ids.push(registration.integration_id); + } + if let Some(module) = registration.js_module { + // The served `?v=` hash and its memo trust this value, so a + // stale literal is a startup error rather than a stale + // script. The cost is one SHA-256 of each carried module + // per registry build, and only the Axum dev server builds + // the registry once, because its `main` builds the router + // before serving. Fastly starts a fresh Wasm instance per + // request, and `edgezero_adapter_cloudflare::run_app` and + // `edgezero_adapter_spin::run_app` both call `build_app` + // inside the per-request entry point, so those three pay + // it on every request. + let actual = hex::encode(Sha256::digest(module.source.as_bytes())); + if actual != module.sha256 { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration `{}` carries a browser module whose declared SHA-256 does not match its source (declared {}, actual {actual})", + registration.integration_id, module.sha256 + ), + })); + } + inner.carried_js.push((registration.integration_id, module)); + } + } + + // A client-cycle Edge Cookie module ships a page script that posts its + // result to the resolve endpoint. The script rides the tsjs bundle, so + // include its module when that module is selected. The same module + // list drives both the served bundle and the injected `")], + after_bundle_inserts: vec![format!( + "", + crate::tsjs::tsjs_single_module_script_src(&module) + )], + ..MiddlewareAction::pass() + } + } } - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - Vec::new() + /// Has `settings` select the stand-in and run its middleware, and no + /// other, on every reader's copy of a page. + pub(crate) fn select_and_place(settings: &mut Settings) { + settings.select_module("testing", MODULE); + crate::html_processor::test_support::place_on_every_page( + settings, + MiddlewarePhase::Serve, + &[MODULE], + ); } - fn tsjs_script_tag_attributes(&self) -> Vec<(&'static str, &'static str)> { - vec![ - ("data-ts-gam-attribution", "false"), - ("data-third-attribute", "third"), - ] + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(_config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .with_js_module(CarriedJsModule { + source: JS, + sha256: JS_SHA256, + }) + .with_standalone_js() + .with_middleware(Arc::new(Head)) + .with_request_filter(Arc::new(Filter)) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) } } - #[test] - fn tsjs_script_tag_attributes_preserve_registration_order_and_default_empty() { - let registry = IntegrationRegistry::from_rewriters_with_head_injectors( - Vec::new(), - Vec::new(), - vec![ - Arc::new(DefaultMetadataHeadInjector), - Arc::new(StaticMetadataHeadInjector), - Arc::new(ConflictingMetadataHeadInjector), - ], - ); + /// A stand-in for an integration whose browser module loads deferred, for + /// core's own tests of how modules are divided between the bundle and + /// deferred tags, and of how a module's settings reach a page template. + /// + /// Selected, it carries a deferred browser module and supplies one fetch + /// middleware under its own name, which writes its two settings into + /// ``. + pub(crate) mod deferred_fixture { + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{CarriedJsModule, IntegrationRegistration}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase}; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "deferred_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.deferred-fixture"; + /// The file of the stand-in's deferred module. + pub(crate) const MODULE_FILE: &str = "tsjs-deferred_fixture.min.js"; + + const JS: &str = "(function(){window.__ts_deferred_fixture_loaded=1;})();"; + // SHA-256 of JS, hex. The registry refuses a literal that is not. + const JS_SHA256: &str = "3947bc5e93ffb5d057f0d500d20ee25d53612b6c8b59cd3f9512c1b64b45f9f6"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + label: String, + #[serde(default)] + timeout_ms: u32, + } - assert_eq!( - registry.tsjs_script_tag_attributes(), - vec![ - ("data-ts-gam-attribution", "true"), - ("data-test-order", "second"), - ("data-third-attribute", "third"), - ], - "should keep the first value for duplicate names and preserve attribute order" - ); - } + impl crate::settings::IntegrationConfig for FixtureSettings {} - // Mock integration proxy for testing - struct MockProxy; + struct Head { + label: String, + timeout_ms: u32, + } - #[async_trait(?Send)] - impl IntegrationProxy for MockProxy { - fn integration_name(&self) -> &'static str { - "test" + impl Middleware for Head { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let config = serde_json::json!({ + "label": self.label, + "timeoutMs": self.timeout_ms, + }); + MiddlewareAction { + head_inserts: vec![format!( + "" + )], + ..MiddlewareAction::pass() + } + } } - fn routes(&self) -> Vec { - vec![] + /// Has `settings` run the stand-in's middleware, and no other, on + /// every page. + pub(crate) fn place(settings: &mut Settings) { + crate::html_processor::test_support::place_on_every_page( + settings, + MiddlewarePhase::Fetch, + &[MODULE], + ); } - async fn handle( - &self, - _settings: &Settings, - _services: &RuntimeServices, - _req: Request, - ) -> Result, Report> { - Ok(Response::new(EdgeBody::empty())) + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .with_js_module(CarriedJsModule { + source: JS, + sha256: JS_SHA256, + }) + .with_deferred_js() + .with_middleware(Arc::new(Head { + label: config.label, + timeout_ms: config.timeout_ms, + })) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) } } - struct EnrichingRequestFilter; - #[derive(Clone, Copy)] - struct RequestAnnotation; + /// A stand-in for a module that changes a page through middleware, for + /// core's own tests of the entries and of the page path. + /// + /// Selected, it supplies three fetch middleware. The one under the + /// module's own name writes a marker carrying the module's `label` + /// setting at the start of ``. The one named `.links` points every + /// link that carries `data-fixture` at a fixed path. The one named + /// `.broken` asks for a selector that does not parse. + /// + /// It supplies two serve middleware as well. The one named `.reader` + /// writes a marker ahead of the script bundle, saying which origin it was + /// told of and whether the request stand-in left its mark, and a script + /// straight after the bundle. The one named `.broken-reader` asks for a + /// selector that does not parse. + pub(crate) mod middleware_fixture { + use std::rc::Rc; + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{AttributeRewriteAction, IntegrationRegistration}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{ + AttributeRewrite, Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, + Preparation, Refusal, + }; + use crate::module_context::ModuleCall; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "middleware_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.middleware-fixture"; + /// The name of the middleware that marks the head. + pub(crate) const HEAD: &str = MODULE; + /// The name of the middleware that moves links. + pub(crate) const LINKS: &str = "testing.middleware-fixture.links"; + /// The name of the middleware whose selector does not parse. + pub(crate) const BROKEN: &str = "testing.middleware-fixture.broken"; + /// The name of the serve middleware that marks a reader's copy. + pub(crate) const READER: &str = "testing.middleware-fixture.reader"; + /// The name of the serve middleware whose selector does not parse. + pub(crate) const BROKEN_READER: &str = "testing.middleware-fixture.broken-reader"; + /// What the reader middleware writes straight after the bundle. + pub(crate) const READER_SCRIPT: &str = ""; + /// Where the links middleware points a link. + pub(crate) const LINK_TARGET: &str = "/fixture/link"; + /// The selector the broken middleware asks for. + pub(crate) const BROKEN_SELECTOR: &str = "a["; + /// The name of the fetch middleware that works on `text/plain`. + pub(crate) const TEXT: &str = "testing.middleware-fixture.text"; + /// The media type the text middleware works on. + pub(crate) const TEXT_MEDIA_TYPE: &str = "text/plain"; + /// The line the text middleware adds at the end of a text body. + pub(crate) const TEXT_LINE: &str = "# added by the middleware fixture\n"; + /// The name of the fetch middleware that prepares. + pub(crate) const PREPARING: &str = "testing.middleware-fixture.preparing"; + /// The name of the fetch middleware that refuses every request. + pub(crate) const REFUSING: &str = "testing.middleware-fixture.refusing"; + /// How long the refusing middleware asks the caller to wait. + pub(crate) const REFUSAL_RETRY_AFTER: std::time::Duration = + std::time::Duration::from_secs(30); + /// What the refusing middleware says. + pub(crate) const REFUSAL_MESSAGE: &str = "the middleware fixture refuses"; + /// What the preparing middleware writes when nothing was prepared. + pub(crate) const NOTHING_PREPARED: &str = + ""; + + /// The marker the preparing middleware writes, naming the module it + /// was called as and the host it was told of. + pub(crate) fn prepared_marker(module: &str, request_host: &str) -> String { + format!( + "" + ) + } - #[async_trait(?Send)] - impl IntegrationRequestFilter for EnrichingRequestFilter { - fn integration_id(&self) -> &'static str { - "enriching" + struct Preparing; + + impl Middleware for Preparing { + fn middleware_id(&self) -> &'static str { + PREPARING + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let marker = prepared_marker(call.module(), context.request_host); + Ok(Box::new(marker) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let marker: String = context + .take_prepared(PREPARING) + .unwrap_or_else(|| NOTHING_PREPARED.to_owned()); + MiddlewareAction { + head_inserts: vec![marker], + ..MiddlewareAction::pass() + } + } } - async fn filter_request( - &self, - input: RequestFilterInput<'_>, - ) -> Result> { - input.request.extensions_mut().insert(RequestAnnotation); - Ok(RequestFilterDecision::Continue(RequestFilterEffects { - request_headers: vec![HeaderMutation::set("x-datadome-isbot", "1")], - response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], - })) + struct Refusing; + + impl Middleware for Refusing { + fn middleware_id(&self) -> &'static str { + REFUSING + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn prepare<'a>( + &'a self, + _call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async { + Err(Refusal::service_unavailable( + REFUSAL_RETRY_AFTER, + REFUSAL_MESSAGE, + )) + })) + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + } + /// The name of the serve middleware that writes a header on a text + /// response. + pub(crate) const TEXT_HEADER: &str = "testing.middleware-fixture.text-header"; + /// The header the text header middleware writes, name and value. + pub(crate) const TEXT_HEADER_NAME: &str = "x-fixture-text"; + pub(crate) const TEXT_HEADER_VALUE: &str = "served"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + label: String, } - } - struct EchoProxy; + impl crate::settings::IntegrationConfig for FixtureSettings {} - #[async_trait(?Send)] - impl IntegrationProxy for EchoProxy { - fn integration_name(&self) -> &'static str { - "echo" + /// The marker the head middleware writes for `label`. + pub(crate) fn head_marker(label: &str) -> String { + format!("") } - fn routes(&self) -> Vec { - vec![] + struct Head { + label: String, } - async fn handle( - &self, - _settings: &Settings, - _services: &RuntimeServices, - req: http::Request, - ) -> Result, Report> { - let response = http::Response::builder() - .status(http::StatusCode::OK) - .header("x-echo-path", req.uri().path()) - .body(EdgeBody::empty()) - .expect("should build echo response"); - Ok(response) + impl Middleware for Head { + fn middleware_id(&self) -> &'static str { + HEAD + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + head_inserts: vec![head_marker(&self.label)], + ..MiddlewareAction::pass() + } + } } - } - #[test] - fn document_state_keeps_multiple_types_for_one_integration() { - let state = IntegrationDocumentState::default(); - let number = state.get_or_insert_with("test", || 7_u32); - let label = state.get_or_insert_with("test", || "first".to_string()); - let repeated_number = state.get_or_insert_with("test", || 99_u32); + /// The marker the reader middleware writes when told of `origin_host`, + /// for a request the request stand-in marked or did not. + pub(crate) fn reader_marker(origin_host: &str, marked: bool) -> String { + format!( + "" + ) + } - assert!( - Arc::ptr_eq(&number, &repeated_number), - "repeated insertion should preserve the original typed state" - ); - assert_eq!( - *state.get::("test").expect("should retrieve number"), - 7, - "should retain numeric state" - ); - assert_eq!( - state - .get::("test") - .expect("should retrieve label") - .as_str(), - "first", - "should retain string state under the same integration ID" - ); - assert_eq!( - label.as_str(), - "first", - "should return inserted string state" - ); - } + struct Reader; - struct CountingStreamFactory(&'static str); + impl Middleware for Reader { + fn middleware_id(&self) -> &'static str { + READER + } - impl IntegrationHtmlStreamProcessorFactory for CountingStreamFactory { - fn integration_id(&self) -> &'static str { - self.0 - } - - fn create(&self, _context: IntegrationHtmlStreamContext) -> Box { - struct CountingStreamProcessor(usize); - - impl StreamProcessor for CountingStreamProcessor { - fn process_chunk( - &mut self, - chunk: &[u8], - _is_last: bool, - ) -> std::io::Result> { - self.0 += 1; - let mut output = self.0.to_string().into_bytes(); - output.extend_from_slice(chunk); - Ok(output) + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + use super::request_fixture; + + let marked = context + .document_state + .get::(request_fixture::ID) + .is_some(); + MiddlewareAction { + head_inserts: vec![reader_marker(context.origin_host, marked)], + after_bundle_inserts: vec![READER_SCRIPT.to_owned()], + ..MiddlewareAction::pass() } } + } + + struct BrokenReader; + + impl Middleware for BrokenReader { + fn middleware_id(&self) -> &'static str { + BROKEN_READER + } - Box::new(CountingStreamProcessor(0)) + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + BROKEN_SELECTOR, + "href", + Rc::new(|_matched| AttributeRewriteAction::keep()), + ))], + ..MiddlewareAction::pass() + } + } } - } - #[test] - fn html_stream_factories_preserve_order_and_create_isolated_sessions() { - let registration = IntegrationRegistration::builder("test") - .with_html_stream_processor(Arc::new(CountingStreamFactory("first"))) - .with_html_stream_processor(Arc::new(CountingStreamFactory("second"))) - .build(); - let identifiers: Vec<_> = registration - .html_stream_processors - .iter() - .map(|factory| factory.integration_id()) - .collect(); - assert_eq!( - identifiers, - ["first", "second"], - "should preserve factory registration order", - ); + struct Links; - let context = IntegrationHtmlStreamContext { - request_host: "proxy.example.com".to_owned(), - request_scheme: "https".to_owned(), - origin_host: "origin.example.com".to_owned(), - document_state: IntegrationDocumentState::default(), - }; - let factory = ®istration.html_stream_processors[0]; - let mut first = factory.create(context.clone()); - let mut second = factory.create(context); + impl Middleware for Links { + fn middleware_id(&self) -> &'static str { + LINKS + } - assert_eq!( - first - .process_chunk(b"a", false) - .expect("should process first session"), - b"1a", - "should initialize the first session counter", - ); - assert_eq!( - second - .process_chunk(b"b", true) - .expect("should process second session"), - b"1b", - "should initialize an independent second session counter", - ); - } + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } - #[test] - fn handle_proxy_passes_http_request_without_fastly_round_trip() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::from_routes(vec![( - http::Method::GET, - "/integrations/test/echo", - (Arc::new(EchoProxy) as Arc, "echo"), - )]); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://test.example.com/integrations/test/echo?x=1") - .body(EdgeBody::empty()) - .expect("should build request"); + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + "a[data-fixture]", + "href", + Rc::new(|_matched| AttributeRewriteAction::replace(LINK_TARGET)), + ))], + ..MiddlewareAction::pass() + } + } + } - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &http::Method::GET, - path: "/integrations/test/echo", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &noop_services(), - req, - })) - .expect("should match route") - .expect("proxy should succeed"); + /// Passes a text body through and adds a line at its end. + struct AppendLine; + + impl crate::streaming_processor::StreamProcessor for AppendLine { + fn process_chunk( + &mut self, + chunk: &[u8], + is_last: bool, + ) -> Result, std::io::Error> { + let mut output = chunk.to_vec(); + if is_last { + output.extend_from_slice(TEXT_LINE.as_bytes()); + } + Ok(output) + } + } - assert_eq!( + struct Text; + + impl Middleware for Text { + fn middleware_id(&self) -> &'static str { + TEXT + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == TEXT_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLine)), + ..MiddlewareAction::pass() + } + } + } + + struct TextHeader; + + impl Middleware for TextHeader { + fn middleware_id(&self) -> &'static str { + TEXT_HEADER + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == TEXT_MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + response_headers: vec![( + http::HeaderName::from_static(TEXT_HEADER_NAME), + http::HeaderValue::from_static(TEXT_HEADER_VALUE), + )], + ..MiddlewareAction::pass() + } + } + } + + struct Broken; + + impl Middleware for Broken { + fn middleware_id(&self) -> &'static str { + BROKEN + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + BROKEN_SELECTOR, + "href", + Rc::new(|_matched| AttributeRewriteAction::keep()), + ))], + ..MiddlewareAction::pass() + } + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .without_js() + .with_middleware(Arc::new(Head { + label: config.label, + })) + .with_middleware(Arc::new(Links)) + .with_middleware(Arc::new(Broken)) + .with_middleware(Arc::new(Reader)) + .with_middleware(Arc::new(BrokenReader)) + .with_middleware(Arc::new(Text)) + .with_middleware(Arc::new(Preparing)) + .with_middleware(Arc::new(Refusing)) + .with_middleware(Arc::new(TextHeader)) + .build(), + )) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } + + /// A stand-in for an integration that tags a page, for core's own tests + /// of the HTML processor and the JavaScript asset proxy. + /// + /// Selected, it supplies one fetch middleware under its own name, which + /// writes one script in `` ahead of the bundle and rewrites the + /// address of its own script to a first-party path. With `mark_bundle` + /// set it also asks for an attribute on the publisher bundle tag. + pub(crate) mod tag_fixture { + use std::rc::Rc; + use std::sync::Arc; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{AttributeRewriteAction, IntegrationRegistration}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, MiddlewareContext, + MiddlewarePhase, + }; + use crate::settings::Settings; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "tag_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.tag-fixture"; + /// The script the stand-in's vendor would serve. + pub(crate) const SCRIPT_URL: &str = "https://cdn.tag-fixture.example/sdk.js"; + /// The first-party path the stand-in rewrites that script to. + pub(crate) const FIRST_PARTY_SCRIPT: &str = "/integrations/tag_fixture/script"; + /// What the stand-in's head insert sets, so a test can find it. + pub(crate) const HEAD_FLAG: &str = "window.__ts_tag_fixture=true;"; + /// The attribute the stand-in asks for on the publisher bundle tag. + pub(crate) const BUNDLE_ATTRIBUTE: &str = "data-ts-tag-fixture"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings { + #[serde(default)] + mark_bundle: bool, + } + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + struct Tag; + + impl Middleware for Tag { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let decide: Rc = Rc::new(|matched| { + if matched.value == SCRIPT_URL { + AttributeRewriteAction::Replace(FIRST_PARTY_SCRIPT.to_owned()) + } else { + AttributeRewriteAction::Keep + } + }); + MiddlewareAction { + head_inserts: vec![format!("")], + element_handlers: vec![Box::new(AttributeRewrite::new("src", decide))], + ..MiddlewareAction::pass() + } + } + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + let Some(config) = settings.module_config::(MODULE)? else { + return Ok(None); + }; + let mut registration = IntegrationRegistration::builder(ID) + .without_js() + .with_middleware(Arc::new(Tag)); + if config.mark_bundle { + registration = registration.with_bundle_tag_attribute(BUNDLE_ATTRIBUTE, "true"); + } + Ok(Some(registration.build())) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } + + /// A stand-in for an integration that rewrites script payloads in two + /// passes, for core's own tests of the page pipeline. + /// + /// Selected, it supplies one fetch middleware under its own name. Its + /// text handler swaps the payload of each `fixture_payload("...")` call + /// for a placeholder that carries a namespace made per document, and + /// keeps the payload. Its stream processor swaps each placeholder back + /// for its payload with the origin host rewritten. A payload pushed with + /// `fixture_payload_open` leaves its group unresolved, and the processor + /// holds its output from that placeholder on until a + /// `fixture_payload_close` arrives. + pub(crate) mod payload_fixture { + use std::io; + use std::sync::{Arc, Mutex, PoisonError}; + + use error_stack::Report; + + use crate::error::TrustedServerError; + use crate::integrations::registry::{IntegrationRegistration, ScriptRewriteAction}; + use crate::integrations::{CORE_SOURCE, IntegrationBuilder}; + use crate::middleware::{ + Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, TextHandler, + }; + use crate::settings::Settings; + use crate::streaming_processor::StreamProcessor; + + /// The integration id the stand-in registers under. + pub(crate) const ID: &str = "payload_fixture"; + /// The name a test's settings select the stand-in by, in `[testing]`. + pub(crate) const MODULE: &str = "testing.payload-fixture"; + /// What every placeholder starts with, so a test can check that none + /// reaches a reader. + pub(crate) const PLACEHOLDER_PREFIX: &str = "__ts_fixture_"; + const PLACEHOLDER_END: &str = "__"; + + /// The builder core's test build lists beside its own. + pub(crate) const BUILDER: IntegrationBuilder = + IntegrationBuilder::new(ID, CORE_SOURCE, register, validate).with_module_name(MODULE); + + struct Payload { + placeholder: String, + original: String, + /// Whether the group this payload opened is still waiting for its + /// close. + unresolved: bool, + } + + /// What one document's text handler has captured so far. + struct Captured { + namespace: String, + payloads: Vec, + } + + impl Default for Captured { + fn default() -> Self { + Self { + namespace: uuid::Uuid::new_v4().simple().to_string(), + payloads: Vec::new(), + } + } + } + + /// The scripts of one document, each held until its last chunk. + struct Scripts { + captured: Arc>, + held: String, + } + + impl Scripts { + /// Swaps the payload of a whole script for a placeholder, or + /// leaves a script that pushes no payload as it is. + fn rewrite_whole(&self, script: &str) -> Option { + let (call, opens, closes) = [ + ("fixture_payload_open(\"", true, false), + ("fixture_payload_close(\"", false, true), + ("fixture_payload(\"", false, false), + ] + .into_iter() + .find(|(call, _, _)| script.contains(*call))?; + let start = script.find(call)? + call.len(); + let end = script.rfind("\")")?; + if end < start { + return None; + } + + let mut captured = self.captured.lock().unwrap_or_else(PoisonError::into_inner); + let placeholder = format!( + "{PLACEHOLDER_PREFIX}{}_{}{PLACEHOLDER_END}", + captured.namespace, + captured.payloads.len() + ); + if closes { + for payload in &mut captured.payloads { + payload.unresolved = false; + } + } + captured.payloads.push(Payload { + placeholder: placeholder.clone(), + original: script[start..end].to_owned(), + unresolved: opens, + }); + + let mut rewritten = script.to_owned(); + rewritten.replace_range(start..end, &placeholder); + Some(rewritten) + } + } + + impl TextHandler for Scripts { + fn selector(&self) -> &str { + "script" + } + + fn decide(&mut self, text: &str, is_last: bool) -> ScriptRewriteAction { + let claimed = !self.held.is_empty() || text.contains("fixture_payload"); + if !claimed { + return ScriptRewriteAction::Keep; + } + self.held.push_str(text); + if !is_last { + return ScriptRewriteAction::RemoveNode; + } + let script = std::mem::take(&mut self.held); + let rewritten = self.rewrite_whole(&script).unwrap_or(script); + ScriptRewriteAction::replace(rewritten) + } + } + + struct Payloads; + + impl Middleware for Payloads { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + /// The text handler and the stream processor of one document + /// share what the handler captures. + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let captured = Arc::new(Mutex::new(Captured::default())); + MiddlewareAction { + text_handlers: vec![Box::new(Scripts { + captured: Arc::clone(&captured), + held: String::new(), + })], + stream: Some(Box::new(Processor { + captured, + origin_host: context.origin_host.to_owned(), + request_host: context.request_host.to_owned(), + held: Vec::new(), + })), + ..MiddlewareAction::pass() + } + } + } + + struct Processor { + captured: Arc>, + origin_host: String, + request_host: String, + /// Output not yet released, because it ends inside a placeholder + /// or starts at one whose group is unresolved. + held: Vec, + } + + fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|at| at + from) + } + + impl StreamProcessor for Processor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + self.held.extend_from_slice(chunk); + let captured = self.captured.lock().unwrap_or_else(PoisonError::into_inner); + let prefix = PLACEHOLDER_PREFIX.as_bytes(); + let mut out = Vec::with_capacity(self.held.len()); + let mut at = 0; + + while let Some(start) = find(&self.held, prefix, at) { + let Some(end) = + find(&self.held, PLACEHOLDER_END.as_bytes(), start + prefix.len()) + else { + // The placeholder runs past this chunk. + out.extend_from_slice(&self.held[at..start]); + at = start; + break; + }; + let end = end + PLACEHOLDER_END.len(); + let found = std::str::from_utf8(&self.held[start..end]) + .ok() + .and_then(|text| { + captured + .payloads + .iter() + .find(|payload| payload.placeholder == text) + }); + match found { + Some(payload) if payload.unresolved && !is_last => { + out.extend_from_slice(&self.held[at..start]); + at = start; + break; + } + Some(payload) => { + out.extend_from_slice(&self.held[at..start]); + out.extend_from_slice( + payload + .original + .replace(&self.origin_host, &self.request_host) + .as_bytes(), + ); + at = end; + } + None => { + out.extend_from_slice(&self.held[at..end]); + at = end; + } + } + } + + let stopped_at_placeholder = find(&self.held, prefix, at) == Some(at); + if is_last { + out.extend_from_slice(&self.held[at..]); + self.held.clear(); + } else if stopped_at_placeholder { + self.held.drain(..at); + } else { + // Keep back a tail that could be the start of a + // placeholder split across chunks. + let rest = &self.held[at..]; + let keep = (1..prefix.len().min(rest.len() + 1)) + .rev() + .find(|len| rest.ends_with(&prefix[..*len])) + .unwrap_or(0); + out.extend_from_slice(&rest[..rest.len() - keep]); + let tail = rest[rest.len() - keep..].to_vec(); + self.held = tail; + } + Ok(out) + } + } + + /// Has `settings` select the stand-in and run its middleware, and no + /// other, on every page. + pub(crate) fn select_and_place(settings: &mut Settings) { + settings.select_module("testing", MODULE); + crate::html_processor::test_support::place_on_every_page( + settings, + MiddlewarePhase::Fetch, + &[MODULE], + ); + } + + fn register( + settings: &Settings, + ) -> Result, Report> { + if settings.module_config::(MODULE)?.is_none() { + return Ok(None); + } + Ok(Some( + IntegrationRegistration::builder(ID) + .with_middleware(Arc::new(Payloads)) + .build(), + )) + } + + /// The stand-in takes no settings, and refuses one it does not know + /// as any module does. + #[derive(Debug, serde::Deserialize, validator::Validate)] + #[serde(deny_unknown_fields)] + struct FixtureSettings {} + + impl crate::settings::IntegrationConfig for FixtureSettings {} + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(MODULE) + .map(|config| config.is_some()) + } + } +} + +#[cfg(test)] +mod tests { + use super::test_support::middleware_fixture as fixture; + use super::test_support::{ + PROBE_JS, PROBE_JS_SHA256, carried_probe_registration, probe_registration, validate_nothing, + }; + use super::*; + use crate::constants::COOKIE_EC; + use crate::middleware::{HTML_MEDIA_TYPE, MiddlewareAction, MiddlewareContext, PhaseEntry}; + use crate::permissions::{Permission, PermissionSet, PermissionState}; + use crate::platform::test_support::noop_services; + use http::{HeaderValue, StatusCode, header}; + + // Mock integration proxy for testing + struct MockProxy; + + #[async_trait(?Send)] + impl IntegrationProxy for MockProxy { + fn integration_name(&self) -> &'static str { + "test" + } + + fn routes(&self) -> Vec { + vec![] + } + + async fn handle( + &self, + _call: crate::module_context::ModuleCall<'_>, + _req: Request, + ) -> Result, Report> { + Ok(Response::new(EdgeBody::empty())) + } + } + + struct EnrichingRequestFilter; + #[derive(Clone, Copy)] + struct RequestAnnotation; + + #[async_trait(?Send)] + impl IntegrationRequestFilter for EnrichingRequestFilter { + fn integration_id(&self) -> &'static str { + "enriching" + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + input.request.extensions_mut().insert(RequestAnnotation); + Ok(RequestFilterDecision::Continue(RequestFilterEffects { + request_headers: vec![HeaderMutation::set("x-probe-isbot", "1")], + response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], + })) + } + } + + /// Records the permission state each filter invocation received, so a test + /// can assert what the registry handed to the filter. + #[derive(Default)] + struct RecordingPermissionsFilter { + seen: std::sync::Mutex>>, + } + + impl RecordingPermissionsFilter { + /// The permission state observed by the last invocation, or `None` when + /// the filter has not run. + fn seen(&self) -> Option> { + self.seen + .lock() + .expect("should lock the recorded permission state") + .clone() + } + } + + #[async_trait(?Send)] + impl IntegrationRequestFilter for RecordingPermissionsFilter { + fn integration_id(&self) -> &'static str { + "recording-permissions" + } + + async fn filter_request( + &self, + input: RequestFilterInput<'_>, + ) -> Result> { + *self + .seen + .lock() + .expect("should lock the recorded permission state") = + Some(input.permissions.cloned()); + Ok(RequestFilterDecision::Continue( + RequestFilterEffects::default(), + )) + } + } + + struct EchoProxy; + + #[async_trait(?Send)] + impl IntegrationProxy for EchoProxy { + fn integration_name(&self) -> &'static str { + "echo" + } + + fn routes(&self) -> Vec { + vec![] + } + + async fn handle( + &self, + _call: crate::module_context::ModuleCall<'_>, + req: http::Request, + ) -> Result, Report> { + let response = http::Response::builder() + .status(http::StatusCode::OK) + .header("x-echo-path", req.uri().path()) + .body(EdgeBody::empty()) + .expect("should build echo response"); + Ok(response) + } + } + + #[test] + fn document_state_keeps_multiple_types_for_one_integration() { + let state = IntegrationDocumentState::default(); + let number = state.get_or_insert_with("test", || 7_u32); + let label = state.get_or_insert_with("test", || "first".to_string()); + let repeated_number = state.get_or_insert_with("test", || 99_u32); + + assert!( + Arc::ptr_eq(&number, &repeated_number), + "repeated insertion should preserve the original typed state" + ); + assert_eq!( + *state.get::("test").expect("should retrieve number"), + 7, + "should retain numeric state" + ); + assert_eq!( + state + .get::("test") + .expect("should retrieve label") + .as_str(), + "first", + "should retain string state under the same integration ID" + ); + assert_eq!( + label.as_str(), + "first", + "should return inserted string state" + ); + } + + #[test] + fn handle_proxy_passes_http_request_without_fastly_round_trip() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::from_routes(vec![( + http::Method::GET, + "/integrations/test/echo", + (Arc::new(EchoProxy) as Arc, "echo"), + )]); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://test.example.com/integrations/test/echo?x=1") + .body(EdgeBody::empty()) + .expect("should build request"); + + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &http::Method::GET, + path: "/integrations/test/echo", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &noop_services(), + req, + })) + .expect("should match route") + .expect("proxy should succeed"); + + assert_eq!( response.status(), http::StatusCode::OK, "should preserve HTTP status" ); assert_eq!( - response.headers()["x-echo-path"], - "/integrations/test/echo", - "should expose the HTTP request path to the proxy" + response.headers()["x-echo-path"], + "/integrations/test/echo", + "should expose the HTTP request path to the proxy" + ); + } + + #[test] + fn filter_request_applies_request_headers_and_returns_response_headers() { + let registry = + IntegrationRegistry::from_request_filters(vec![Arc::new(EnrichingRequestFilter)]); + let settings = crate::test_support::tests::create_test_settings(); + let services = crate::platform::test_support::noop_services(); + let mut req = Request::builder() + .method(Method::GET) + .uri("https://example.com/page") + .body(EdgeBody::empty()) + .expect("should build request"); + + let outcome = + futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: None, + })) + .expect("should run request filter"); + + assert_eq!( + req.headers() + .get("x-probe-isbot") + .and_then(|value| value.to_str().ok()), + Some("1"), + "should apply a filter's request enrichment before routing" + ); + assert!( + req.extensions().get::().is_some(), + "should preserve private request annotations for downstream routing" + ); + match outcome { + RequestFilterRegistryOutcome::Continue(effects) => { + assert_eq!( + effects.response_headers, + vec![HeaderMutation::set("x-dd-b", "allowed")], + "should return downstream response header effects for finalization" + ); + } + RequestFilterRegistryOutcome::Respond { .. } => panic!("should continue routing"), + } + } + + #[test] + fn filter_request_passes_the_resolved_permission_state_to_each_filter() { + let filter = Arc::new(RecordingPermissionsFilter::default()); + let registry = IntegrationRegistry::from_request_filters(vec![ + filter.clone() as Arc + ]); + let settings = crate::test_support::tests::create_test_settings(); + let services = crate::platform::test_support::noop_services(); + let permissions = + PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)); + let mut req = Request::builder() + .method(Method::GET) + .uri("https://example.com/page") + .body(EdgeBody::empty()) + .expect("should build request"); + + futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: Some(&permissions), + })) + .expect("should run request filter"); + + assert_eq!( + filter.seen(), + Some(Some(permissions)), + "the filter should observe exactly the permission state resolved for the request" + ); + + // A path that builds no EC context passes no permissions, and the + // filter must see that absence rather than an empty state. + futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: None, + })) + .expect("should run request filter"); + + assert_eq!( + filter.seen(), + Some(None), + "an absent permission state should reach the filter as `None`" + ); + } + + #[test] + fn test_exact_route_matching() { + let routes = vec![( + Method::GET, + "/integrations/test/exact", + (Arc::new(MockProxy) as Arc, "test"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match exact route + assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + + // Should not match different paths + assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); + assert!(!registry.has_route(&Method::GET, "/integrations/test/exact/nested")); + + // Should not match different methods + assert!(!registry.has_route(&Method::POST, "/integrations/test/exact")); + } + + #[test] + fn test_wildcard_route_matching() { + let routes = vec![( + Method::GET, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match paths under the wildcard prefix + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); + assert!(registry.has_route( + &Method::GET, + "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" + )); + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/page-view")); + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/a/b/c/d/e")); + + // Should not match paths that don't start with the prefix + assert!(!registry.has_route(&Method::GET, "/integrations/lockr/sdk")); + assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other")); + assert!(!registry.has_route(&Method::GET, "/integrations/other/api/settings")); + + // Should not match different methods + assert!(!registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + } + + #[test] + fn test_wildcard_and_exact_routes_coexist() { + let routes = vec![ + ( + Method::GET, + "/integrations/test/api/*", + (Arc::new(MockProxy) as Arc, "test"), + ), + ( + Method::GET, + "/integrations/test/exact", + (Arc::new(MockProxy) as Arc, "test"), + ), + ]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Exact route should match + assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + + // Wildcard routes should match + assert!(registry.has_route(&Method::GET, "/integrations/test/api/anything")); + assert!(registry.has_route(&Method::GET, "/integrations/test/api/nested/path")); + + // Non-matching should fail + assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); + } + + #[test] + fn test_multiple_wildcard_routes() { + let routes = vec![ + ( + Method::GET, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + ), + ( + Method::POST, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + ), + ( + Method::GET, + "/integrations/testlight/api/*", + ( + Arc::new(MockProxy) as Arc, + "testlight", + ), + ), + ]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Lockr GET routes should match + assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); + + // Lockr POST routes should match + assert!(registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + + // Testlight routes should match + assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/auction")); + assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/any-path")); + + // Cross-integration paths should not match + assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other-endpoint")); + assert!(!registry.has_route(&Method::GET, "/integrations/other/api/test")); + } + + #[test] + fn test_wildcard_preserves_casing() { + let routes = vec![( + Method::GET, + "/integrations/lockr/api/*", + (Arc::new(MockProxy) as Arc, "lockr"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match with camelCase preserved + assert!(registry.has_route( + &Method::GET, + "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" + )); + assert!(registry.has_route( + &Method::GET, + "/integrations/lockr/api/publisher/app/v1/identitylockr/settings" + )); + } + + #[test] + fn test_wildcard_edge_cases() { + let routes = vec![( + Method::GET, + "/api/*", + (Arc::new(MockProxy) as Arc, "test"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match paths under /api/ + assert!(registry.has_route(&Method::GET, "/api/v1")); + assert!(registry.has_route(&Method::GET, "/api/v1/users")); + + // Should not match /api without trailing content + // The current implementation requires a / after the prefix + assert!(!registry.has_route(&Method::GET, "/api")); + + // Should not match partial prefix matches + assert!(!registry.has_route(&Method::GET, "/apiv1")); + } + + #[test] + fn test_helper_methods_create_namespaced_routes() { + let proxy = Arc::new(MockProxy); + + // Test all HTTP method helpers + let get_endpoint = proxy.get("/users"); + assert_eq!(get_endpoint.method, Method::GET); + assert_eq!(get_endpoint.path, "/integrations/test/users"); + + let post_endpoint = proxy.post("/users"); + assert_eq!(post_endpoint.method, Method::POST); + assert_eq!(post_endpoint.path, "/integrations/test/users"); + + let put_endpoint = proxy.put("/users"); + assert_eq!(put_endpoint.method, Method::PUT); + assert_eq!(put_endpoint.path, "/integrations/test/users"); + + let delete_endpoint = proxy.delete("/users"); + assert_eq!(delete_endpoint.method, Method::DELETE); + assert_eq!(delete_endpoint.path, "/integrations/test/users"); + + let patch_endpoint = proxy.patch("/users"); + assert_eq!(patch_endpoint.method, Method::PATCH); + assert_eq!(patch_endpoint.path, "/integrations/test/users"); + } + + #[test] + fn test_put_delete_patch_routes() { + let routes = vec![ + ( + Method::PUT, + "/integrations/test/users", + (Arc::new(MockProxy) as Arc, "test"), + ), + ( + Method::DELETE, + "/integrations/test/users", + (Arc::new(MockProxy) as Arc, "test"), + ), + ( + Method::PATCH, + "/integrations/test/users", + (Arc::new(MockProxy) as Arc, "test"), + ), + ]; + + let registry = IntegrationRegistry::from_routes(routes); + + // Should match PUT, DELETE, and PATCH routes + assert!(registry.has_route(&Method::PUT, "/integrations/test/users")); + assert!(registry.has_route(&Method::DELETE, "/integrations/test/users")); + assert!(registry.has_route(&Method::PATCH, "/integrations/test/users")); + + // Should not match other methods on same path + assert!(!registry.has_route(&Method::GET, "/integrations/test/users")); + assert!(!registry.has_route(&Method::POST, "/integrations/test/users")); + } + + // Tests for EC ID header on proxy responses + use crate::test_support::tests::create_test_settings; + + /// Mock proxy that returns a simple 200 OK response + struct EcTestProxy; + + #[async_trait(?Send)] + impl IntegrationProxy for EcTestProxy { + fn integration_name(&self) -> &'static str { + "ec_test" + } + + fn routes(&self) -> Vec { + vec![ + IntegrationEndpoint { + method: Method::GET, + path: "/integrations/test/ec".to_owned(), + }, + IntegrationEndpoint { + method: Method::POST, + path: "/integrations/test/ec".to_owned(), + }, + ] + } + + async fn handle( + &self, + _call: crate::module_context::ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + let mut response = Response::builder() + .status(StatusCode::OK) + .body(EdgeBody::from("test response")) + .expect("should build test response"); + if let Some(ec) = req.headers().get(HEADER_X_TS_EC.clone()) { + response + .headers_mut() + .insert(http::HeaderName::from_static("x-echo-ts-ec"), ec.clone()); + } + Ok(response) + } + } + + #[test] + fn handle_proxy_removes_ec_id_header_on_request() { + let settings = create_test_settings(); + let routes = vec![( + Method::GET, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]; + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/ec") + .body(EdgeBody::empty()) + .expect("should build request"); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + HeaderValue::from_static("some-ec-value"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let services = noop_services(); + + // Call handle_proxy (uses futures executor in test environment) + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })); + + // Should have matched and returned a response + assert!(result.is_some(), "should find route and handle request"); + let response = result.unwrap(); + assert!(response.is_ok(), "handler should succeed"); + + let response = response.unwrap(); + + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "should not have x-ts-ec header on integration request" + ); + } + + /// A route that answers with what its module call handed it. + struct StateRoute { + declared: PermissionSet, + } + + impl StateRoute { + fn report( + &self, + _req: Request, + request: crate::module_context::ModuleRequest<'_>, + permissions: &PermissionState, + edge_cookie: Option>, + ) -> Result, Report> { + let body = format!( + "{} storage={} id={}", + request.host(), + permissions.is_set(Permission::StoreOnDevice), + edge_cookie.map_or("none", |id| id.as_str()), + ); + Ok(Response::builder() + .status(StatusCode::OK) + .body(EdgeBody::from(body)) + .expect("should build the response")) + } + } + + #[async_trait(?Send)] + impl IntegrationProxy for StateRoute { + fn integration_name(&self) -> &'static str { + "state" + } + + fn routes(&self) -> Vec { + vec![IntegrationEndpoint::get("/integrations/test/state")] + } + + fn required_permissions(&self) -> PermissionSet { + self.declared + } + + async fn handle( + &self, + call: ModuleCall<'_>, + req: Request, + ) -> Result, Report> { + call.inject_with(self, req, Self::report)? + } + } + + /// A route is handed the permissions resolved for its request, and the + /// request's Edge Cookie identifier when it declares the permission the + /// identifier's use needs. + #[test] + fn a_route_is_handed_its_requests_permissions_and_identity() { + let settings = create_test_settings(); + let storage = PermissionSet::none().with(Permission::StoreOnDevice); + let answer = |declared: PermissionSet| { + let registry = IntegrationRegistry::from_routes(vec![( + Method::GET, + "/integrations/test/state", + ( + Arc::new(StateRoute { declared }) as Arc, + "state", + ), + )]); + let req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/state") + .header(header::HOST, "test-publisher.com") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut ec_context = EcContext::new_for_test( + Some("an-id".to_owned()), + crate::consent::ConsentContext::default(), + ) + .with_module_for_test(Arc::new(crate::ec::module::HmacModule::new( + crate::redacted::Redacted::new("test-secret-key-32-bytes-minimum".to_owned()), + ))); + let services = noop_services(); + let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/state", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })) + .expect("should find the route") + .expect("the route should answer"); + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("the answer is text") + }; + + assert_eq!( + answer(storage), + "test-publisher.com storage=true id=an-id", + "a route declaring storage is handed the identifier" + ); + assert_eq!( + answer(PermissionSet::none()), + "test-publisher.com storage=true id=none", + "and one declaring nothing still sees the permissions but not the identifier" + ); + } + + #[test] + fn handle_proxy_rejects_invalid_ec_request_header() { + let settings = create_test_settings(); + let routes = vec![( + Method::GET, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]; + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/ec") + .body(EdgeBody::empty()) + .expect("should build request"); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + HeaderValue::from_static("evil;injected"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + + let services = crate::platform::test_support::noop_services(); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })) + .expect("should handle proxy request"); + + let response = result.expect("handler should succeed"); + + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "should not reflect the tampered request header to the integration" + ); + } + + #[test] + fn handle_proxy_removes_request_ec_header_even_when_consent_denied() { + let settings = create_test_settings(); + let routes = vec![( + Method::GET, + "/integrations/test/ec", + (Arc::new(EcTestProxy) as Arc, "test"), + )]; + + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::GET) + .uri("https://test.example.com/integrations/test/ec") + .body(EdgeBody::empty()) + .expect("should build request"); + req.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(&format!( + "ts-{COOKIE_EC}={}", + crate::test_support::tests::VALID_SYNTHETIC_ID + )) + .expect("should build Cookie header"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + + let services = crate::platform::test_support::noop_services(); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })) + .expect("should handle proxy request"); + + let response = result.expect("proxy handle should succeed"); + + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "should not set x-ts-ec on integration request" + ); + } + + #[test] + fn handle_proxy_works_with_post_method() { + let settings = create_test_settings(); + let routes = vec![( + Method::POST, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]; + let registry = IntegrationRegistry::from_routes(routes); + + let mut req = Request::builder() + .method(Method::POST) + .uri("https://test-publisher.com/integrations/test/ec") + .body(EdgeBody::from("test body")) + .expect("should build POST request"); + req.headers_mut().insert( + HEADER_X_TS_EC.clone(), + HeaderValue::from_static("some-ec-value"), + ); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + + let services = crate::platform::test_support::noop_services(); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::POST, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + req, + })); + + assert!(result.is_some(), "Should find POST route"); + let response = result.unwrap(); + assert!(response.is_ok(), "Handler should succeed"); + + let response = response.unwrap(); + assert!( + response.headers().get("x-echo-ts-ec").is_none(), + "POST integration request should not include x-ts-ec" + ); + } + + #[test] + fn js_module_ids_defer_a_deferred_module_and_include_core_js_only_modules() { + let mut settings = crate::test_support::tests::create_test_settings(); + enable_deferred_module(&mut settings); + + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); + + let all = registry.js_module_ids(); + let immediate = registry.js_module_ids_immediate(); + let deferred = registry.js_module_ids_deferred(); + let module = test_support::deferred_fixture::ID; + + assert!( + all.contains(&module), + "should include the deferred module in the module IDs" + ); + assert!( + immediate.contains(&"creative"), + "should include creative in immediate IDs" + ); + assert!( + !immediate.contains(&"sourcepoint"), + "should not include Sourcepoint unless it is named" + ); + assert!( + !immediate.contains(&module), + "should not include the deferred module in immediate IDs" + ); + assert!( + deferred.contains(&module), + "should serve the module as a deferred module" + ); + } + + #[test] + fn js_module_ids_skip_named_integrations_without_generated_js_module() { + let settings = settings_naming("testing.probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should create registry"); + let all = registry.js_module_ids(); + + assert!( + !all.contains(&"probe"), + "should not include named integrations without generated JS modules" + ); + + let metadata = registry.registered_integrations(); + assert!( + metadata.iter().any(|integration| integration.id == "probe"), + "should still register named Rust-only integrations" + ); + } + + #[test] + fn js_module_ids_include_client_fixed_when_module_selected() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.ec.module = Some(crate::ec::module::EcModuleSelection::from( + crate::ec::module::CLIENT_FIXED_MODULE_KEY, + )); + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + + assert!( + registry + .js_module_ids_immediate() + .contains(&"ec_client_fixed"), + "selecting the `client_fixed` module should inject its demo page script" + ); + } + + #[test] + fn js_module_ids_exclude_client_fixed_without_module() { + let registry = + IntegrationRegistry::new(&crate::test_support::tests::create_test_settings()) + .expect("should create registry"); + + assert!( + !registry + .js_module_ids_immediate() + .contains(&"ec_client_fixed"), + "the demo page script should not ship unless the `client_fixed` module is selected" + ); + } + + #[test] + fn js_module_ids_deferred_empty_when_prebid_is_not_named() { + let mut settings = crate::test_support::tests::create_test_settings(); + // The shared fixture names prebid, and this asks what the registry + // serves when it does not. + settings.auction.modules.clear(); + + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); + + let deferred = registry.js_module_ids_deferred(); + assert!( + deferred.is_empty(), + "should have no deferred IDs when prebid is not named" + ); + } + + #[test] + fn js_module_ids_split_is_exhaustive() { + let mut settings = crate::test_support::tests::create_test_settings(); + enable_deferred_module(&mut settings); + + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); + + let all = registry.js_module_ids(); + assert!( + !registry.js_module_ids_deferred().is_empty(), + "should have a deferred module to divide from the bundle" + ); + let mut recombined = registry.js_module_ids_immediate(); + recombined.extend(registry.js_module_ids_deferred()); + recombined.sort_unstable(); + + let mut all_sorted = all; + all_sorted.sort_unstable(); + + assert_eq!( + recombined, all_sorted, + "should reconstruct full module list from immediate + deferred" + ); + } + + /// Claims the id of the JavaScript asset proxy, which is core's own. + fn duplicate_core_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder( + crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, + ) + .build(), + )) + } + + /// The shared fixture with the module `name` selected in its section, so a + /// test builder is built the way any integration an operator names is. + fn settings_naming(name: &str) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + let section = name.split_once('.').map_or("testing", |(folder, _)| folder); + settings.select_module(section, name); + settings + } + + #[test] + fn with_registrations_adds_an_external_builder_after_the_built_ins() { + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with an external builder"); + + assert!( + registry.integration_runs("probe"), + "should register the external integration" + ); + let ids = registry.registered_builder_ids(); + assert_eq!( + ids.last().copied(), + Some("probe"), + "should order the external builder after every built-in" + ); + } + + const SHARED_VENDOR: &str = "example"; + + fn shared_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("shared_probe").build(), + )) + } + + /// The `shared_probe` builder, run by `[shared.example]`. + fn shared_probe_builder() -> crate::integrations::IntegrationBuilder { + crate::integrations::IntegrationBuilder::new( + "shared_probe", + "seam-probe", + shared_probe_registration, + validate_nothing, + ) + .with_shared_settings(SHARED_VENDOR) + } + + fn with_shared_table(mut settings: Settings) -> Settings { + settings.shared = + serde_json::from_value(serde_json::json!({ SHARED_VENDOR: { "key": "value" } })) + .expect("should read the shared tables"); + settings + } + + #[test] + fn a_builder_its_shared_table_runs_is_built_only_when_the_table_is_present() { + let extra = [shared_probe_builder()]; + + let without = IntegrationRegistry::with_registrations( + &crate::test_support::tests::create_test_settings(), + &extra, + ) + .expect("nothing is built, which is not an error"); + assert!(!without.integration_runs("shared_probe")); + + let with = IntegrationRegistry::with_registrations( + &with_shared_table(crate::test_support::tests::create_test_settings()), + &extra, + ) + .expect("should build the builder its table runs"); + assert!(with.integration_runs("shared_probe")); + } + + #[test] + fn a_shared_table_no_builder_reads_is_refused() { + let refused = IntegrationRegistry::with_registrations( + &with_shared_table(crate::test_support::tests::create_test_settings()), + &[], + ) + .err() + .expect("nothing reads the table") + .to_string(); + + assert!( + refused.contains("[shared.example] is read by no module"), + "{refused}" + ); + } + + #[test] + fn a_builder_its_shared_table_runs_answers_to_no_selection() { + let refused = IntegrationRegistry::with_registrations( + &with_shared_table(settings_naming("shared_probe")), + &[shared_probe_builder()], + ) + .err() + .expect("the table runs the builder, so no section can select it") + .to_string(); + + assert!(refused.contains("selects `shared_probe`"), "{refused}"); + } + + #[test] + fn a_selection_nothing_supplies_names_the_shared_table_that_is_absent() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.geo.module = Some("example".to_owned()); + + let refused = IntegrationRegistry::with_registrations(&settings, &[shared_probe_builder()]) + .err() + .expect("no geo module is supplied") + .to_string(); + + assert!( + refused.contains("There is no [shared.example]"), + "the message has to name the table to add: {refused}" + ); + } + + #[test] + fn a_list_selection_nothing_supplies_names_the_shared_table_that_is_absent() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.select_module("identity", "example.id"); + + let refused = IntegrationRegistry::with_registrations(&settings, &[shared_probe_builder()]) + .err() + .expect("no identity module is supplied") + .to_string(); + + assert!( + refused.contains("[identity] selects `example.id`") + && refused.contains("There is no [shared.example]"), + "the message has to name the table to add: {refused}" + ); + } + + #[test] + fn a_builder_its_shared_table_runs_is_refused_a_module_name() { + let mut settings = with_shared_table(crate::test_support::tests::create_test_settings()); + settings.geo.module = Some("example".to_owned()); + + let refused = IntegrationRegistry::with_registrations( + &settings, + &[shared_probe_builder().with_module_name("geo.example")], + ) + .err() + .expect("a section could select what the table runs") + .to_string(); + + assert!( + refused.contains("is run by [shared.example] and also names the module `geo.example`"), + "{refused}" + ); + } + + #[test] + fn with_registrations_rejects_a_duplicate_integration_id_naming_both_sources() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [crate::integrations::IntegrationBuilder::new( + crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, + "seam-probe", + duplicate_core_registration, + validate_nothing, + ) + .with_module_name("testing.duplicate")]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a duplicate integration id"); + + let message = error.to_string(); + assert!( + message.contains(crate::integrations::js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID) + && message.contains("trusted-server-core") + && message.contains("seam-probe"), + "error should name the id and both sources: {message}" + ); + } + + /// A builder that registers from the auction plan holds its id like any + /// other, so a second builder claiming it is refused, naming both sources, + /// whether or not the plan-backed integration runs. + #[test] + fn with_registrations_rejects_a_second_builder_claiming_a_plan_backed_id() { + fn register_nothing( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + fn nothing_from_the_plan( + _settings: &Settings, + _plan: &crate::auction::plan::AuctionPlan, + ) -> Result, Report> { + Ok(None) + } + + let settings = crate::test_support::tests::create_test_settings(); + let id = "plan_backed"; + let extra = [ + crate::integrations::IntegrationBuilder::new( + id, + "plan-backed-crate", + register_nothing, + validate_nothing, + ) + .with_plan_registration(nothing_from_the_plan), + crate::integrations::IntegrationBuilder::new( + id, + "seam-probe", + register_nothing, + validate_nothing, + ), + ]; + + let Err(error) = IntegrationRegistry::with_registrations(&settings, &extra) else { + panic!("should refuse a second builder claiming `{id}`"); + }; + + let message = error.to_string(); + assert!( + message.contains(id) + && message.contains("plan-backed-crate") + && message.contains("seam-probe"), + "error should name `{id}` and both sources: {message}" + ); + } + + /// Selects core's stand-in, which registers a deferred browser module. + fn enable_deferred_module(settings: &mut Settings) { + settings.select_module("testing", test_support::deferred_fixture::MODULE); + } + + /// Selects core's stand-in, which registers a standalone browser module. + fn enable_standalone_module(settings: &mut Settings) { + settings.select_module("testing", test_support::request_fixture::MODULE); + } + + fn carried_probe_builder() -> crate::integrations::IntegrationBuilder { + crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe") + } + + const ZERO_SHA256: &str = "0000000000000000000000000000000000000000000000000000000000000000"; + + fn lying_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: ZERO_SHA256, + }) + .build(), + )) + } + + fn disabled_carried_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .without_js() + .build(), + )) + } + + #[test] + fn probe_js_hash_literal_matches_its_source() { + assert_eq!( + hex::encode(Sha256::digest(PROBE_JS.as_bytes())), + PROBE_JS_SHA256, + "should keep the PROBE_JS_SHA256 literal equal to the hash of PROBE_JS" + ); + } + + #[test] + fn a_carried_js_module_is_served_in_the_immediate_parts() { + let settings = settings_naming("probe"); + let extra = [carried_probe_builder()]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with a carried module"); + + let parts = registry.js_parts_immediate(); + assert_eq!( + parts.first().map(|part| part.id), + Some("core"), + "should put core first in the immediate parts" + ); + let probe = parts + .iter() + .find(|part| part.id == "probe") + .expect("should include the carried probe module in the immediate parts"); + assert_eq!( + probe.source, PROBE_JS, + "should serve the carried source verbatim" + ); + assert_eq!( + probe.sha256, PROBE_JS_SHA256, + "should carry the declared hash on the part" + ); + assert!( + registry.js_module_ids_immediate().contains(&"probe"), + "should list the carried module among the immediate module ids" + ); + } + + #[test] + fn a_carried_js_module_with_a_wrong_hash_is_rejected_at_registry_build() { + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + lying_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a carried module whose declared hash is wrong"); + + let message = error.to_string(); + assert!( + message.contains("probe") && message.contains("does not match"), + "error should name the integration and say the hash does not match: {message}" + ); + assert!( + message.contains(ZERO_SHA256) && message.contains(PROBE_JS_SHA256), + "error should quote the declared and the actual hash: {message}" + ); + } + + #[test] + fn js_part_is_none_for_an_integration_registered_without_js() { + // The carried lookup would answer `Some` on its own, so this proves the + // without-JS check runs first. No built-in integration can stand in, because + // none registers without a browser script. + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + disabled_carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + + assert!( + registry.integration_runs("probe"), + "should still register the integration" + ); + assert!( + registry.js_part("probe").is_none(), + "should not serve a module for an integration registered without JS" + ); + assert!( + !registry.js_module_ids().contains(&"probe"), + "should keep a without-JS integration out of the bundle module ids" + ); + } + + #[test] + fn the_last_js_delivery_flag_set_on_the_builder_wins() { + let disabled_last = IntegrationRegistration::builder("probe") + .with_standalone_js() + .without_js() + .build(); + assert!( + disabled_last.js_disabled && !disabled_last.js_standalone && !disabled_last.js_deferred, + "without_js after with_standalone_js should leave only disabled set" + ); + + let deferred_last = IntegrationRegistration::builder("probe") + .with_standalone_js() + .with_deferred_js() + .build(); + assert!( + deferred_last.js_deferred && !deferred_last.js_standalone && !deferred_last.js_disabled, + "with_deferred_js after with_standalone_js should leave only deferred set" + ); + + let deferred_after_disabled = IntegrationRegistration::builder("probe") + .without_js() + .with_deferred_js() + .build(); + assert!( + deferred_after_disabled.js_deferred + && !deferred_after_disabled.js_disabled + && !deferred_after_disabled.js_standalone, + "with_deferred_js after without_js should leave only deferred set" + ); + + let standalone_last = IntegrationRegistration::builder("probe") + .without_js() + .with_deferred_js() + .with_standalone_js() + .build(); + assert!( + standalone_last.js_standalone + && !standalone_last.js_disabled + && !standalone_last.js_deferred, + "with_standalone_js last should leave only standalone set" + ); + } + + #[test] + fn a_standalone_js_module_is_served_alone_and_not_in_the_bundle() { + let mut settings = crate::test_support::tests::create_test_settings(); + enable_standalone_module(&mut settings); + + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + + assert!( + !registry + .js_module_ids() + .contains(&test_support::request_fixture::ID), + "should keep a standalone module out of the bundle module ids" + ); + assert!( + registry + .js_part(test_support::request_fixture::ID) + .is_some(), + "should serve the standalone module as a part" + ); + assert_eq!( + registry.js_standalone_ids(), + vec![test_support::request_fixture::ID], + "should list the standalone module that runs" + ); + assert!( + registry.js_part("lockr").is_none(), + "should not serve a module for an integration that does not run" + ); + } + + #[test] + fn js_parts_all_covers_bundle_deferred_and_standalone_modules() { + let mut settings = settings_naming("probe"); + enable_deferred_module(&mut settings); + enable_standalone_module(&mut settings); + let extra = [carried_probe_builder()]; + + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + + let ids = registry + .js_parts_all() + .into_iter() + .map(|part| part.id) + .collect::>(); + for expected in [ + "core", + "creative", + "probe", + test_support::deferred_fixture::ID, + test_support::request_fixture::ID, + ] { + assert_eq!( + ids.iter().filter(|id| **id == expected).count(), + 1, + "should list `{expected}` exactly once in {ids:?}" + ); + } + } + + /// A builder function for an integration that never registers, so its + /// preparer is the only thing the registry can take from it. + fn never_registering_builder( + _settings: &Settings, + ) -> Result, Report> { + Ok(None) + } + + /// Preparers cannot capture, because a builder holds plain function + /// pointers, so they record the order they ran in here. + static PREPARER_ORDER: Mutex> = Mutex::new(Vec::new()); + + fn record_first_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .push("first"); + Ok(()) + } + + fn record_second_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .push("second"); + Ok(()) + } + + /// Records whether the preparer registered after the failing one ran. + static PREPARER_AFTER_FAILURE: Mutex> = Mutex::new(Vec::new()); + + const FAILING_PREPARER_MESSAGE: &str = "probe preparer refused the request"; + + fn failing_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + Err(Report::new(TrustedServerError::Configuration { + message: FAILING_PREPARER_MESSAGE.to_owned(), + })) + } + + fn record_after_failure_preparer( + _settings: &Settings, + _request: &mut Request, + ) -> Result<(), Report> { + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .push("after"); + Ok(()) + } + + fn plain_request() -> Request { + Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/article") + .body(EdgeBody::empty()) + .expect("should build request") + } + + #[test] + fn prepare_request_runs_every_preparer_in_registration_order_named_or_not() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-first", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-first") + .with_request_preparer(record_first_preparer), + crate::integrations::IntegrationBuilder::new( + "probe-second", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-second") + .with_request_preparer(record_second_preparer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with request preparers"); + assert!( + !registry.integration_runs("probe-first"), + "should leave the first probe integration unnamed, so it does not run" + ); + assert!( + !registry.integration_runs("probe-second"), + "should leave the second probe integration unnamed, so it does not run" + ); + PREPARER_ORDER + .lock() + .expect("should lock the preparer order") + .clear(); + let mut request = plain_request(); + + registry + .prepare_request(&settings, &mut request) + .expect("should run every request preparer"); + + assert_eq!( + *PREPARER_ORDER + .lock() + .expect("should lock the preparer order"), + vec!["first", "second"], + "should run both preparers in registration order even though neither integration runs" + ); + } + + #[test] + fn prepare_request_surfaces_the_first_preparer_error() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-failing", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-failing") + .with_request_preparer(failing_preparer), + crate::integrations::IntegrationBuilder::new( + "probe-after-failure", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-after-failure") + .with_request_preparer(record_after_failure_preparer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with request preparers"); + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .clear(); + let mut request = plain_request(); + + let error = registry + .prepare_request(&settings, &mut request) + .expect_err("should surface the failing preparer's error"); + + assert!( + error.to_string().contains(FAILING_PREPARER_MESSAGE), + "should keep the preparer's message intact: {error}" + ); + assert!( + PREPARER_AFTER_FAILURE + .lock() + .expect("should lock the after-failure record") + .is_empty(), + "should not run a preparer registered after the failing one" + ); + } + + // Each probe supplies a middleware that changes nothing, so a test can + // read the order registrations were made in from the names the registry + // lists. + const SECTION_MIDDLEWARE: &str = "testing.probe-section"; + const PLAN_MIDDLEWARE: &str = "testing.probe-plan"; + + fn section_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe-section") + .with_middleware(Arc::new(Named { + id: SECTION_MIDDLEWARE, + phases: FETCH_ONLY, + })) + .build(), + )) + } + + /// Registers when the plan's auction is enabled, whatever the sections + /// select. + fn plan_probe_registration( + _settings: &Settings, + plan: &AuctionPlan, + ) -> Result, Report> { + Ok(plan.enabled().then(|| { + IntegrationRegistration::builder("probe-plan") + .with_middleware(Arc::new(Named { + id: PLAN_MIDDLEWARE, + phases: FETCH_ONLY, + })) + .build() + })) + } + + #[test] + fn a_registration_from_the_plan_runs_unselected_and_ahead_of_a_section_s_module() { + let mut settings = settings_naming("testing.probe-section"); + // The section's module is listed first, and what the plan registers + // still goes ahead of it. + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-section", + "seam-probe", + section_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe-section"), + crate::integrations::IntegrationBuilder::new( + "probe-plan", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-plan") + .with_plan_registration(plan_probe_registration), + ]; + let probe_middleware = |registry: &IntegrationRegistry| { + registry + .middleware_ids() + .into_iter() + .filter(|name| *name == SECTION_MIDDLEWARE || *name == PLAN_MIDDLEWARE) + .collect::>() + }; + + settings.auction.enabled = false; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + assert!( + !registry.integration_runs("probe-plan"), + "should register nothing where the function finds nothing in the plan" + ); + assert_eq!(probe_middleware(®istry), vec![SECTION_MIDDLEWARE]); + + settings.auction.enabled = true; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry"); + assert!( + registry.integration_runs("probe-plan"), + "should register from the plan a module no section selects" + ); + assert_eq!( + probe_middleware(®istry), + vec![PLAN_MIDDLEWARE, SECTION_MIDDLEWARE], + "should register what the plan supplies ahead of a section's module" + ); + } + + /// Appends to a header on each run, so a test can count the runs on the + /// request itself. + fn counting_preparer( + _settings: &Settings, + request: &mut Request, + ) -> Result<(), Report> { + request + .headers_mut() + .append("x-probe-prepared", HeaderValue::from_static("1")); + Ok(()) + } + + #[test] + fn prepare_request_runs_the_preparers_once_for_a_request() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe-counting", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-counting") + .with_request_preparer(counting_preparer)]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with a request preparer"); + let mut request = plain_request(); + + registry + .prepare_request(&settings, &mut request) + .expect("should prepare the request"); + registry + .prepare_request(&settings, &mut request) + .expect("should accept a request that is already prepared"); + + assert_eq!( + request.headers().get_all("x-probe-prepared").iter().count(), + 1, + "should run a preparer once however often the request is prepared" + ); + + let mut next = plain_request(); + registry + .prepare_request(&settings, &mut next) + .expect("should prepare the next request"); + assert_eq!( + next.headers().get_all("x-probe-prepared").iter().count(), + 1, + "should prepare each request" + ); + } + + /// What the second probe's request hook leaves for its finalizer. + #[derive(Debug, PartialEq)] + struct ProbeNote(&'static str); + + fn first_finalizer(_state: &IntegrationRequestState, response: &mut Response) { + response + .headers_mut() + .append("x-probe-finalized", HeaderValue::from_static("first")); + } + + fn second_finalizer(state: &IntegrationRequestState, response: &mut Response) { + let value = match state.get::("probe-second") { + Some(_) => "second-with-its-note", + None => "second", + }; + response + .headers_mut() + .append("x-probe-finalized", HeaderValue::from_static(value)); + } + + #[test] + fn finalize_response_runs_every_finalizer_in_registration_order_with_the_request_state() { + let settings = crate::test_support::tests::create_test_settings(); + let extra = [ + crate::integrations::IntegrationBuilder::new( + "probe-first", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-first") + .with_response_finalizer(first_finalizer), + crate::integrations::IntegrationBuilder::new( + "probe-second", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-second") + .with_response_finalizer(second_finalizer), + ]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build registry with response finalizers"); + let finalized = |response: &Response| { + response + .headers() + .get_all("x-probe-finalized") + .iter() + .map(|value| value.to_str().expect("should be text").to_owned()) + .collect::>() + }; + + let mut request = plain_request(); + IntegrationRequestState::insert(&mut request, "probe-second", ProbeNote("left")); + let mut response = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + registry.finalize_response(&IntegrationRequestState::of(&request), &mut response); + + assert_eq!( + finalized(&response), + vec!["first", "second-with-its-note"], + "should run both finalizers in registration order, neither integration \ + running, and hand each what was left on the request" + ); + + let mut unmarked = Response::builder() + .body(EdgeBody::empty()) + .expect("should build response"); + registry.finalize_response(&IntegrationRequestState::default(), &mut unmarked); + assert_eq!( + finalized(&unmarked), + vec!["first", "second"], + "should hand a finalizer no state for a request nothing was left on" + ); + } + + #[test] + fn what_a_module_leaves_on_a_request_reaches_a_document_when_it_is_seeded() { + let mut request = plain_request(); + assert!( + IntegrationRequestState::of(&request).is_empty(), + "should hold nothing for a request no module left a value on" + ); + + IntegrationRequestState::insert(&mut request, "probe", ProbeNote("first")); + IntegrationRequestState::insert(&mut request, "probe", ProbeNote("second")); + IntegrationRequestState::insert(&mut request, "other", ProbeNote("other")); + let state = IntegrationRequestState::of(&request); + + assert!(!state.is_empty(), "should hold what was left"); + assert_eq!( + state.get::("probe").as_deref(), + Some(&ProbeNote("second")), + "should keep the last value of one type left for an integration" + ); + assert!( + state.get::("absent").is_none(), + "should hold nothing for an integration that left nothing" + ); + + let document = IntegrationDocumentState::default(); + assert!( + document.get::("probe").is_none(), + "should start a document with none of the request's values" + ); + state.seed(&document); + assert_eq!( + document.get::("probe").as_deref(), + Some(&ProbeNote("second")), + "should copy the value into the document's state" + ); + assert_eq!( + document.get::("other").as_deref(), + Some(&ProbeNote("other")), + "should copy every integration's value" ); } #[test] - fn filter_request_applies_request_headers_and_returns_response_headers() { - let registry = - IntegrationRegistry::from_request_filters(vec![Arc::new(EnrichingRequestFilter)]); + fn the_request_stand_in_marks_a_navigation_and_strips_its_query() { + use super::test_support::request_fixture; + let settings = crate::test_support::tests::create_test_settings(); - let services = crate::platform::test_support::noop_services(); - let mut req = Request::builder() + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let mut navigation = Request::builder() .method(Method::GET) - .uri("https://example.com/page") + .uri("https://publisher.example.com/article?fixture_request=1&keep=yes") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut subresource = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/data.json?fixture_request=1") + .header("sec-fetch-dest", "empty") .body(EdgeBody::empty()) .expect("should build request"); - let outcome = - futures::executor::block_on(registry.filter_request(RequestFilterRegistryInput { - settings: &settings, - services: &services, - req: &mut req, - geo_info: None, - })) - .expect("should run request filter"); + registry + .prepare_request(&settings, &mut navigation) + .expect("should prepare the navigation"); + registry + .prepare_request(&settings, &mut subresource) + .expect("should prepare the subresource request"); assert_eq!( - req.headers() - .get("x-datadome-isbot") - .and_then(|value| value.to_str().ok()), - Some("1"), - "should apply DataDome-style request enrichment before routing" + navigation.uri().query(), + Some("keep=yes"), + "should strip the stand-in's query and keep the rest" ); assert!( - req.extensions().get::().is_some(), - "should preserve private request annotations for downstream routing" + IntegrationRequestState::of(&navigation) + .get::(request_fixture::ID) + .is_some(), + "should leave the mark on a document navigation, selected or not" ); - match outcome { - RequestFilterRegistryOutcome::Continue(effects) => { - assert_eq!( - effects.response_headers, - vec![HeaderMutation::set("x-dd-b", "allowed")], - "should return downstream response header effects for finalization" - ); + assert_eq!( + subresource.uri().query(), + None, + "should strip the stand-in's query from any request" + ); + assert!( + IntegrationRequestState::of(&subresource).is_empty(), + "should leave no mark on a request for something other than a document" + ); + } + + /// Example country code returned by the test geo module. `ZZ` is the + /// user-assigned code, so it names no real place. + const GEO_PROBE_COUNTRY: &str = "ZZ"; + + /// A geo module that resolves one fixed location, so a test can tell the + /// module's module apart from the "no location" one. + #[derive(Debug)] + struct FixedCountryGeo; + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformGeo for FixedCountryGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &crate::platform::RuntimeServices, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: "Example City".to_owned(), + country: GEO_PROBE_COUNTRY.to_owned(), + continent: "Example".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + })) + } + } + + /// The name the `geo-probe` registration's geo module is selected by. + const GEO_PROBE_MODULE: &str = "testing.geo-probe"; + + /// Builds a `geo-probe` registration that declares a geo module. + fn geo_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("geo-probe") + .with_geo_module(GEO_PROBE_MODULE, Arc::new(FixedCountryGeo)) + .build(), + )) + } + + /// A device module that declares it needs a permission, which nothing can + /// enforce per request yet. + #[derive(Debug)] + struct PermissionDemandingDevice; + + /// The name the `device-probe` registration's device module is selected + /// by. + const DEVICE_PROBE_MODULE: &str = "testing.device-probe"; + + #[async_trait::async_trait(?Send)] + impl crate::ec::device::DeviceModule for PermissionDemandingDevice { + fn id(&self) -> &'static str { + DEVICE_PROBE_MODULE + } + + async fn detect( + &self, + _call: crate::module_context::ModuleCall<'_>, + ) -> crate::ec::device::DeviceSignals { + crate::ec::device::DeviceSignals { + is_mobile: 2, + ja4_class: None, + platform_class: None, + h2_fp_hash: None, + known_browser: None, + looks_like_browser: false, + // Nothing here establishes whether a person is there, and + // `Unknown` is the value that says so rather than `Low`. + human_confidence: crate::ec::device::HumanConfidence::Unknown, } - RequestFilterRegistryOutcome::Respond { .. } => panic!("should continue routing"), } + + fn required_permissions(&self) -> crate::permissions::PermissionSet { + crate::permissions::PermissionSet::none() + .with(crate::permissions::Permission::StoreOnDevice) + } + } + + fn device_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("device-probe") + .with_device_module(DEVICE_PROBE_MODULE, Arc::new(PermissionDemandingDevice)) + .build(), + )) + } + + fn device_probe_builders() -> [crate::integrations::IntegrationBuilder; 1] { + [crate::integrations::IntegrationBuilder::new( + "device-probe", + "seam-probe", + device_probe_registration, + validate_nothing, + ) + .with_module_name(DEVICE_PROBE_MODULE)] } #[test] - fn test_exact_route_matching() { - let routes = vec![( - Method::GET, - "/integrations/test/exact", - (Arc::new(MockProxy) as Arc, "test"), - )]; + fn a_modules_device_module_declaring_permissions_is_refused_at_startup() { + let mut settings = settings_naming("device-probe"); + settings.device.module = Some(DEVICE_PROBE_MODULE.to_owned()); - let registry = IntegrationRegistry::from_routes(routes); + let error = IntegrationRegistry::with_registrations(&settings, &device_probe_builders()) + .err() + .expect("should refuse a device module whose permissions nothing can enforce"); - // Should match exact route - assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + let rendered = format!("{error:?}"); + assert!( + rendered.contains("device-probe"), + "the failure should name the module, got: {rendered}" + ); + assert!( + rendered.contains(&format!( + "requiring `{}`", + crate::permissions::Permission::StoreOnDevice.as_str() + )), + "the failure should say why it was refused, got: {rendered}" + ); + } - // Should not match different paths - assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); - assert!(!registry.has_route(&Method::GET, "/integrations/test/exact/nested")); + fn geo_probe_builders() -> [crate::integrations::IntegrationBuilder; 1] { + [crate::integrations::IntegrationBuilder::new( + "geo-probe", + "seam-probe", + geo_probe_registration, + validate_nothing, + ) + .with_module_name(GEO_PROBE_MODULE)] + } - // Should not match different methods - assert!(!registry.has_route(&Method::POST, "/integrations/test/exact")); + /// The names the two prefix test modules run under. + const PREFIX_PROBE_MODULE: &str = "testing.prefix-probe"; + const PREFIX_READER_MODULE: &str = "testing.prefix-reader"; + + /// A prefix module giving `_ex` and `ex.`, so a test can tell it + /// apart from core's own. + #[derive(Debug)] + struct ExamplePrefix; + + impl ExamplePrefix { + fn server_key(&self, settings: &Settings) -> crate::platform::ServerKeyPrefix { + crate::platform::ServerKeyPrefix::new("ex", &settings.publisher.domain) + } } - #[test] - fn test_wildcard_route_matching() { - let routes = vec![( - Method::GET, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - )]; + impl PrefixModule for ExamplePrefix { + fn public_prefix( + &self, + _call: ModuleCall<'_>, + ) -> Result { + Ok(crate::prefix::PublicPrefix::new(["_ex"]).expect("should accept the prefix")) + } - let registry = IntegrationRegistry::from_routes(routes); + fn server_key_prefix( + &self, + call: ModuleCall<'_>, + ) -> Result { + call.inject(self, Self::server_key) + } + } - // Should match paths under the wildcard prefix - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); - assert!(registry.has_route( - &Method::GET, - "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" - )); - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/page-view")); - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/a/b/c/d/e")); + /// A prefix module naming the reader's evidence, which its call never + /// carries. + #[derive(Debug)] + struct EvidenceReadingPrefix; - // Should not match paths that don't start with the prefix - assert!(!registry.has_route(&Method::GET, "/integrations/lockr/sdk")); - assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other")); - assert!(!registry.has_route(&Method::GET, "/integrations/other/api/settings")); + impl EvidenceReadingPrefix { + fn public( + &self, + evidence: &dyn crate::evidence::RequestInfo, + ) -> crate::prefix::PublicPrefix { + crate::prefix::PublicPrefix::new([evidence.user_agent()]) + .expect("should accept the prefix") + } + } - // Should not match different methods - assert!(!registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + impl PrefixModule for EvidenceReadingPrefix { + fn public_prefix( + &self, + call: ModuleCall<'_>, + ) -> Result { + call.inject(self, Self::public) + } + + fn server_key_prefix( + &self, + _call: ModuleCall<'_>, + ) -> Result { + Ok(crate::platform::ServerKeyPrefix::new("ev", "example.com")) + } } - #[test] - fn test_wildcard_and_exact_routes_coexist() { - let routes = vec![ - ( - Method::GET, - "/integrations/test/api/*", - (Arc::new(MockProxy) as Arc, "test"), - ), - ( - Method::GET, - "/integrations/test/exact", - (Arc::new(MockProxy) as Arc, "test"), - ), - ]; + fn prefix_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("prefix-probe") + .with_prefix_module(PREFIX_PROBE_MODULE, Arc::new(ExamplePrefix)) + .build(), + )) + } - let registry = IntegrationRegistry::from_routes(routes); + fn prefix_reader_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("prefix-reader") + .with_prefix_module(PREFIX_READER_MODULE, Arc::new(EvidenceReadingPrefix)) + .build(), + )) + } - // Exact route should match - assert!(registry.has_route(&Method::GET, "/integrations/test/exact")); + fn prefix_probe_builders() -> [crate::integrations::IntegrationBuilder; 2] { + [ + crate::integrations::IntegrationBuilder::new( + "prefix-probe", + "seam-probe", + prefix_probe_registration, + validate_nothing, + ) + .with_module_name(PREFIX_PROBE_MODULE), + crate::integrations::IntegrationBuilder::new( + "prefix-reader", + "seam-probe", + prefix_reader_registration, + validate_nothing, + ) + .with_module_name(PREFIX_READER_MODULE), + ] + } - // Wildcard routes should match - assert!(registry.has_route(&Method::GET, "/integrations/test/api/anything")); - assert!(registry.has_route(&Method::GET, "/integrations/test/api/nested/path")); + /// Settings running both probe modules on `www.example.com`, with + /// `[prefix] module` naming `selected` when given. + fn settings_selecting_prefix_module(selected: Option<&str>) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.domain = "www.example.com".to_owned(); + for id in ["prefix-probe", "prefix-reader"] { + settings + .insert_module_config("testing", &format!("testing.{id}"), &serde_json::json!({})) + .expect("should select the test module"); + } + settings.prefix.module = selected.map(str::to_owned); + settings + } - // Non-matching should fail - assert!(!registry.has_route(&Method::GET, "/integrations/test/other")); + #[test] + fn with_no_prefix_module_selected_the_registry_gives_core_ts() { + let settings = settings_selecting_prefix_module(None); + let registry = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .expect("should build the registry"); + + let prefixes = + registry.prefixes(&settings, crate::module_context::test_support::request("/")); + + assert_eq!(registry.prefix_module().name(), "ts"); + assert_eq!(prefixes.public().as_str(), "ts"); + assert_eq!(prefixes.public().cookie_name("ec"), "ts-ec"); + assert_eq!(prefixes.server_key().as_str(), "ts.example.com"); } #[test] - fn test_multiple_wildcard_routes() { - let routes = vec![ - ( - Method::GET, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - ), - ( - Method::POST, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - ), - ( - Method::GET, - "/integrations/testlight/api/*", - ( - Arc::new(MockProxy) as Arc, - "testlight", - ), - ), - ]; + fn a_prefix_selector_naming_ts_selects_cores_own() { + let settings = settings_selecting_prefix_module(Some("ts")); + let registry = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .expect("should accept the name the registry reports core's module by"); - let registry = IntegrationRegistry::from_routes(routes); + let prefixes = + registry.prefixes(&settings, crate::module_context::test_support::request("/")); - // Lockr GET routes should match - assert!(registry.has_route(&Method::GET, "/integrations/lockr/api/settings")); + assert_eq!(registry.prefix_module().name(), "ts"); + assert_eq!(prefixes.public().cookie_name("ec"), "ts-ec"); + assert_eq!(prefixes.server_key().as_str(), "ts.example.com"); + } - // Lockr POST routes should match - assert!(registry.has_route(&Method::POST, "/integrations/lockr/api/settings")); + #[test] + fn the_selected_prefix_module_gives_the_prefixes() { + let settings = settings_selecting_prefix_module(Some(PREFIX_PROBE_MODULE)); + let registry = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .expect("should build the registry"); - // Testlight routes should match - assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/auction")); - assert!(registry.has_route(&Method::GET, "/integrations/testlight/api/any-path")); + let prefixes = + registry.prefixes(&settings, crate::module_context::test_support::request("/")); - // Cross-integration paths should not match - assert!(!registry.has_route(&Method::GET, "/integrations/lockr/other-endpoint")); - assert!(!registry.has_route(&Method::GET, "/integrations/other/api/test")); + assert_eq!(registry.prefix_module().name(), PREFIX_PROBE_MODULE); + assert_eq!(prefixes.public().cookie_name("ec"), "_ex-ec"); + assert_eq!(prefixes.server_key().as_str(), "ex.example.com"); } #[test] - fn test_wildcard_preserves_casing() { - let routes = vec![( - Method::GET, - "/integrations/lockr/api/*", - (Arc::new(MockProxy) as Arc, "lockr"), - )]; + fn a_prefix_module_nothing_supplies_is_refused() { + let settings = settings_selecting_prefix_module(Some("missing")); - let registry = IntegrationRegistry::from_routes(routes); + let error = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .err() + .expect("a prefix module nothing supplies should be refused"); - // Should match with camelCase preserved - assert!(registry.has_route( - &Method::GET, - "/integrations/lockr/api/publisher/app/v1/identityLockr/settings" - )); - assert!(registry.has_route( - &Method::GET, - "/integrations/lockr/api/publisher/app/v1/identitylockr/settings" - )); + assert!( + error.to_string().contains("[prefix] module") && error.to_string().contains("missing"), + "{error}" + ); } #[test] - fn test_wildcard_edge_cases() { - let routes = vec![( - Method::GET, - "/api/*", - (Arc::new(MockProxy) as Arc, "test"), - )]; + fn a_prefix_module_naming_what_its_call_never_carries_is_refused() { + let settings = settings_selecting_prefix_module(Some(PREFIX_READER_MODULE)); - let registry = IntegrationRegistry::from_routes(routes); + let error = IntegrationRegistry::with_registrations(&settings, &prefix_probe_builders()) + .err() + .expect("a prefix module that cannot answer should be refused at startup"); + + assert!( + error.to_string().contains(PREFIX_READER_MODULE) + && error.to_string().contains("request evidence"), + "{error}" + ); + } + + /// Settings whose `[geo] module` names `module`, which may be a value + /// core resolves itself, such as `none` or `platform`. + fn settings_with_geo_selector(module: &str) -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.geo.module = Some(module.to_owned()); + settings + } + + /// The same, with the module selected in its section as well, which is + /// what a deployment running a module's geo module writes. + fn settings_selecting_geo_module(name: &str) -> Settings { + let mut settings = settings_with_geo_selector(name); + let section = name.split_once('.').map_or("testing", |(folder, _)| folder); + settings.select_module(section, name); + settings + } + + #[tokio::test] + async fn geo_selector_resolves_the_geo_module_the_selected_module_declares() { + let settings = settings_selecting_geo_module(GEO_PROBE_MODULE); + + let registry = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .expect("should build registry with a module geo module"); + + let module = registry + .geo_module() + .expect("should resolve the module's geo module"); + let resolved = module + .lookup(None, &noop_services()) + .await + .expect("should look up without failing") + .expect("should resolve a location"); + assert_eq!( + resolved.country, GEO_PROBE_COUNTRY, + "should resolve through the module's own module" + ); + } + + #[tokio::test] + async fn geo_module_none_resolves_no_location() { + let settings = settings_with_geo_selector("none"); + + let registry = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .expect("should build registry with the disabled geo module"); + + let module = registry + .geo_module() + .expect("should resolve the disabled geo module"); + assert!( + module + .lookup(None, &noop_services()) + .await + .expect("should look up without failing") + .is_none(), + "should resolve no location when the selector is `none`" + ); + } + + #[tokio::test] + async fn an_unset_geo_selector_resolves_the_disabled_module_and_platform_opts_in() { + // Unset is the permission model's privacy default: it resolves the + // disabled module, so no client IP ever reaches a host geo service. + // It is deliberately not the same as leaving the adapter's own lookup + // in place, which is what `platform` spells. + let settings = crate::test_support::tests::create_test_settings(); + assert!( + settings.geo.module.is_none(), + "the shared test settings should leave the geo selector unset" + ); + + let registry = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .expect("should build registry with an unset geo selector"); + + let module = registry + .geo_module() + .expect("an unset selector should resolve the disabled module, not nothing"); + assert!( + module + .lookup(None, &noop_services()) + .await + .expect("should look up without failing") + .is_none(), + "the disabled module should resolve no location and make no host call" + ); + + // `platform` is the explicit opt-in to the adapter's own lookup, and it + // is the one case that resolves nothing here so the adapter's module + // stands. + let platform = settings_with_geo_selector(GEO_MODULE_PLATFORM); + let registry = IntegrationRegistry::with_registrations(&platform, &geo_probe_builders()) + .expect("should build registry with the platform geo selector"); + assert!( + registry.geo_module().is_none(), + "`platform` should leave the adapter's own host lookup in place" + ); + } - // Should match paths under /api/ - assert!(registry.has_route(&Method::GET, "/api/v1")); - assert!(registry.has_route(&Method::GET, "/api/v1/users")); + #[test] + fn geo_selector_rejects_a_module_that_declares_no_geo_module() { + let settings = settings_selecting_geo_module("testing.probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - // Should not match /api without trailing content - // The current implementation requires a / after the prefix - assert!(!registry.has_route(&Method::GET, "/api")); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a module that declares no geo module"); - // Should not match partial prefix matches - assert!(!registry.has_route(&Method::GET, "/apiv1")); + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `testing.probe`") + && message.contains("It runs no geo module") + && message.contains("is selected and supplies no geo module"), + "error should name the module and say it supplies no geo module: {message}" + ); } #[test] - fn test_helper_methods_create_namespaced_routes() { - let proxy = Arc::new(MockProxy); + fn geo_selector_rejects_a_module_no_section_selects() { + // The module is registered, but no section selects it, so its geo + // module is not there to select. + let settings = settings_with_geo_selector("testing.probe-unnamed"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe-unnamed", + "seam-probe", + never_registering_builder, + validate_nothing, + ) + .with_module_name("testing.probe-unnamed")]; - // Test all HTTP method helpers - let get_endpoint = proxy.get("/users"); - assert_eq!(get_endpoint.method, Method::GET); - assert_eq!(get_endpoint.path, "/integrations/test/users"); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should reject a module no section selects"); - let post_endpoint = proxy.post("/users"); - assert_eq!(post_endpoint.method, Method::POST); - assert_eq!(post_endpoint.path, "/integrations/test/users"); + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `testing.probe-unnamed`") + && message.contains("is a module no section selects"), + "error should name the module and say no section selects it: {message}" + ); + } - let put_endpoint = proxy.put("/users"); - assert_eq!(put_endpoint.method, Method::PUT); - assert_eq!(put_endpoint.path, "/integrations/test/users"); + /// An id no builder supplies is refused here, where the adapter's and a + /// vendor crate's builders are known, rather than in the settings. + #[test] + fn an_id_no_builder_supplies_is_refused_at_registry_build() { + let settings = settings_naming("a_vendors_own_integration"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - let delete_endpoint = proxy.delete("/users"); - assert_eq!(delete_endpoint.method, Method::DELETE); - assert_eq!(delete_endpoint.path, "/integrations/test/users"); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse an id no builder in this deployment supplies"); - let patch_endpoint = proxy.patch("/users"); - assert_eq!(patch_endpoint.method, Method::PATCH); - assert_eq!(patch_endpoint.path, "/integrations/test/users"); + let message = error.to_string(); + assert!( + message.contains("a_vendors_own_integration"), + "should name the id nothing supplies: {message}" + ); + assert!( + message.contains("The testing modules it supplies are") && message.contains("probe"), + "should list the modules of that type this deployment does supply: {message}" + ); } + /// A demand or ad server implementation is not a page integration, so a + /// section naming one is refused like any name no builder supplies. #[test] - fn test_put_delete_patch_routes() { - let routes = vec![ - ( - Method::PUT, - "/integrations/test/users", - (Arc::new(MockProxy) as Arc, "test"), - ), - ( - Method::DELETE, - "/integrations/test/users", - (Arc::new(MockProxy) as Arc, "test"), - ), - ( - Method::PATCH, - "/integrations/test/users", - (Arc::new(MockProxy) as Arc, "test"), - ), - ]; + fn an_auction_implementation_named_in_a_section_is_refused() { + for (section, name) in [ + ("auction", "fixture"), + ("auction", "ad-server.fixture"), + ("auction", "plain-fixture"), + ] { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.select_module(section, name); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse an implementation named as a page module"); + + let message = error.to_string(); + assert!( + message.contains(&format!("[{section}] selects `{name}`")), + "should name the section and the implementation: {message}" + ); + } + } - let registry = IntegrationRegistry::from_routes(routes); + /// The same id is accepted once a builder supplies it, so a vendor crate + /// an adapter composes in is named the same way a built-in is. + #[test] + fn an_id_a_supplied_builder_claims_is_accepted() { + let settings = settings_naming("probe"); + let extra = [crate::integrations::IntegrationBuilder::new( + "probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; - // Should match PUT, DELETE, and PATCH routes - assert!(registry.has_route(&Method::PUT, "/integrations/test/users")); - assert!(registry.has_route(&Method::DELETE, "/integrations/test/users")); - assert!(registry.has_route(&Method::PATCH, "/integrations/test/users")); + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should accept an id a supplied builder claims"); - // Should not match other methods on same path - assert!(!registry.has_route(&Method::GET, "/integrations/test/users")); - assert!(!registry.has_route(&Method::POST, "/integrations/test/users")); + assert!( + registry.integration_runs("probe"), + "the named module should run" + ); } - // Tests for EC ID header on proxy responses - use crate::test_support::tests::create_test_settings; + #[test] + fn geo_selector_rejects_a_name_no_module_supplies() { + // Nothing supplies the name, as a geo module or as a module at all. + let settings = settings_with_geo_selector("absent-module"); - /// Mock proxy that returns a simple 200 OK response - struct EcTestProxy; + let error = IntegrationRegistry::with_registrations(&settings, &geo_probe_builders()) + .err() + .expect("should reject a selector naming nothing this deployment runs"); - #[async_trait(?Send)] - impl IntegrationProxy for EcTestProxy { - fn integration_name(&self) -> &'static str { - "ec_test" - } + let message = error.to_string(); + assert!( + message.contains("`[geo] module` names `absent-module`") + && message.contains("which no module this deployment runs supplies") + && !message.contains("no section selects"), + "error should name what was written and say nothing supplies it: {message}" + ); + } - fn routes(&self) -> Vec { - vec![ - IntegrationEndpoint { - method: Method::GET, - path: "/integrations/test/ec".to_owned(), - }, - IntegrationEndpoint { - method: Method::POST, - path: "/integrations/test/ec".to_owned(), - }, - ] + /// A device module that needs no permission, for a registration that + /// supplies more than one type of module. + #[derive(Debug)] + struct ExampleDevice; + + #[async_trait::async_trait(?Send)] + impl crate::ec::device::DeviceModule for ExampleDevice { + fn id(&self) -> &'static str { + "device.example" } - async fn handle( + async fn detect( &self, - _settings: &Settings, - _services: &RuntimeServices, - req: Request, - ) -> Result, Report> { - let mut response = Response::builder() - .status(StatusCode::OK) - .body(EdgeBody::from("test response")) - .expect("should build test response"); - if let Some(ec) = req.headers().get(HEADER_X_TS_EC.clone()) { - response - .headers_mut() - .insert(http::HeaderName::from_static("x-echo-ts-ec"), ec.clone()); + _call: crate::module_context::ModuleCall<'_>, + ) -> crate::ec::device::DeviceSignals { + crate::ec::device::DeviceSignals { + is_mobile: 1, + ja4_class: None, + platform_class: Some("example".to_owned()), + h2_fp_hash: None, + known_browser: None, + looks_like_browser: true, + human_confidence: crate::ec::device::HumanConfidence::Unknown, } - Ok(response) } } - #[test] - fn handle_proxy_removes_ec_id_header_on_request() { - let settings = create_test_settings(); - let routes = vec![( - Method::GET, - "/integrations/test/ec", - ( - Arc::new(EcTestProxy) as Arc, - "ec_test", - ), - )]; - let registry = IntegrationRegistry::from_routes(routes); + /// An Edge Cookie module that answers to `edgecookie.example`. + #[derive(Debug)] + struct ExampleEc; - let mut req = Request::builder() - .method(Method::GET) - .uri("https://test-publisher.com/integrations/test/ec") - .body(EdgeBody::empty()) - .expect("should build request"); - req.headers_mut().insert( - HEADER_X_TS_EC.clone(), - HeaderValue::from_static("some-ec-value"), - ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - let services = noop_services(); + #[async_trait::async_trait(?Send)] + impl EdgeCookieModule for ExampleEc { + fn id(&self) -> &'static str { + "edgecookie.example" + } - // Call handle_proxy (uses futures executor in test environment) - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::GET, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })); + fn code(&self) -> crate::ec::module::ModuleCode { + crate::module_code!("t0rg") + } - // Should have matched and returned a response - assert!(result.is_some(), "should find route and handle request"); - let response = result.unwrap(); - assert!(response.is_ok(), "handler should succeed"); + async fn generate( + &self, + _call: crate::module_context::ModuleCall<'_>, + ) -> Result> { + Ok(crate::ec::module::GeneratedEdgeCookie::default()) + } + } - let response = response.unwrap(); + /// Builds one registration that supplies a geo, a device and an Edge + /// Cookie module, each under the name of its own type. + fn example_vendor_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("example-vendor") + .with_geo_module("geo.example", Arc::new(FixedCountryGeo)) + .with_device_module("device.example", Arc::new(ExampleDevice)) + .with_ec_module("edgecookie.example", Arc::new(ExampleEc)) + .build(), + )) + } - assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "should not have x-ts-ec header on integration request" - ); + fn example_vendor_builders() -> [crate::integrations::IntegrationBuilder; 1] { + [crate::integrations::IntegrationBuilder::new( + "example-vendor", + "seam-probe", + example_vendor_registration, + validate_nothing, + ) + .with_module_name("testing.example-vendor")] } + /// One registration can supply a module of each type, and each selector + /// finds its own by the short name, the type folder left off. #[test] - fn handle_proxy_rejects_invalid_ec_request_header() { - let settings = create_test_settings(); - let routes = vec![( - Method::GET, - "/integrations/test/ec", - ( - Arc::new(EcTestProxy) as Arc, - "ec_test", - ), - )]; - let registry = IntegrationRegistry::from_routes(routes); + fn each_selector_resolves_its_own_type_of_module_from_one_registration() { + let mut settings = settings_naming("example-vendor"); + settings.geo.module = Some("example".to_owned()); + settings.device.module = Some("example".to_owned()); + settings.ec.module = Some(EcModuleSelection::from("example")); - let mut req = Request::builder() - .method(Method::GET) - .uri("https://test-publisher.com/integrations/test/ec") - .body(EdgeBody::empty()) - .expect("should build request"); - req.headers_mut().insert( - HEADER_X_TS_EC.clone(), - HeaderValue::from_static("evil;injected"), - ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should build registry with one module of each type"); - let services = crate::platform::test_support::noop_services(); + assert!( + registry.geo_module().is_some(), + "`[geo] module = \"example\"` should resolve `geo.example`" + ); + assert_eq!( + registry + .device_module() + .expect("`[device] module = \"example\"` should resolve `device.example`") + .id(), + "device.example", + "should resolve the registration's device module" + ); + assert_eq!( + registry + .ec_module() + .expect("`[ec] module = \"example\"` should resolve `edgecookie.example`") + .id(), + "edgecookie.example", + "should resolve the registration's Edge Cookie module" + ); + } - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::GET, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })) - .expect("should handle proxy request"); + /// A name written in full selects the same module, and a short name is + /// read within the selector's own type only. + #[test] + fn a_selector_reads_a_full_name_and_keeps_a_short_name_within_its_type() { + let mut settings = settings_naming("example-vendor"); + settings.geo.module = Some("geo.example".to_owned()); - let response = result.expect("handler should succeed"); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should resolve a geo module written in full"); + assert!( + registry.geo_module().is_some(), + "`[geo] module = \"geo.example\"` should resolve the module" + ); + // `device.example` is a device module, so `[geo]` does not find it + // under the short name of another type. + let mut settings = settings_naming("example-vendor"); + settings.geo.module = Some("device.example".to_owned()); + let error = IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .err() + .expect("should refuse a device module named as a geo module"); + let message = error.to_string(); assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "should not reflect the tampered request header to the integration" + message.contains("`[geo] module` names `device.example`") + && message.contains("The geo modules it runs are [example]"), + "error should list the geo modules by the name the section writes: {message}" ); } + /// `[ec] module` may name a label whose block names the implementation, + /// and the registry resolves the implementation, as core does. #[test] - fn handle_proxy_removes_request_ec_header_even_when_consent_denied() { - let settings = create_test_settings(); - let routes = vec![( - Method::GET, - "/integrations/test/ec", - (Arc::new(EcTestProxy) as Arc, "test"), - )]; + fn ec_selector_resolves_the_implementation_a_labelled_block_names() { + let mut settings = settings_naming("example-vendor"); + settings.ec.module = Some(EcModuleSelection::from("primary")); + settings.ec.module_blocks.insert( + "primary".to_owned(), + crate::settings::EcModuleBlock { + implementation: Some("example".to_owned()), + settings: crate::settings::EcModuleSettings::Injected(serde_json::Map::new()), + }, + ); - let registry = IntegrationRegistry::from_routes(routes); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should build registry with a labelled Edge Cookie selection"); - let mut req = Request::builder() - .method(Method::GET) - .uri("https://test.example.com/integrations/test/ec") - .body(EdgeBody::empty()) - .expect("should build request"); - req.headers_mut().insert( - header::COOKIE, - HeaderValue::from_str(&format!( - "{}={}", - COOKIE_TS_EC, - crate::test_support::tests::VALID_SYNTHETIC_ID - )) - .expect("should build Cookie header"), + assert_eq!( + registry + .ec_module() + .expect("should resolve the implementation the block names") + .id(), + "edgecookie.example", + "should resolve the registration's Edge Cookie module" ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - - let services = crate::platform::test_support::noop_services(); + } - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::GET, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })) - .expect("should handle proxy request"); + /// Core's own Edge Cookie modules are not the registry's to supply, so a + /// selection naming one resolves nothing here and is not an error. + #[test] + fn ec_selector_naming_a_module_of_cores_resolves_nothing_here() { + let settings = settings_naming("example-vendor"); + assert!( + settings + .ec + .module + .as_ref() + .is_some_and(|selection| selection.key() == crate::ec::module::HMAC_MODULE_KEY), + "the shared test settings should select core's HMAC module" + ); - let response = result.expect("proxy handle should succeed"); + let registry = + IntegrationRegistry::with_registrations(&settings, &example_vendor_builders()) + .expect("should build registry with core's own Edge Cookie module selected"); assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "should not set x-ts-ec on integration request" + registry.ec_module().is_none(), + "core resolves its own module, so the registry should supply none" ); } + fn misnamed_ec_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("misnamed") + .with_ec_module("edgecookie.another", Arc::new(ExampleEc)) + .build(), + )) + } + + /// Core checks the selected Edge Cookie module by its own id, so one + /// declared under another name is refused where the mistake is made. #[test] - fn handle_proxy_works_with_post_method() { - let settings = create_test_settings(); - let routes = vec![( - Method::POST, - "/integrations/test/ec", - ( - Arc::new(EcTestProxy) as Arc, - "ec_test", - ), - )]; - let registry = IntegrationRegistry::from_routes(routes); + fn an_edge_cookie_module_declared_under_another_name_is_refused() { + let settings = settings_naming("misnamed"); + let extra = [crate::integrations::IntegrationBuilder::new( + "misnamed", + "seam-probe", + misnamed_ec_registration, + validate_nothing, + ) + .with_module_name("testing.misnamed")]; - let mut req = Request::builder() - .method(Method::POST) - .uri("https://test-publisher.com/integrations/test/ec") - .body(EdgeBody::from("test body")) - .expect("should build POST request"); - req.headers_mut().insert( - HEADER_X_TS_EC.clone(), - HeaderValue::from_static("some-ec-value"), + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse an Edge Cookie module declared under another name"); + + let message = error.to_string(); + assert!( + message.contains("`edgecookie.another`") && message.contains("`edgecookie.example`"), + "error should give both names: {message}" ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + } - let services = crate::platform::test_support::noop_services(); + fn second_geo_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("geo-second") + .with_geo_module(GEO_PROBE_MODULE, Arc::new(FixedCountryGeo)) + .build(), + )) + } - let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { - method: &Method::POST, - path: "/integrations/test/ec", - settings: &settings, - kv: None, - ec_context: &mut ec_context, - services: &services, - req, - })); + /// Two registrations supplying a module of one type under one name would + /// leave a selector meaning either, so the pair is refused. + #[test] + fn two_registrations_declaring_one_geo_module_name_are_refused() { + let mut settings = settings_naming("geo-probe"); + settings.select_module("testing", "testing.geo-second"); + let extra = [ + geo_probe_builders()[0], + crate::integrations::IntegrationBuilder::new( + "geo-second", + "seam-probe", + second_geo_registration, + validate_nothing, + ) + .with_module_name("testing.geo-second"), + ]; - assert!(result.is_some(), "Should find POST route"); - let response = result.unwrap(); - assert!(response.is_ok(), "Handler should succeed"); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse one geo module name declared twice"); - let response = response.unwrap(); + let message = error.to_string(); assert!( - response.headers().get("x-echo-ts-ec").is_none(), - "POST integration request should not include x-ts-ec" + message.contains("the geo module `testing.geo-probe` is declared twice") + && message.contains("`geo-probe`") + && message.contains("`geo-second`"), + "error should name the module and both integrations: {message}" ); } - #[test] - fn js_module_ids_defer_prebid_and_include_core_js_only_modules() { - let settings = crate::test_support::tests::create_test_settings(); - let mut settings_with_prebid = settings; - settings_with_prebid - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - "timeout_ms": 1000, - "debug": false - }), - ) - .expect("should insert prebid config"); + fn html_entry(path: Option<&str>, names: &[&str]) -> PhaseEntry { + PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: path.map(str::to_owned), + middleware: names.iter().map(|name| (*name).to_owned()).collect(), + } + } - let registry = IntegrationRegistry::with_plan( - &settings_with_prebid, - Arc::new( - crate::auction::compile_auction_plan(&settings_with_prebid) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + /// Settings that select core's middleware stand-in and hold `entries`. + fn fixture_settings_with_entries(entries: Vec) -> Settings { + let mut settings = settings_naming(fixture::MODULE); + settings.fetch = PhaseEntries::new(entries); + settings + } - let all = registry.js_module_ids(); - let immediate = registry.js_module_ids_immediate(); - let deferred = registry.js_module_ids_deferred(); + #[test] + fn a_selected_module_s_middleware_run_where_an_entry_names_them() { + let settings = fixture_settings_with_entries(vec![ + html_entry(Some("/news/"), &[fixture::LINKS, fixture::HEAD]), + html_entry(None, &[fixture::HEAD]), + ]); - assert!( - all.contains(&"prebid"), - "should include the prebid shim in embedded TSJS module IDs" + let registry = IntegrationRegistry::new(&settings) + .expect("should build a registry whose entries name registered middleware"); + + assert_eq!( + registry.middleware_ids(), + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER, + crate::service_version::MODULE, + ], + "should list the middleware the selected module supplies, and core's own after them" ); - assert!( - immediate.contains(&"creative"), - "should include creative in immediate IDs" + assert_eq!( + registry.middleware_in(MiddlewarePhase::Fetch), + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING + ], + "should list the middleware that run in the fetch phase" ); - assert!( - !immediate.contains(&"sourcepoint"), - "should not include Sourcepoint unless explicitly enabled" + assert_eq!( + registry.middleware_in(MiddlewarePhase::Serve), + [ + fixture::READER, + fixture::BROKEN_READER, + fixture::PREPARING, + fixture::TEXT_HEADER, + crate::service_version::MODULE + ], + "should list the middleware that run in the serve phase, core's own last" ); - assert!( - !immediate.contains(&"prebid"), - "should not include prebid in immediate IDs" + let chain_for = |media_type: &str, path: &str| { + registry + .middleware_chain(&settings.fetch, MiddlewarePhase::Fetch, media_type, path) + .ids() + }; + assert_eq!( + chain_for(HTML_MEDIA_TYPE, "/news/today"), + [fixture::LINKS, fixture::HEAD], + "should run the first covering entry's middleware in the order it names them" ); + assert_eq!(chain_for(HTML_MEDIA_TYPE, "/"), [fixture::HEAD]); assert!( - deferred.contains(&"prebid"), - "should serve the prebid shim as a deferred module" + chain_for("text/css", "/news/site.css").is_empty(), + "should run nothing on a media type no entry covers" ); } + /// An entry for `text/plain` naming the middleware that works on it. + fn text_entry(names: &[&str]) -> PhaseEntry { + PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: names.iter().map(|name| (*name).to_owned()).collect(), + } + } + #[test] - fn js_module_ids_skip_enabled_integrations_without_generated_js_module() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config("nextjs", &serde_json::json!({ "enabled": true })) - .expect("should insert nextjs config"); + fn an_entry_runs_a_middleware_on_the_media_type_it_handles() { + let settings = fixture_settings_with_entries(vec![ + html_entry(None, &[fixture::HEAD]), + text_entry(&[fixture::TEXT]), + ]); + + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + + let chain_for = |media_type: &str| { + registry + .middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + media_type, + "/a.txt", + ) + .ids() + }; + assert_eq!( + chain_for(fixture::TEXT_MEDIA_TYPE), + [fixture::TEXT], + "should run the text middleware on a text response" + ); + assert_eq!( + chain_for(HTML_MEDIA_TYPE), + [fixture::HEAD], + "should run the page middleware on a page and not the text one" + ); + } - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let all = registry.js_module_ids(); + #[test] + fn an_entry_naming_a_middleware_under_a_media_type_it_does_not_handle_is_refused() { + for (media_type, name) in [ + (fixture::TEXT_MEDIA_TYPE, fixture::HEAD), + (HTML_MEDIA_TYPE, fixture::TEXT), + ] { + let settings = fixture_settings_with_entries(vec![PhaseEntry { + media_type: media_type.to_owned(), + path: None, + middleware: vec![name.to_owned()], + }]); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a middleware under a media type it does not handle"); + + let message = error.to_string(); + assert!( + message.contains(&format!( + "[[fetch]] entry 1 covers `{media_type}` and names `{name}`, which does not \ + work on that media type" + )), + "should name the entry, the media type and the middleware: {message}" + ); + } + } - assert!( - !all.contains(&"nextjs"), - "should not include enabled integrations without generated JS modules" - ); + #[test] + fn a_module_no_section_selects_supplies_no_middleware() { + let settings = crate::test_support::tests::create_test_settings(); + + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); - let metadata = registry.registered_integrations(); assert!( - metadata - .iter() - .any(|integration| integration.id == "nextjs"), - "should still register enabled Rust-only integrations" + !registry.middleware_ids().contains(&fixture::HEAD), + "should register nothing of a module that does not run" ); } #[test] - fn js_module_ids_include_explicitly_enabled_cmp_mirrors() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config("sourcepoint", &serde_json::json!({ "enabled": true })) - .expect("should insert sourcepoint config"); - settings - .integrations - .insert_config("osano", &serde_json::json!({ "enabled": true })) - .expect("should insert osano config"); + fn an_entry_naming_a_middleware_nothing_supplies_is_refused_with_what_is_supplied() { + let settings = fixture_settings_with_entries(vec![ + html_entry(Some("/news/"), &[fixture::HEAD]), + html_entry(None, &[fixture::HEAD, "testing.nothing"]), + ]); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); - let immediate = registry.js_module_ids_immediate(); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse an entry naming a middleware nothing supplies"); + let message = error.to_string(); assert!( - immediate.contains(&"sourcepoint"), - "should include Sourcepoint when explicitly enabled" - ); - assert!( - immediate.contains(&"osano"), - "should include Osano when explicitly enabled" + message.contains( + "[[fetch]] entry 2 names `testing.nothing`, which no module that runs supplies" + ) && message.contains(&format!( + "[{}, {}, {}, {}, {}, {}, {}, {}, {}, {}]", + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER, + crate::service_version::MODULE + )), + "should name the entry and list what could be named: {message}" ); - - let metadata = registry.registered_integrations(); assert!( - metadata.iter().any(|integration| integration.id == "osano"), - "should include JS-only Osano registration in metadata" + !message.contains("no section selects"), + "should not blame a selection when the name is no module's: {message}" ); } #[test] - fn js_module_ids_deferred_empty_when_prebid_disabled() { + fn an_entry_naming_the_middleware_of_a_module_no_section_selects_says_so() { let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": false, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - }), - ) - .expect("should update prebid config"); + settings.fetch = PhaseEntries::new(vec![html_entry(None, &[fixture::LINKS])]); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse an entry naming the middleware of a module that does not run"); - let deferred = registry.js_module_ids_deferred(); + let message = error.to_string(); assert!( - deferred.is_empty(), - "should have no deferred IDs when prebid is disabled" + message.contains(&format!("[[fetch]] entry 1 names `{}`", fixture::LINKS)) + && message.contains(&format!( + "`{}` is a module no section selects, so nothing it supplies is running", + fixture::MODULE + )), + "should say the module is not selected: {message}" ); } #[test] - fn js_module_ids_defer_prebid_shim_when_external_bundle_is_configured() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js" - }), - ) - .expect("should update prebid config"); + fn a_middleware_no_entry_names_is_listed_for_the_startup_warning() { + let unnamed = |entries: Vec| -> Vec<&'static str> { + let settings = fixture_settings_with_entries(entries); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + unnamed_middleware(&settings, ®istry.inner) + .into_iter() + .map(|(integration, middleware)| { + assert_eq!(integration, fixture::ID, "should name the supplier"); + middleware.middleware_id() + }) + .collect() + }; - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create registry"); + assert_eq!( + unnamed(Vec::new()), + [ + fixture::HEAD, + fixture::LINKS, + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER, + ], + "should list every middleware when there are no entries" + ); + assert_eq!( + unnamed(vec![ + html_entry(Some("/news/"), &[fixture::LINKS]), + html_entry(None, &[fixture::HEAD]), + ]), + [ + fixture::BROKEN, + fixture::READER, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER + ], + "should leave out a middleware any entry names" + ); + } + #[test] + fn an_entry_naming_a_middleware_in_a_phase_it_does_not_run_in_is_refused() { + let mut settings = settings_naming(fixture::MODULE); + settings.serve = PhaseEntries::new(vec![html_entry(None, &[fixture::HEAD])]); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a fetch middleware named in a serve entry"); + let message = error.to_string(); assert!( - registry.js_module_ids().contains(&"prebid"), - "external bundle mode should include the prebid shim in embedded TSJS modules" + message.contains(&format!("[[serve]] entry 1 names `{}`", fixture::HEAD)) + && message.contains("does not run in that phase. It runs in [[fetch]]"), + "should say which phase the middleware runs in: {message}" ); + + let mut settings = settings_naming(fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![html_entry(None, &[fixture::READER])]); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a serve middleware named in a fetch entry"); + let message = error.to_string(); assert!( - !registry.js_module_ids_immediate().contains(&"prebid"), - "the prebid shim should not load in the immediate TSJS bundle" + message.contains(&format!("[[fetch]] entry 1 names `{}`", fixture::READER)) + && message.contains("does not run in that phase. It runs in [[serve]]"), + "should say which phase the middleware runs in: {message}" ); - assert!( - registry.js_module_ids_deferred().contains(&"prebid"), - "the prebid shim should load as a deferred TSJS module" + } + + #[test] + fn each_phase_has_entries_of_its_own() { + let mut settings = settings_naming(fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![html_entry(None, &[fixture::HEAD])]); + settings.serve = PhaseEntries::new(vec![html_entry(Some("/news/"), &[fixture::READER])]); + let registry = IntegrationRegistry::new(&settings) + .expect("should build a registry with an entry in each phase"); + + let chain_for = |phase: MiddlewarePhase, path: &str| { + registry + .middleware_chain(settings.phase_entries(phase), phase, HTML_MEDIA_TYPE, path) + .ids() + }; + assert_eq!( + chain_for(MiddlewarePhase::Fetch, "/news/today"), + [fixture::HEAD] + ); + assert_eq!( + chain_for(MiddlewarePhase::Serve, "/news/today"), + [fixture::READER] ); assert!( - registry.has_route(&Method::GET, "/integrations/prebid/bundle.js"), - "external bundle mode should register the first-party bundle route" + chain_for(MiddlewarePhase::Serve, "/sport/today").is_empty(), + "should run no serve middleware on a path only the fetch entries cover" + ); + let unnamed: Vec<&str> = unnamed_middleware(&settings, ®istry.inner) + .into_iter() + .map(|(_, middleware)| middleware.middleware_id()) + .collect(); + assert_eq!( + unnamed, + [ + fixture::LINKS, + fixture::BROKEN, + fixture::BROKEN_READER, + fixture::TEXT, + fixture::PREPARING, + fixture::REFUSING, + fixture::TEXT_HEADER + ], + "should count a middleware as named only by an entry of a phase it runs in" ); } - #[test] - fn js_module_ids_split_is_exhaustive() { - let settings = crate::test_support::tests::create_test_settings(); - let mut settings_with_prebid = settings; - settings_with_prebid - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - "timeout_ms": 1000, - "debug": false - }), - ) - .expect("should insert prebid config"); + /// A middleware under a name of the test's choosing, running in the + /// phases given. + struct Named { + id: &'static str, + phases: &'static [MiddlewarePhase], + } - let registry = IntegrationRegistry::with_plan( - &settings_with_prebid, - Arc::new( - crate::auction::compile_auction_plan(&settings_with_prebid) - .expect("should compile auction plan"), - ), + impl Middleware for Named { + fn middleware_id(&self) -> &'static str { + self.id + } + + fn phases(&self) -> &[MiddlewarePhase] { + self.phases + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } + } + + const FETCH_ONLY: &[MiddlewarePhase] = &[MiddlewarePhase::Fetch]; + + fn first_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("first_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "testing.shared-name", + phases: FETCH_ONLY, + })) + .build(), + )) + } + + fn second_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("second_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "testing.shared-name", + phases: FETCH_ONLY, + })) + .build(), + )) + } + + fn misnamed_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("misnamed_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "Not A Name", + phases: FETCH_ONLY, + })) + .build(), + )) + } + + fn phaseless_supplier( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("phaseless_supplier") + .without_js() + .with_middleware(Arc::new(Named { + id: "testing.phaseless", + phases: &[], + })) + .build(), + )) + } + + /// A builder for `register`, selected in `settings` under a module name + /// made from its id. + fn selected_supplier( + settings: &mut Settings, + id: &'static str, + module: &'static str, + register: crate::integrations::IntegrationBuilderFn, + ) -> crate::integrations::IntegrationBuilder { + settings.select_module("testing", module); + crate::integrations::IntegrationBuilder::new( + id, + "a-vendor-crate", + register, + validate_nothing, ) - .expect("should create registry"); + .with_module_name(module) + } - let all = registry.js_module_ids(); - let mut recombined = registry.js_module_ids_immediate(); - recombined.extend(registry.js_module_ids_deferred()); - recombined.sort_unstable(); + #[test] + fn two_modules_supplying_one_middleware_name_are_refused() { + let mut settings = crate::test_support::tests::create_test_settings(); + let extra = [ + selected_supplier( + &mut settings, + "first_supplier", + "testing.first-supplier", + first_supplier, + ), + selected_supplier( + &mut settings, + "second_supplier", + "testing.second-supplier", + second_supplier, + ), + ]; - let mut all_sorted = all; - all_sorted.sort_unstable(); + let error = IntegrationRegistry::with_registrations(&settings, &extra) + .err() + .expect("should refuse one middleware name supplied twice"); + + let message = error.to_string(); + assert!( + message.contains("`first_supplier` and `second_supplier` both supply a middleware") + && message.contains("`testing.shared-name`"), + "should name both suppliers and the name: {message}" + ); + } + + fn marking_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("marking") + .without_js() + .with_bundle_tag_attribute("data-example-mode", "first") + .with_bundle_tag_attribute("data-example-flag", "true") + .build(), + )) + } + + fn disagreeing_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("disagreeing") + .without_js() + .with_bundle_tag_attribute("data-example-mode", "second") + .with_bundle_tag_attribute("data-example-other", "kept") + .build(), + )) + } + + fn unsafe_name_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("unsafe_name") + .without_js() + .with_bundle_tag_attribute("onload=alert(1) data-x", "true") + .build(), + )) + } + + fn unsafe_value_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("unsafe_value") + .without_js() + .with_bundle_tag_attribute("data-example-mode", "\"> + "#; + + let mut plain = create_html_processor(config()); + let without = String::from_utf8( + crate::streaming_processor::StreamProcessor::process_chunk( + &mut plain, + html.as_bytes(), + true, + ) + .expect("should process the document"), + ) + .expect("should leave the document UTF-8"); + + assert_eq!(page(&chain_of(Vec::new()), html), without); +} + +#[test] +fn head_markup_goes_ahead_of_the_bundle_in_the_order_the_middleware_are_named() { + let chain = chain_of(vec![ + head("example.first", ""), + head("example.second", ""), + ]); + + let page = page(&chain, PAGE); + + let first = index_of(&page, "name=\"first\""); + let second = index_of(&page, "name=\"second\""); + let bundle = index_of(&page, BUNDLE_TAG); + let title = index_of(&page, ""); + assert!( + first < second && second < bundle && bundle < title, + "should write head markup in the order named, ahead of the bundle and of the \ + page's own head: {page}" + ); + assert_eq!( + page.matches("name=\"first\"").count(), + 1, + "should write head markup once" + ); +} + +#[test] +fn markup_for_after_the_bundle_goes_between_the_bundle_and_the_page_s_own_head() { + let chain = chain_of(vec![middleware("example.after", |_| MiddlewareAction { + after_bundle_inserts: vec!["<script id=\"after\"></script>".to_owned()], + ..MiddlewareAction::pass() + })]); + + let page = page(&chain, PAGE); + + let bundle = index_of(&page, BUNDLE_TAG); + let after = index_of(&page, "id=\"after\""); + let title = index_of(&page, "<title>"); + assert!( + bundle < after && after < title, + "should write the markup straight after the bundle: {page}" + ); +} + +#[test] +fn a_page_with_no_head_gets_no_head_markup() { + let chain = chain_of(vec![head("example.first", "<meta name=\"first\">")]); + + let page = page(&chain, "<p>a fragment</p>"); + + assert_eq!( + page, "<p>a fragment</p>", + "should leave a document with no head as it arrived" + ); +} + +#[test] +fn an_element_handler_is_asked_about_an_address_as_it_will_be_served() { + let chain = chain_of(vec![element("example.seen", "a.x", "href", |matched| { + AttributeRewriteAction::replace(format!( + "seen:{}:{}:{}", + matched.element_name, matched.attribute_name, matched.value + )) + })]); + + let page = page( + &chain, + r#"<html><body><a class="x" href="https://origin.example.com/page">x</a><a href="https://origin.example.com/other">y</a></body></html>"#, + ); + + assert!( + page.contains(r#"<a class="x" href="seen:a:href:https://test.example.com/page">"#), + "should hand the handler the address after core moved it to the reader's host: {page}" + ); + assert!( + page.contains(r#"<a href="https://test.example.com/other">"#), + "should leave an element the selector does not match to core: {page}" + ); +} + +#[test] +fn one_decision_can_be_asked_about_each_of_several_attributes() { + let chain = chain_of(vec![middleware("example.each", |_| { + let decide: Rc<AttributeRewriteFn> = Rc::new(|matched| { + if matched.value == "/vendor.js" { + AttributeRewriteAction::replace(format!( + "/first-party.js#{}", + matched.attribute_name + )) + } else { + AttributeRewriteAction::keep() + } + }); + MiddlewareAction { + element_handlers: AttributeRewrite::each(&["src", "href"], &decide), + ..MiddlewareAction::pass() + } + })]); + + let page = page( + &chain, + r#"<html><body><script src="/vendor.js"></script><link href="/vendor.js"><img src="/other.png"><a data-src="/vendor.js">x</a></body></html>"#, + ); + + assert!( + page.contains(r#"<script src="/first-party.js#src"></script>"#) + && page.contains(r#"<link href="/first-party.js#href">"#), + "should ask the one decision about each attribute named: {page}" + ); + assert!( + page.contains(r#"<img src="/other.png">"#) && page.contains(r#"<a data-src="/vendor.js">"#), + "should leave another value and another attribute alone: {page}" + ); +} + +#[test] +fn a_module_s_tests_can_call_a_middleware_with_a_test_page_s_context() { + use crate::html_processor::test_support::{ORIGIN_HOST, REQUEST_HOST}; + + let document_state = crate::integrations::IntegrationDocumentState::default(); + let context = test_support::context(MiddlewarePhase::Serve, &document_state); + + assert_eq!(context.phase, MiddlewarePhase::Serve); + assert_eq!( + ( + context.request_scheme, + context.request_host, + context.origin_host + ), + ("https", REQUEST_HOST, ORIGIN_HOST), + "should describe the page the recording helpers serve" + ); +} + +#[test] +fn an_element_handler_can_remove_the_element() { + let chain = chain_of(vec![element("example.remove", "a.gone", "href", |_| { + AttributeRewriteAction::remove_element() + })]); + + let page = page( + &chain, + r#"<html><body><a class="gone" href="/x">removed</a><a class="gone">no address</a><a href="/y">kept</a></body></html>"#, + ); + + assert!( + !page.contains("removed"), + "should remove the element and what it holds: {page}" + ); + assert!( + page.contains(r#"<a class="gone">no address</a>"#), + "should leave an element that carries no such attribute as it is: {page}" + ); + assert!(page.contains(r#"<a href="/y">kept</a>"#)); +} + +#[test] +fn handlers_on_one_element_are_asked_in_order_each_seeing_the_last_value() { + let chain = chain_of(vec![ + element("example.one", "a", "href", |_| { + AttributeRewriteAction::replace("/one") + }), + element("example.two", "a[href]", "href", |matched| { + AttributeRewriteAction::replace(format!("{}/two", matched.value)) + }), + ]); + + let page = page( + &chain, + r#"<html><body><a href="/start">x</a></body></html>"#, + ); + + assert!( + page.contains(r#"<a href="/one/two">"#), + "should hand the second handler the value the first left: {page}" + ); +} + +#[test] +fn an_element_an_earlier_handler_removed_is_not_asked_about_again() { + let asked = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&asked); + let chain = chain_of(vec![ + element("example.remove", "a", "href", |_| { + AttributeRewriteAction::remove_element() + }), + element("example.count", "a", "href", move |_| { + counted.fetch_add(1, Ordering::SeqCst); + AttributeRewriteAction::keep() + }), + ]); + + let page = page(&chain, r#"<html><body><a href="/x">x</a></body></html>"#); + + assert!(!page.contains("<a "), "should remove the element: {page}"); + assert_eq!( + asked.load(Ordering::SeqCst), + 0, + "should not ask a later handler about an element that is already gone" + ); +} + +#[test] +fn a_text_handler_is_handed_the_text_as_the_one_before_it_left_it() { + let chain = chain_of(vec![ + text("example.one", "script.x", || { + |text: &str, _is_last: bool| { + if text.contains("alpha") { + ScriptRewriteAction::replace(text.replace("alpha", "beta")) + } else { + ScriptRewriteAction::keep() + } + } + }), + text("example.two", "script", || { + |text: &str, _is_last: bool| { + if text.contains("beta") { + ScriptRewriteAction::replace(text.replace("beta", "gamma")) + } else { + ScriptRewriteAction::keep() + } + } + }), + ]); + + let page = page( + &chain, + r#"<html><body><script class="x">var a = "alpha";</script><script>var b = "alpha";</script></body></html>"#, + ); + + assert!( + page.contains(r#"<script class="x">var a = "gamma";</script>"#), + "should hand the second handler what the first wrote: {page}" + ); + assert!( + page.contains(r#"<script>var b = "alpha";</script>"#), + "should leave a script only the second matches, which holds nothing it changes: {page}" + ); +} + +#[test] +fn text_a_handler_removed_is_empty_for_the_next_which_may_write_in_its_place() { + let chain = chain_of(vec![ + text("example.remove", "script", || { + |_text: &str, _is_last: bool| ScriptRewriteAction::remove_node() + }), + text("example.write", "script", || { + |text: &str, is_last: bool| { + assert_eq!(text, "", "should hand over nothing where text was removed"); + if is_last { + ScriptRewriteAction::replace("written()") + } else { + ScriptRewriteAction::keep() + } + } + }), + ]); + + let page = page( + &chain, + "<html><body><script>removed()</script></body></html>", + ); + + assert!( + page.contains("<script>written()</script>"), + "should write the later handler's text in place of what was removed: {page}" + ); +} + +#[test] +fn a_text_handler_s_replacement_is_written_as_it_is() { + let chain = chain_of(vec![text("example.script", "script", || { + |text: &str, _is_last: bool| { + if text.is_empty() { + ScriptRewriteAction::keep() + } else { + ScriptRewriteAction::replace("if (a && b < c || d > e) { go(\"?x=1&y=2\"); }") + } + } + })]); + + let page = page(&chain, "<html><body><script>old()</script></body></html>"); + + assert!( + page.contains("<script>if (a && b < c || d > e) { go(\"?x=1&y=2\"); }</script>"), + "should write a script's text with no character escaped: {page}" + ); +} + +#[test] +fn a_handler_that_holds_a_script_writes_it_back_whole() { + // The script reaches the handler in pieces, because the document is read + // seven bytes at a time, and the word to change lies across pieces. + let chain = chain_of(vec![text("example.hold", "script", || { + let mut held = String::new(); + move |text: &str, is_last: bool| { + held.push_str(text); + if !is_last { + return ScriptRewriteAction::remove_node(); + } + ScriptRewriteAction::replace(std::mem::take(&mut held).replace("ORIGINAL", "CHANGED")) + } + })]); + + let page = page_with( + config(), + &chain, + "<html><body><script>one(); two(\"ORIGINAL\"); three();</script><p>after</p></body></html>", + 7, + ); + + assert!( + page.contains("<script>one(); two(\"CHANGED\"); three();</script><p>after</p>"), + "should write the whole script once, in order, with the word changed: {page}" + ); +} + +/// Appends a comment to the document, and swaps one word for another. +struct Rewording { + from: &'static str, + to: &'static str, + comment: &'static str, +} + +impl crate::streaming_processor::StreamProcessor for Rewording { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> io::Result<Vec<u8>> { + let mut text = String::from_utf8_lossy(chunk).replace(self.from, self.to); + if is_last { + text.push_str(self.comment); + } + Ok(text.into_bytes()) + } +} + +#[test] +fn a_stream_processor_is_handed_the_document_the_handlers_left() { + let chain = chain_of(vec![ + middleware("example.first", |_| MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::matching( + "a", + "href", + Rc::new(|_| AttributeRewriteAction::replace("/from-handler")), + ))], + stream: Some(Box::new(Rewording { + from: "/from-handler", + to: "/from-first", + comment: "<!--first-->", + })), + ..MiddlewareAction::pass() + }), + middleware("example.second", |_| MiddlewareAction { + stream: Some(Box::new(Rewording { + from: "/from-first", + to: "/from-second", + comment: "<!--second-->", + })), + ..MiddlewareAction::pass() + }), + ]); + + let page = page( + &chain, + r#"<html><body><a href="/start">x</a></body></html>"#, + ); + + assert!( + page.contains(r#"<a href="/from-second">"#), + "should hand each processor what the handlers and the processor before it left: {page}" + ); + assert!( + page.ends_with("<!--first--><!--second-->"), + "should run the processors in the order their middleware are named: {page}" + ); +} + +/// What one middleware leaves in the document's state for another. +struct Shared(&'static str); + +#[test] +fn the_middleware_of_one_document_share_its_state_and_see_nothing_a_request_left() { + const KEY: &str = "example"; + let chain = chain_of(vec![ + middleware("example.leaves", |context| { + context + .document_state + .get_or_insert_with(KEY, || Shared("left by the first")); + MiddlewareAction::pass() + }), + middleware("example.reads", |context| { + let shared = context + .document_state + .get::<Shared>(KEY) + .map_or("nothing", |shared| shared.0); + let request_mark = context + .document_state + .get::<request_fixture::Mark>(request_fixture::ID) + .is_some(); + MiddlewareAction { + head_inserts: vec![format!( + "<meta name=\"state\" content=\"{shared}; request mark {request_mark}\">" + )], + ..MiddlewareAction::pass() + } + }), + ]); + // The request left a mark, which a serve middleware may read. + let config = config().with_request_state(request_fixture::marked()); + + let page = page_with(config, &chain, PAGE, 8192); + + assert!( + page.contains("content=\"left by the first; request mark false\""), + "should share the document's state along the chain, with nothing of the request in \ + it: {page}" + ); +} + +#[test] +fn each_document_is_planned_afresh() { + let created = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&created); + let chain = chain_of(vec![middleware("example.counts", move |context| { + let nth = counted.fetch_add(1, Ordering::SeqCst) + 1; + let earlier = context.document_state.get::<Shared>("example").is_some(); + context + .document_state + .get_or_insert_with("example", || Shared("this document")); + MiddlewareAction { + head_inserts: vec![format!( + "<meta name=\"nth\" content=\"{nth}; earlier state {earlier}\">" + )], + ..MiddlewareAction::pass() + } + })]); + + let first = page(&chain, PAGE); + let second = page(&chain, PAGE); + + assert!(first.contains("content=\"1; earlier state false\"")); + assert!( + second.contains("content=\"2; earlier state false\""), + "should ask the middleware again for the next document, with a state of its own: \ + {second}" + ); + assert_eq!(created.load(Ordering::SeqCst), 2); +} + +#[test] +fn a_middleware_is_told_the_phase_the_hosts_and_how_much_it_may_hold() { + let chain = chain_of(vec![middleware("example.context", |context| { + MiddlewareAction { + head_inserts: vec![format!( + "<meta name=\"context\" content=\"{} {}://{} from {} holding {}\">", + context.phase, + context.request_scheme, + context.request_host, + context.origin_host, + context.max_buffered_script_bytes + )], + ..MiddlewareAction::pass() + } + })]); + let most = create_test_settings().publisher.max_buffered_body_bytes; + + let page = page(&chain, PAGE); + + assert!( + page.contains(&format!( + "content=\"fetch https://{REQUEST_HOST} from {ORIGIN_HOST} holding {most}\"" + )), + "should tell the middleware about the document: {page}" + ); +} + +#[test] +fn a_selector_that_does_not_parse_refuses_the_document_naming_the_middleware() { + let chains = [ + chain_of(vec![ + head("example.fine", "<meta name=\"fine\">"), + element("example.broken", "a[", "href", |_| { + AttributeRewriteAction::keep() + }), + ]), + chain_of(vec![text("example.broken", "a[", || { + |_text: &str, _is_last: bool| ScriptRewriteAction::keep() + })]), + ]; + + for chain in chains { + let message = create_html_processor_with_middleware(config(), &chain, None) + .err() + .expect("should refuse to plan the document"); + + assert!( + message.contains("the fetch middleware `example.broken` cannot run") + && message.contains("`a[` is not a usable selector"), + "should name the middleware and the selector: {message}" + ); + } +} + +#[test] +fn a_fetch_middleware_that_writes_a_response_header_is_refused() { + let chain = chain_of(vec![header("example.version", "x-example", "1")]); + + let message = create_html_processor_with_middleware(config(), &chain, None) + .err() + .expect("should refuse to plan the document"); + + assert!( + message.contains("the fetch middleware `example.version` cannot run") + && message.contains("writes a response header"), + "should name the middleware and the reason: {message}" + ); +} + +#[test] +fn a_serve_middleware_s_headers_are_written_on_the_reader_s_response() { + let chain = serve_chain_of(vec![ + header("example.a", "x-example", "a"), + header("example.b", "x-example", "b"), + header("example.c", "x-other", "c"), + ]); + + let pass = create_serve_processor(&chain, &reader(&IntegrationRequestState::default())) + .expect("should plan the reader's middleware"); + let mut response = http::Response::new(()); + write_response_headers(&mut response, pass.response_headers); + + assert!( + pass.processor.is_none(), + "should leave the document as it arrived" + ); + assert_eq!( + response + .headers() + .get("x-example") + .map(HeaderValue::as_bytes), + Some(b"b".as_slice()), + "a later middleware's header should replace an earlier one's" + ); + assert_eq!( + response.headers().get("x-other").map(HeaderValue::as_bytes), + Some(b"c".as_slice()), + "should write a header no other middleware wrote" + ); +} + +#[test] +fn the_inline_path_hands_back_the_serve_chain_s_headers() { + let serve = serve_chain_of(vec![header("example.version", "x-example", "1")]); + + let (_, headers) = + create_html_processor_with_middleware(config(), &chain_of(Vec::new()), Some(&serve)) + .expect("should plan the document"); + + assert_eq!( + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect::<Vec<_>>(), + [("x-example", b"1".as_slice())], + "should hand back the serve chain's headers for the response" + ); +} + +#[test] +fn an_action_with_only_a_header_is_not_a_pass() { + let action = MiddlewareAction { + response_headers: vec![( + HeaderName::from_static("x-example"), + HeaderValue::from_static("1"), + )], + ..MiddlewareAction::pass() + }; + + assert!(!action.is_pass(), "a header is a change to the response"); + assert!(MiddlewareAction::pass().is_pass()); +} + +#[test] +fn a_middleware_is_told_the_platform_the_deployment_runs_on() { + let platform = PlatformIdentity { + service_version: "39".to_owned(), + ..PlatformIdentity::named("example-platform") + }; + let document_state = IntegrationDocumentState::default(); + + let context = test_support::context_on(MiddlewarePhase::Serve, &document_state, &platform); + + assert_eq!(context.platform.name, "example-platform"); + assert_eq!(context.platform.service_version, "39"); + assert_eq!( + test_support::context(MiddlewarePhase::Serve, &document_state) + .platform + .name, + "test-platform", + "a test context should name the test platform" + ); +} + +#[test] +fn a_chain_reports_its_phase_and_the_names_in_the_order_they_run() { + let chain = chain_of(vec![ + head("example.b", "<meta name=\"b\">"), + head("example.a", "<meta name=\"a\">"), + ]); + + assert_eq!(chain.phase(), MiddlewarePhase::Fetch); + assert_eq!(chain.ids(), ["example.b", "example.a"]); + assert!(!chain.is_empty()); + assert!(chain_of(Vec::new()).is_empty()); +} + +// --------------------------------------------------------------------------- +// What the document pipeline does with a serve chain +// --------------------------------------------------------------------------- + +/// [`PAGE`] as core leaves it for a store, with the script bundle in its +/// head and one link. +fn rewritten_page() -> String { + page( + &chain_of(Vec::new()), + r#"<html><head><title>Pagex"#, + ) +} + +#[test] +fn a_serve_chain_writes_head_markup_straight_before_the_bundle_and_the_rest_straight_after() { + let chain = serve_chain_of(vec![ + middleware("example.first", |_| MiddlewareAction { + head_inserts: vec!["".to_owned()], + after_bundle_inserts: vec!["".to_owned()], + ..MiddlewareAction::pass() + }), + middleware("example.second", |_| MiddlewareAction { + head_inserts: vec!["".to_owned()], + after_bundle_inserts: vec!["".to_owned()], + ..MiddlewareAction::pass() + }), + ]); + let rewritten = rewritten_page(); + + let page = served(&chain, &rewritten); + + assert!( + page.contains("" + ), + "should write the markup for after the bundle straight after it: {page}" + ); + assert_eq!( + page.matches(BUNDLE_TAG).count(), + 1, + "should write no second bundle: {page}" + ); + assert_eq!( + page.replace("", "") + .replace( + "", + "" + ), + rewritten, + "should change nothing else of a document core has already rewritten" + ); +} + +#[test] +fn a_serve_chain_writes_head_markup_once_where_the_page_repeats_the_bundle_s_id() { + let chain = serve_chain_of(vec![head("example.first", "")]); + let rewritten = rewritten_page().replace( + "", + "", + ); + + let page = served(&chain, &rewritten); + + assert_eq!( + page.matches("name=\"first\"").count(), + 1, + "should write head markup at core's tag alone: {page}" + ); + assert!( + index_of(&page, "name=\"first\"") < index_of(&page, ""), + "should write it in the head: {page}" + ); +} + +#[test] +fn a_reader_s_copy_with_no_bundle_gets_no_head_markup() { + let chain = serve_chain_of(vec![head("example.first", "<meta name=\"first\">")]); + + let page = served(&chain, "<p>a fragment</p>"); + + assert_eq!(page, "<p>a fragment</p>"); +} + +#[test] +fn a_serve_chain_s_handlers_work_on_the_page_as_core_left_it() { + let chain = serve_chain_of(vec![ + element("example.link", "a", "href", |matched| { + AttributeRewriteAction::replace(format!("{}?reader=1", matched.value)) + }), + text("example.script", "script", || { + |text: &str, _is_last: bool| { + if text.contains("var a = 1;") { + ScriptRewriteAction::replace(text.replace("var a = 1;", "var a = 2 && 3;")) + } else { + ScriptRewriteAction::keep() + } + } + }), + ]); + + let page = served(&chain, &rewritten_page()); + + assert!( + page.contains(r#"<a href="https://test.example.com/x?reader=1">"#), + "should hand an element handler the address core already moved: {page}" + ); + assert!( + page.contains("<script>var a = 2 && 3;</script>"), + "should write a text handler's replacement as it is: {page}" + ); +} + +#[test] +fn a_serve_chain_s_stream_processor_runs_over_the_reader_s_copy() { + let chain = serve_chain_of(vec![middleware("example.stream", |_| MiddlewareAction { + stream: Some(Box::new(Rewording { + from: "<title>Page", + to: "Reader", + comment: "", + })), + ..MiddlewareAction::pass() + })]); + + let page = served(&chain, &rewritten_page()); + + assert!(page.contains("Reader") && page.ends_with("")); +} + +#[test] +fn a_serve_middleware_reads_what_the_request_left() { + let chain = serve_chain_of(vec![middleware("example.reads", |context| { + let marked = context + .document_state + .get::(request_fixture::ID) + .is_some(); + MiddlewareAction { + head_inserts: vec![format!( + "", + context.phase, context.request_scheme, context.request_host, context.origin_host + )], + ..MiddlewareAction::pass() + } + })]); + let rewritten = rewritten_page(); + + let marked = served_with(&chain, &request_fixture::marked(), &rewritten, 8192); + let unmarked = served(&chain, &rewritten); + + let told = format!("serve https://{REQUEST_HOST} from {ORIGIN_HOST}"); + assert!( + marked.contains(&format!("content=\"{told}; marked true\"")), + "should hand a serve middleware what the request left: {marked}" + ); + assert!( + unmarked.contains(&format!("content=\"{told}; marked false\"")), + "should hand it nothing where the request left nothing: {unmarked}" + ); +} + +#[test] +fn a_serve_chain_with_nothing_to_do_makes_no_processor() { + let state = IntegrationRequestState::default(); + + let none = create_serve_processor(&serve_chain_of(Vec::new()), &reader(&state)) + .expect("should plan an empty chain"); + assert!( + none.processor.is_none(), + "should make no processor for no middleware" + ); + + let passes = serve_chain_of(vec![middleware("example.pass", |_| { + MiddlewareAction::pass() + })]); + let none = + create_serve_processor(&passes, &reader(&state)).expect("should plan a chain that passes"); + assert!( + none.processor.is_none(), + "should make no processor where every middleware leaves the page alone" + ); +} + +#[test] +fn a_serve_selector_that_does_not_parse_refuses_the_reader_s_copy() { + let chain = serve_chain_of(vec![element("example.broken", "a[", "href", |_| { + AttributeRewriteAction::keep() + })]); + + let message = create_serve_processor(&chain, &reader(&IntegrationRequestState::default())) + .err() + .expect("should refuse to plan the reader's copy"); + + assert!( + message.contains("the serve middleware `example.broken` cannot run") + && message.contains("`a[` is not a usable selector"), + "should name the phase, the middleware and the selector: {message}" + ); +} + +#[test] +fn on_a_page_that_is_not_stored_the_serve_chain_follows_the_fetch_chain_on_the_one_pass() { + let fetch = chain_of(vec![ + head("example.fetch-head", ""), + element("example.fetch-link", "a", "href", |_| { + AttributeRewriteAction::replace("/fetch") + }), + ]); + let serve = serve_chain_of(vec![ + head("example.serve-head", ""), + element("example.serve-link", "a", "href", |matched| { + AttributeRewriteAction::replace(format!("{}/serve", matched.value)) + }), + ]); + let html = + r#"Pagex"#; + + // Read whole, and then seven bytes at a time, so the second rewriter is + // fed the first one's output in pieces. + for chunk_size in [8192, 7] { + let (processor, _) = create_html_processor_with_middleware(config(), &fetch, Some(&serve)) + .expect("should plan both chains"); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size, + }, + processor, + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should process the document"); + let page = String::from_utf8(output).expect("should leave the document UTF-8"); + + assert!( + page.contains(r#""#), + "should hand the serve chain the page as the fetch chain left it: {page}" + ); + let fetch_head = index_of(&page, "name=\"fetch\""); + let serve_head = index_of(&page, "name=\"serve\""); + assert!( + fetch_head < serve_head && serve_head < index_of(&page, BUNDLE_TAG), + "should write the serve chain's head markup after the fetch chain's and ahead of \ + the bundle: {page}" + ); + } +} + +// --------------------------------------------------------------------------- +// The settings +// --------------------------------------------------------------------------- + +fn settings_with_document(extra: &str) -> Result { + let document = format!( + "{}\n{extra}", + crate::test_support::tests::crate_test_settings_str() + ); + crate::settings::Settings::from_toml(&document).map_err(|error| format!("{error:?}")) +} + +#[test] +fn fetch_entries_are_read_from_the_settings_in_the_order_written() { + let settings = settings_with_document( + r#" +[[fetch]] +media_type = "text/html" +path = "/news/" +middleware = ["example.news", "example.all"] + +[[fetch]] +media_type = "text/html" +middleware = ["example.all"] +"#, + ) + .expect("should read settings with fetch entries"); + + assert_eq!( + settings.phase_entries(MiddlewarePhase::Fetch), + &PhaseEntries::new(vec![ + entry(Some("/news/"), &["example.news", "example.all"]), + entry(None, &["example.all"]), + ]) + ); +} + +#[test] +fn serve_entries_are_read_from_the_settings_beside_the_fetch_entries() { + let settings = settings_with_document( + r#" +[[fetch]] +media_type = "text/html" +middleware = ["example.all"] + +[[serve]] +media_type = "text/html" +path = "/news/" +middleware = ["example.reader"] +"#, + ) + .expect("should read settings with an entry in each phase"); + + assert_eq!( + settings.phase_entries(MiddlewarePhase::Fetch), + &PhaseEntries::new(vec![entry(None, &["example.all"])]) + ); + assert_eq!( + settings.phase_entries(MiddlewarePhase::Serve), + &PhaseEntries::new(vec![entry(Some("/news/"), &["example.reader"])]) + ); + let written = serde_json::to_value(&settings).expect("should write the settings"); + assert_eq!( + written.get("serve"), + Some(&json!([ + { "media_type": "text/html", "path": "/news/", "middleware": ["example.reader"] } + ])), + "should write the serve entries a document holds" + ); +} + +#[test] +fn settings_with_a_misshapen_serve_entry_are_refused_naming_the_phase() { + let message = + settings_with_document("\n[[serve]]\nmedia_type = \"text/html\"\nmiddleware = []\n") + .expect_err("should refuse the settings"); + + assert!( + message.contains("[[serve]] entry 1 names no middleware"), + "should name the phase and the entry: {message}" + ); +} + +#[test] +fn settings_without_entries_write_no_fetch_key() { + let settings = settings_with_document("").expect("should read settings with no entries"); + let written = serde_json::to_value(&settings).expect("should write the settings"); + assert!( + written.get("fetch").is_none() && written.get("serve").is_none(), + "should leave both keys out of a stored document that wrote no entries" + ); + + let settings = settings_with_document( + "\n[[fetch]]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\n", + ) + .expect("should read settings with one entry"); + let written = serde_json::to_value(&settings).expect("should write the settings"); + assert_eq!( + written.get("fetch"), + Some(&json!([{ "media_type": "text/html", "middleware": ["example.all"] }])), + "should write the entries a document holds" + ); +} + +#[test] +fn settings_with_a_misshapen_entry_are_refused() { + let cases = [ + ( + "\n[[fetch]]\nmedia_type = \"image/png\"\nmiddleware = [\"example.all\"]\n", + "[[fetch]] entry 1 covers `image/png`", + ), + ( + "\n[fetch]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\n", + "A phase is a list of entries, each written as [[fetch]]", + ), + ( + "\n[[fetch]]\nmedia_type = \"text/html\"\nmiddleware = [\"example.all\"]\nenabled = true\n", + "unknown field `enabled`", + ), + ( + "\n[[fetch]]\nmedia_type = \"text/html\"\n", + "missing field `middleware`", + ), + ]; + + for (document, expected) in cases { + let message = settings_with_document(document).expect_err("should refuse the settings"); + assert!( + message.contains(expected), + "should say `{expected}`: {message}" + ); + } +} + +#[test] +fn deploy_validation_refuses_a_misshapen_entry() { + let mut settings = create_test_settings(); + settings.fetch = PhaseEntries::new(vec![ + entry(None, &["example.all"]), + entry(Some("/news/"), &["example.news"]), + ]); + + let error = crate::config::validate_settings_for_deploy(&settings) + .expect_err("should refuse entries with one that is never reached"); + + assert!( + error + .to_string() + .contains("[[fetch]] entry 2 is never reached"), + "should name the entry: {error}" + ); +} + +#[test] +fn the_making_entry_has_a_path_covering_the_request_and_names_a_middleware() { + let entries = PhaseEntries::new(vec![ + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads.txt".to_owned()), + middleware: Vec::new(), + }, + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads".to_owned()), + middleware: vec!["example.sellers".to_owned()], + }, + PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec!["example.page".to_owned()], + }, + ]); + + assert_eq!( + entries + .making("/ads.txt") + .map(|entry| entry.middleware.as_slice()), + Some(&["example.sellers".to_owned()][..]), + "should pass the entry naming no middleware and take the first that names one" + ); + assert!( + entries.making("/robots.txt").is_none(), + "should make nothing where no entry with a path covers the request, whatever covers every path" + ); +} + +#[test] +fn what_is_prepared_is_taken_once_by_name_and_as_its_type() { + let prepared = Prepared::default(); + assert!(prepared.is_empty()); + let fetch = MiddlewarePhase::Fetch; + prepared.insert(fetch, "example.one", Box::new(String::from("one"))); + prepared.insert(fetch, "example.two", Box::new(7_u32)); + + assert_eq!( + prepared.take::(fetch, "example.one"), + None, + "should leave a value of another type where it is" + ); + assert_eq!( + prepared.take::(fetch, "example.one").as_deref(), + Some("one") + ); + assert_eq!( + prepared.take::(fetch, "example.one"), + None, + "should give what was prepared once" + ); + assert_eq!(prepared.take::(fetch, "example.two"), Some(7)); + assert!(prepared.is_empty()); +} + +/// Prepares the name it is called as, and writes it into the head. +struct Prepares; + +impl Middleware for Prepares { + fn middleware_id(&self) -> &'static str { + "example.prepares" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + Ok(Box::new(call.module().to_owned()) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let prepared: String = context + .take_prepared("example.prepares") + .unwrap_or_else(|| "nothing prepared".to_owned()); + MiddlewareAction { + head_inserts: vec![prepared], + ..MiddlewareAction::pass() + } + } +} + +/// Prepares nothing. +struct Plain; + +impl Middleware for Plain { + fn middleware_id(&self) -> &'static str { + "example.plain" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } +} + +/// Refuses every request. +struct Refuses; + +impl Middleware for Refuses { + fn middleware_id(&self) -> &'static str { + "example.refuses" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn prepare<'a>( + &'a self, + _call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async { + Err(Refusal::service_unavailable( + Duration::from_secs(5), + "later", + )) + })) + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction::pass() + } +} + +#[tokio::test] +async fn a_chain_keeps_what_each_middleware_prepares_under_its_name() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + vec![Arc::new(Plain), Arc::new(Prepares)], + ); + let document_state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(MiddlewarePhase::Fetch, &document_state) + }; + + chain + .prepare(ModuleContext::empty(), &context) + .await + .expect("should prepare"); + let plan = chain.plan(&context).expect("should plan"); + + assert_eq!( + plan.head_inserts, + vec!["example.prepares".to_owned()], + "should hand create what the middleware prepared, called as its own name" + ); + assert!( + prepared.is_empty(), + "should have given the preparation to create" + ); +} + +/// Prepares the phase it is called in, and writes it into the head. +struct PreparesItsPhase; + +impl Middleware for PreparesItsPhase { + fn middleware_id(&self) -> &'static str { + "example.both" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + _call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + let phase = context.phase.to_string(); + Some(Box::pin(async move { + Ok(Box::new(phase) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + head_inserts: vec![ + context + .take_prepared::("example.both") + .unwrap_or_else(|| "nothing prepared".to_owned()), + ], + ..MiddlewareAction::pass() + } + } +} + +#[tokio::test] +async fn a_middleware_preparing_in_both_phases_keeps_what_each_phase_prepared() { + let prepared = Prepared::default(); + let document_state = IntegrationDocumentState::default(); + let chains: Vec<(MiddlewarePhase, MiddlewareChain)> = MiddlewarePhase::ALL + .into_iter() + .map(|phase| { + ( + phase, + MiddlewareChain::new(phase, HTML_MEDIA_TYPE, vec![Arc::new(PreparesItsPhase)]), + ) + }) + .collect(); + // A document that is not stored is prepared in both phases before + // either phase is planned, so both preparations share one store. + for (phase, chain) in &chains { + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(*phase, &document_state) + }; + chain + .prepare(ModuleContext::empty(), &context) + .await + .expect("should prepare"); + } + let mut plans = Vec::new(); + for (phase, chain) in &chains { + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(*phase, &document_state) + }; + plans.push(chain.plan(&context).expect("should plan").head_inserts); + } + + assert_eq!( + plans, + vec![vec!["fetch".to_owned()], vec!["serve".to_owned()]], + "should hand each phase what was prepared in it" + ); + assert!(prepared.is_empty(), "should have given every preparation"); +} + +#[tokio::test] +async fn the_first_refusal_ends_the_preparation() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + vec![Arc::new(Refuses), Arc::new(Prepares)], + ); + let document_state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = MiddlewareContext { + prepared: &prepared, + ..test_support::context(MiddlewarePhase::Fetch, &document_state) + }; + + let refusal = chain + .prepare(ModuleContext::empty(), &context) + .await + .expect_err("should refuse"); + + assert_eq!(refusal.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(refusal.retry_after, Some(Duration::from_secs(5))); + assert_eq!(refusal.message, "later"); + assert!( + prepared.is_empty(), + "should prepare nothing after the refusal" + ); +} + +#[test] +fn an_html_entry_with_a_path_makes_nothing() { + let entries = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/news/".to_owned()), + middleware: vec!["example.page".to_owned()], + }]); + + assert!( + entries.making("/news/missing").is_none(), + "should leave a page the origin does not have as the origin's answer" + ); +} + +#[test] +fn the_entry_with_the_longest_covering_path_makes_the_file() { + let entries = PhaseEntries::new(vec![ + PhaseEntry { + media_type: "text/css".to_owned(), + path: Some("/".to_owned()), + middleware: vec!["example.styles".to_owned()], + }, + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads.txt".to_owned()), + middleware: vec!["example.sellers".to_owned()], + }, + ]); + + assert_eq!( + entries + .making("/ads.txt") + .map(|entry| entry.media_type.as_str()), + Some("text/plain"), + "should make the file from the entry for its own path, not one written ahead for every path" + ); + assert_eq!( + entries + .making("/site.css") + .map(|entry| entry.media_type.as_str()), + Some("text/css"), + "should still make a file only the shorter path covers" + ); +} + +#[test] +fn an_html_entry_ahead_does_not_shadow_the_entry_that_makes_the_file() { + let entries = PhaseEntries::new(vec![ + PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/".to_owned()), + middleware: vec!["example.page".to_owned()], + }, + PhaseEntry { + media_type: "text/plain".to_owned(), + path: Some("/ads.txt".to_owned()), + middleware: vec!["example.sellers".to_owned()], + }, + ]); + + assert_eq!( + entries + .making("/ads.txt") + .map(|entry| entry.media_type.as_str()), + Some("text/plain"), + "should make the file from the text/plain entry, not as HTML" + ); +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 17b3d624f6..59deccec97 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -27,206 +27,18 @@ fn fastly_sdk_pattern() -> regex::Regex { .expect("should compile migration guard regex") } +// The list of files this guard checks is generated by `build.rs` from every +// `.rs` file under `src`, so a file added to core joins the guard on its own +// and a file that leaves core (a vendor integration moving to its own crate, +// for instance) leaves the guard without anyone editing this file. The +// generated file defines `CHECKED_SOURCES`. +// +// `migration_guards.rs` itself is left out, because the banned pattern appears +// here as the regex literal the guard matches with. +include!(concat!(env!("OUT_DIR"), "/migration_guard_sources.rs")); + fn checked_sources() -> &'static [(&'static str, &'static str)] { - &[ - ( - "asset_image_optimizer.rs", - include_str!("asset_image_optimizer.rs"), - ), - ("auction/config.rs", include_str!("auction/config.rs")), - ("auction/context.rs", include_str!("auction/context.rs")), - ("auction/endpoints.rs", include_str!("auction/endpoints.rs")), - ("auction/formats.rs", include_str!("auction/formats.rs")), - ("auction/mod.rs", include_str!("auction/mod.rs")), - ( - "auction/orchestrator.rs", - include_str!("auction/orchestrator.rs"), - ), - ("auction/provider.rs", include_str!("auction/provider.rs")), - ( - "auction/test_support.rs", - include_str!("auction/test_support.rs"), - ), - ("auction/types.rs", include_str!("auction/types.rs")), - ( - "auction_config_types.rs", - include_str!("auction_config_types.rs"), - ), - ("auth.rs", include_str!("auth.rs")), - ( - "consent/extraction.rs", - include_str!("consent/extraction.rs"), - ), - ("consent/gpp.rs", include_str!("consent/gpp.rs")), - ( - "consent/jurisdiction.rs", - include_str!("consent/jurisdiction.rs"), - ), - ("consent/mod.rs", include_str!("consent/mod.rs")), - ("consent/tcf.rs", include_str!("consent/tcf.rs")), - ("consent/types.rs", include_str!("consent/types.rs")), - ( - "consent/us_privacy.rs", - include_str!("consent/us_privacy.rs"), - ), - ("consent_config.rs", include_str!("consent_config.rs")), - ("constants.rs", include_str!("constants.rs")), - ("cookies.rs", include_str!("cookies.rs")), - ("creative.rs", include_str!("creative.rs")), - ("ec/auth.rs", include_str!("ec/auth.rs")), - ("ec/batch_sync.rs", include_str!("ec/batch_sync.rs")), - ("ec/consent.rs", include_str!("ec/consent.rs")), - ("ec/cookies.rs", include_str!("ec/cookies.rs")), - ("ec/device.rs", include_str!("ec/device.rs")), - ("ec/eids.rs", include_str!("ec/eids.rs")), - ("ec/finalize.rs", include_str!("ec/finalize.rs")), - ("ec/generation.rs", include_str!("ec/generation.rs")), - ("ec/identify.rs", include_str!("ec/identify.rs")), - ("ec/kv.rs", include_str!("ec/kv.rs")), - ("ec/kv_types.rs", include_str!("ec/kv_types.rs")), - ("ec/mod.rs", include_str!("ec/mod.rs")), - ("ec/partner.rs", include_str!("ec/partner.rs")), - ("ec/prebid_eids.rs", include_str!("ec/prebid_eids.rs")), - ("ec/pull_sync.rs", include_str!("ec/pull_sync.rs")), - ("ec/rate_limiter.rs", include_str!("ec/rate_limiter.rs")), - ("ec/registry.rs", include_str!("ec/registry.rs")), - ("edge_cookie.rs", include_str!("edge_cookie.rs")), - ("error.rs", include_str!("error.rs")), - ("geo.rs", include_str!("geo.rs")), - ("host_header.rs", include_str!("host_header.rs")), - ("host_rewrite.rs", include_str!("host_rewrite.rs")), - ("html_processor.rs", include_str!("html_processor.rs")), - ("http_util.rs", include_str!("http_util.rs")), - ( - "integrations/adserver_mock.rs", - include_str!("integrations/adserver_mock.rs"), - ), - ("integrations/aps.rs", include_str!("integrations/aps.rs")), - ( - "integrations/datadome.rs", - include_str!("integrations/datadome.rs"), - ), - ( - "integrations/didomi.rs", - include_str!("integrations/didomi.rs"), - ), - ( - "integrations/google_tag_manager.rs", - include_str!("integrations/google_tag_manager.rs"), - ), - ("integrations/gpt.rs", include_str!("integrations/gpt.rs")), - ( - "integrations/gpt_diagnostics.rs", - include_str!("integrations/gpt_diagnostics.rs"), - ), - ( - "integrations/lockr.rs", - include_str!("integrations/lockr.rs"), - ), - ("integrations/mod.rs", include_str!("integrations/mod.rs")), - ( - "integrations/nextjs/mod.rs", - include_str!("integrations/nextjs/mod.rs"), - ), - ( - "integrations/nextjs/rsc.rs", - include_str!("integrations/nextjs/rsc.rs"), - ), - ( - "integrations/nextjs/rsc_placeholders.rs", - include_str!("integrations/nextjs/rsc_placeholders.rs"), - ), - ( - "integrations/nextjs/rsc_stream.rs", - include_str!("integrations/nextjs/rsc_stream.rs"), - ), - ( - "integrations/nextjs/script_rewriter.rs", - include_str!("integrations/nextjs/script_rewriter.rs"), - ), - ( - "integrations/nextjs/shared.rs", - include_str!("integrations/nextjs/shared.rs"), - ), - ( - "integrations/permutive.rs", - include_str!("integrations/permutive.rs"), - ), - ( - "integrations/prebid.rs", - include_str!("integrations/prebid.rs"), - ), - ( - "integrations/registry.rs", - include_str!("integrations/registry.rs"), - ), - ( - "integrations/sourcepoint.rs", - include_str!("integrations/sourcepoint.rs"), - ), - ( - "integrations/testlight.rs", - include_str!("integrations/testlight.rs"), - ), - ("lib.rs", include_str!("lib.rs")), - ("models.rs", include_str!("models.rs")), - ("openrtb.rs", include_str!("openrtb.rs")), - ("platform/error.rs", include_str!("platform/error.rs")), - ("platform/http.rs", include_str!("platform/http.rs")), - ( - "platform/image_optimizer.rs", - include_str!("platform/image_optimizer.rs"), - ), - ("platform/kv.rs", include_str!("platform/kv.rs")), - ("platform/mod.rs", include_str!("platform/mod.rs")), - ( - "platform/test_support.rs", - include_str!("platform/test_support.rs"), - ), - ("platform/traits.rs", include_str!("platform/traits.rs")), - ("platform/types.rs", include_str!("platform/types.rs")), - ("proxy.rs", include_str!("proxy.rs")), - ("publisher.rs", include_str!("publisher.rs")), - ("redacted.rs", include_str!("redacted.rs")), - ( - "request_signing/discovery.rs", - include_str!("request_signing/discovery.rs"), - ), - ( - "request_signing/endpoints.rs", - include_str!("request_signing/endpoints.rs"), - ), - ( - "request_signing/jwks.rs", - include_str!("request_signing/jwks.rs"), - ), - ( - "request_signing/mod.rs", - include_str!("request_signing/mod.rs"), - ), - ( - "request_signing/rotation.rs", - include_str!("request_signing/rotation.rs"), - ), - ( - "request_signing/signing.rs", - include_str!("request_signing/signing.rs"), - ), - ("rsc_flight.rs", include_str!("rsc_flight.rs")), - ("s3_sigv4.rs", include_str!("s3_sigv4.rs")), - ("settings.rs", include_str!("settings.rs")), - ("settings_data.rs", include_str!("settings_data.rs")), - ( - "streaming_processor.rs", - include_str!("streaming_processor.rs"), - ), - ( - "streaming_replacer.rs", - include_str!("streaming_replacer.rs"), - ), - ("test_support.rs", include_str!("test_support.rs")), - ("tsjs.rs", include_str!("tsjs.rs")), - ] + CHECKED_SOURCES } #[test] @@ -241,3 +53,33 @@ fn core_modules_do_not_reference_the_fastly_sdk() { ); } } + +/// A generation bug that produced an empty or truncated list would leave the +/// guard above passing while checking nothing, so this pins the list to the +/// tree it is meant to cover. +#[test] +fn the_generated_list_covers_the_crate() { + let sources = checked_sources(); + + assert!( + sources.len() > 100, + "should check every core source file, found {}", + sources.len() + ); + assert!( + sources.iter().any(|&(path, _)| path == "lib.rs"), + "should check the crate root" + ); + assert!( + sources + .iter() + .any(|&(path, _)| path == "integrations/registry.rs"), + "should check files in subdirectories" + ); + assert!( + !sources + .iter() + .any(|&(path, _)| path == "migration_guards.rs"), + "should leave out the guard's own source, which holds the banned pattern" + ); +} diff --git a/crates/trusted-server-core/src/models.rs b/crates/trusted-server-core/src/models.rs deleted file mode 100644 index 87cadcb94f..0000000000 --- a/crates/trusted-server-core/src/models.rs +++ /dev/null @@ -1,285 +0,0 @@ -//! Data models for ad serving and callbacks. -//! -//! This module defines the structures used for communication with ad servers -//! and tracking callbacks. - -use serde::Deserialize; - -/// Response from an ad server containing creative details. -/// -/// Contains all the information needed to display an ad and track -/// its performance through various callbacks. -#[allow( - dead_code, - reason = "model includes fields deserialized for external ad-server compatibility" -)] -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct AdResponse { - /// Network identifier for the ad network. - pub network_id: String, - /// Site identifier where the ad will be displayed. - pub site_id: String, - /// Page identifier within the site. - pub page_id: String, - /// Format identifier for the ad format. - pub format_id: String, - /// Advertiser identifier. - pub advertiser_id: String, - /// Campaign identifier. - pub campaign_id: String, - /// Insertion order identifier. - pub insertion_id: String, - /// Creative identifier. - pub creative_id: String, - /// URL of the creative asset to display. - pub creative_url: String, - /// List of tracking callbacks for various events. - pub callbacks: Vec, -} - -/// Tracking callback for ad events. -/// -/// Represents a URL that should be called when specific ad events occur, -/// such as impressions, clicks, or viewability milestones. -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct Callback { - /// Type of callback (e.g., "impression", "click", "viewable"). - #[serde(rename = "type")] - pub callback_type: String, - /// URL to call when the event occurs. - pub url: String, -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn test_callback_deserialization() { - let json_data = json!({ - "type": "impression", - "url": "https://example.com/track/impression" - }); - - let callback: Callback = - serde_json::from_value(json_data).expect("should deserialize callback"); - assert_eq!(callback.callback_type, "impression"); - assert_eq!(callback.url, "https://example.com/track/impression"); - } - - #[test] - fn test_ad_response_full_deserialization() { - let json_data = json!({ - "networkId": "12345", - "siteId": "67890", - "pageId": "11111", - "formatId": "22222", - "advertiserId": "33333", - "campaignId": "44444", - "insertionId": "55555", - "creativeId": "66666", - "creativeUrl": "https://cdn.example.com/creative/12345.jpg", - "callbacks": [ - { - "type": "impression", - "url": "https://track.example.com/impression/12345" - }, - { - "type": "click", - "url": "https://track.example.com/click/12345" - }, - { - "type": "viewability", - "url": "https://track.example.com/viewability/12345" - } - ] - }); - - let ad_response: AdResponse = - serde_json::from_value(json_data).expect("should deserialize ad response"); - - assert_eq!(ad_response.network_id, "12345"); - assert_eq!(ad_response.site_id, "67890"); - assert_eq!(ad_response.page_id, "11111"); - assert_eq!(ad_response.format_id, "22222"); - assert_eq!(ad_response.advertiser_id, "33333"); - assert_eq!(ad_response.campaign_id, "44444"); - assert_eq!(ad_response.insertion_id, "55555"); - assert_eq!(ad_response.creative_id, "66666"); - assert_eq!( - ad_response.creative_url, - "https://cdn.example.com/creative/12345.jpg" - ); - - assert_eq!(ad_response.callbacks.len(), 3); - assert_eq!(ad_response.callbacks[0].callback_type, "impression"); - assert_eq!( - ad_response.callbacks[0].url, - "https://track.example.com/impression/12345" - ); - assert_eq!(ad_response.callbacks[1].callback_type, "click"); - assert_eq!(ad_response.callbacks[2].callback_type, "viewability"); - } - - #[test] - fn test_ad_response_empty_callbacks() { - let json_data = json!({ - "networkId": "12345", - "siteId": "67890", - "pageId": "11111", - "formatId": "22222", - "advertiserId": "33333", - "campaignId": "44444", - "insertionId": "55555", - "creativeId": "66666", - "creativeUrl": "https://cdn.example.com/creative/12345.jpg", - "callbacks": [] - }); - - let ad_response: AdResponse = serde_json::from_value(json_data) - .expect("should deserialize ad response with empty callbacks"); - assert_eq!(ad_response.callbacks.len(), 0); - } - - #[test] - fn test_ad_response_missing_field() { - // Missing required field should fail - let json_data = json!({ - "networkId": "12345", - "siteId": "67890", - // Missing pageId - "formatId": "22222", - "advertiserId": "33333", - "campaignId": "44444", - "insertionId": "55555", - "creativeId": "66666", - "creativeUrl": "https://cdn.example.com/creative/12345.jpg", - "callbacks": [] - }); - - let result: Result = serde_json::from_value(json_data); - result.unwrap_err(); - } - - #[test] - fn test_ad_response_case_sensitivity() { - // Test camelCase to snake_case conversion - let json_str = r#"{ - "networkId": "net123", - "siteId": "site456", - "pageId": "page789", - "formatId": "format000", - "advertiserId": "adv111", - "campaignId": "camp222", - "insertionId": "ins333", - "creativeId": "cre444", - "creativeUrl": "https://example.com/creative.png", - "callbacks": [] - }"#; - - let ad_response: AdResponse = - serde_json::from_str(json_str).expect("should deserialize ad response from str"); - assert_eq!(ad_response.network_id, "net123"); - assert_eq!(ad_response.site_id, "site456"); - assert_eq!(ad_response.page_id, "page789"); - assert_eq!(ad_response.format_id, "format000"); - } - - #[test] - fn test_callback_missing_field() { - let json_data = json!({ - "type": "impression" - // Missing url field - }); - - let result: Result = serde_json::from_value(json_data); - result.unwrap_err(); - } - - #[test] - fn test_callback_extra_fields() { - // Extra fields should be ignored - let json_data = json!({ - "type": "conversion", - "url": "https://example.com/track/conversion", - "extra": "ignored", - "another": 123 - }); - - let callback: Callback = serde_json::from_value(json_data) - .expect("should deserialize callback with extra fields"); - assert_eq!(callback.callback_type, "conversion"); - assert_eq!(callback.url, "https://example.com/track/conversion"); - } - - #[test] - fn test_ad_response_debug_format() { - let callback = Callback { - callback_type: "test".to_owned(), - url: "https://test.com".to_owned(), - }; - - let ad_response = AdResponse { - network_id: "123".to_owned(), - site_id: "456".to_owned(), - page_id: "789".to_owned(), - format_id: "000".to_owned(), - advertiser_id: "111".to_owned(), - campaign_id: "222".to_owned(), - insertion_id: "333".to_owned(), - creative_id: "444".to_owned(), - creative_url: "https://example.com/ad.jpg".to_owned(), - callbacks: vec![callback], - }; - - let debug_str = format!("{ad_response:?}"); - assert!(debug_str.contains("AdResponse")); - assert!(debug_str.contains("network_id")); - assert!(debug_str.contains("123")); - } - - #[test] - fn test_callback_debug_format() { - let callback = Callback { - callback_type: "debug_test".to_owned(), - url: "https://debug.test.com".to_owned(), - }; - - let debug_str = format!("{callback:?}"); - assert!(debug_str.contains("Callback")); - assert!(debug_str.contains("callback_type")); - assert!(debug_str.contains("debug_test")); - assert!(debug_str.contains("url")); - assert!(debug_str.contains("https://debug.test.com")); - } - - #[test] - fn test_various_callback_types() { - let callback_types = vec![ - "impression", - "click", - "viewability", - "conversion", - "engagement", - "complete", - "firstQuartile", - "midpoint", - "thirdQuartile", - ]; - - for cb_type in callback_types { - let json_data = json!({ - "type": cb_type, - "url": format!("https://example.com/track/{}", cb_type) - }); - - let callback: Callback = - serde_json::from_value(json_data).expect("should deserialize callback type"); - assert_eq!(callback.callback_type, cb_type); - assert_eq!(callback.url, format!("https://example.com/track/{cb_type}")); - } - } -} diff --git a/crates/trusted-server-core/src/module_context.rs b/crates/trusted-server-core/src/module_context.rs new file mode 100644 index 0000000000..e53ef8231e --- /dev/null +++ b/crates/trusted-server-core/src/module_context.rs @@ -0,0 +1,724 @@ +//! One context for every module, and the values a module's function may +//! name as parameters. +//! +//! Core builds one [`ModuleContext`] where it calls modules, from what the +//! request already holds. The context borrows everything it carries, so +//! building it copies no request data, and every module called there borrows +//! the same context. Each module is handed a [`ModuleCall`], being the context +//! together with the permissions that module declares. +//! +//! A module's function names what it needs as parameters, and +//! [`ModuleCall::inject`] passes each one in from the context. A parameter may +//! be any type that implements [`FromModuleContext`], an `Option` of one, which +//! is `None` when the value cannot be passed, or a tuple of them. The function +//! shapes that can be called are in [`inject`]. +//! +//! | Parameter | What it is | Absent where | +//! | --- | --- | --- | +//! | [`ModuleRequest`] | method, path, query, host and scheme | never | +//! | `&dyn RequestInfo` | the reader's evidence, headers and client IP | in a route, which holds the request itself | +//! | `&ClientInfo` | the reader's connection | where the device is classified | +//! | `&PermissionState` | the permissions resolved for the request | before they are resolved, which is where the device is classified and where the signal modules run | +//! | `&ConsentContext` | the decoded consent signals | where the device is classified | +//! | `&Settings` | the deployment's settings | outside any request | +//! | `&RuntimeServices` | stores, caches, backends and the HTTP client | outside any request | +//! | `&GeoInfo` | the reader's location | no location resolved, or withheld | +//! | `&DeviceSignals` | the device classification | not classified, or withheld | +//! | [`EdgeCookie`] | the request's Edge Cookie identifier | none, or withheld | +//! +//! A module asked outside any request, such as a signal module asked on its +//! own, is called in [`ModuleContext::empty`], which carries the request line +//! of the table and nothing else. +//! +//! # Permissions +//! +//! Geo, the device signals and the Edge Cookie identifier each carry the +//! permissions the module that produced them declares. Such a value is passed +//! only to a module that declares every one of those permissions and is granted +//! them on this request. A value whose producer declares none is passed to +//! every module, because evidence is not rationed and its use is what the +//! permissions govern. When a value is withheld, an `Option` parameter is +//! `None`, and a call naming the value itself is skipped, with the reason in +//! the debug log. + +use core::fmt; + +use edgezero_core::body::Body as EdgeBody; +use http::Method; + +use crate::consent::ConsentContext; +use crate::ec::EcContext; +use crate::ec::device::DeviceSignals; +use crate::error::TrustedServerError; +use crate::evidence::RequestInfo; +use crate::geo::GeoInfo; +use crate::permissions::{PermissionSet, PermissionState}; +use crate::platform::{ClientInfo, RuntimeServices}; +use crate::settings::Settings; + +pub mod inject; + +pub use inject::{Process, ProcessWith}; + +/// What a request resolved to: its method and address, and the host and +/// scheme it was made for. +/// +/// Carries nothing about who asked, so it is in every context. +#[derive(Debug, Clone, Copy)] +pub struct ModuleRequest<'r> { + method: &'r Method, + path: &'r str, + query: &'r str, + host: &'r str, + scheme: &'r str, +} + +impl<'r> ModuleRequest<'r> { + /// A request for `path` on `host` over `scheme`, with no query. + #[must_use] + pub const fn new(method: &'r Method, host: &'r str, scheme: &'r str, path: &'r str) -> Self { + Self { + method, + path, + query: "", + host, + scheme, + } + } + + /// What `request` resolved to, borrowed from it, the host and scheme read + /// as the publisher path reads them. + #[must_use] + pub fn of(request: &'r http::Request, client: &ClientInfo) -> Self { + Self::new( + request.method(), + crate::http_util::request_host(request), + crate::http_util::request_scheme(request, client), + request.uri().path(), + ) + .with_query(request.uri().query().unwrap_or_default()) + } + + /// The same request with the query string, without its leading `?`. + #[must_use] + pub fn with_query(self, query: &'r str) -> Self { + Self { query, ..self } + } + + /// The request method. + #[must_use] + pub fn method(&self) -> &'r Method { + self.method + } + + /// The URL path, without the query string. + #[must_use] + pub fn path(&self) -> &'r str { + self.path + } + + /// The query string, without its leading `?`, or `""`. + #[must_use] + pub fn query(&self) -> &'r str { + self.query + } + + /// The publisher-facing host the reader asked for. + #[must_use] + pub fn host(&self) -> &'r str { + self.host + } + + /// The publisher-facing scheme the reader asked for. + #[must_use] + pub fn scheme(&self) -> &'r str { + self.scheme + } +} + +/// A [`ModuleRequest`] that owns its parts, for a caller that must keep them +/// past the borrow of the request, such as one that hands the request on. +/// +/// The default is a `GET` of nothing, which a caller holds where it captured +/// no request. +#[derive(Debug, Clone, Default)] +pub struct ResolvedRequest { + method: Method, + path: String, + query: String, + host: String, + scheme: String, +} + +impl ResolvedRequest { + /// The method, address, host and scheme of `request`, the host and + /// scheme read as the publisher path reads them. + #[must_use] + pub fn of(request: &http::Request, client: &ClientInfo) -> Self { + let view = ModuleRequest::of(request, client); + Self { + method: view.method.clone(), + path: view.path.to_owned(), + query: view.query.to_owned(), + host: view.host.to_owned(), + scheme: view.scheme.to_owned(), + } + } + + /// The method, address, host and scheme `view` carries. + #[must_use] + pub fn from_view(view: &ModuleRequest<'_>) -> Self { + Self { + method: view.method.clone(), + path: view.path.to_owned(), + query: view.query.to_owned(), + host: view.host.to_owned(), + scheme: view.scheme.to_owned(), + } + } + + /// The same request with no query, for an answer that is held and served + /// to every caller, which must not depend on what one caller put in the + /// address. + #[must_use] + pub fn without_query(mut self) -> Self { + self.query.clear(); + self + } + + /// The borrowed view a context carries. + #[must_use] + pub fn view(&self) -> ModuleRequest<'_> { + ModuleRequest::new(&self.method, &self.host, &self.scheme, &self.path) + .with_query(&self.query) + } +} + +/// The request's Edge Cookie identifier, as the module that created it wrote +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EdgeCookie<'r>(&'r str); + +impl<'r> EdgeCookie<'r> { + /// The identifier. + #[must_use] + pub fn as_str(&self) -> &'r str { + self.0 + } +} + +/// A value whose use needs the permissions its producer declares. +struct Gated<'r, T: ?Sized> { + value: &'r T, + requires: PermissionSet, +} + +impl Clone for Gated<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Gated<'_, T> {} + +/// Everything a module may be passed for one request. +/// +/// Built once where core calls modules, by borrowing what the request already +/// holds, and borrowed by every module called there. A module never sees it +/// directly. It is handed a [`ModuleCall`] and names what it needs, see +/// [`ModuleCall::inject`]. +#[derive(Clone, Copy)] +pub struct ModuleContext<'r> { + request: ModuleRequest<'r>, + evidence: Option<&'r dyn RequestInfo>, + client: Option<&'r ClientInfo>, + permissions: Option<&'r PermissionState>, + consent: Option<&'r ConsentContext>, + settings: Option<&'r Settings>, + services: Option<&'r RuntimeServices>, + geo: Option>, + device: Option>, + edge_cookie: Option>, +} + +impl<'r> ModuleContext<'r> { + /// A context carrying `request` and nothing else. + /// + /// This is the context of a module asked outside any request, until the + /// builder methods add to it. + #[must_use] + pub const fn new(request: ModuleRequest<'r>) -> Self { + Self { + request, + evidence: None, + client: None, + permissions: None, + consent: None, + settings: None, + services: None, + geo: None, + device: None, + edge_cookie: None, + } + } + + /// The same context carrying the reader's evidence. + #[must_use] + pub fn with_evidence(self, evidence: &'r dyn RequestInfo) -> Self { + Self { + evidence: Some(evidence), + ..self + } + } + + /// The same context carrying the reader's connection. + #[must_use] + pub fn with_client(self, client: &'r ClientInfo) -> Self { + Self { + client: Some(client), + ..self + } + } + + /// The same context carrying the permissions resolved for the request. + #[must_use] + pub fn with_permissions(self, permissions: &'r PermissionState) -> Self { + Self { + permissions: Some(permissions), + ..self + } + } + + /// The same context carrying the request's decoded consent signals. + #[must_use] + pub fn with_consent(self, consent: &'r ConsentContext) -> Self { + Self { + consent: Some(consent), + ..self + } + } + + /// The same context carrying the deployment's settings. + #[must_use] + pub fn with_settings(self, settings: &'r Settings) -> Self { + Self { + settings: Some(settings), + ..self + } + } + + /// The same context carrying the request's services. + #[must_use] + pub fn with_services(self, services: &'r RuntimeServices) -> Self { + Self { + services: Some(services), + ..self + } + } + + /// The same context carrying the reader's location, whose use needs + /// `requires`, being what the geo module that resolved it declares. + #[must_use] + pub fn with_geo(self, geo: &'r GeoInfo, requires: PermissionSet) -> Self { + Self { + geo: Some(Gated { + value: geo, + requires, + }), + ..self + } + } + + /// The same context carrying the device signals, whose use needs + /// `requires`, being what the device module that derived them declares. + #[must_use] + pub fn with_device(self, device: &'r DeviceSignals, requires: PermissionSet) -> Self { + Self { + device: Some(Gated { + value: device, + requires, + }), + ..self + } + } + + /// The same context carrying the request's Edge Cookie identifier, whose + /// use needs `requires`, being what the Edge Cookie module declares. + #[must_use] + pub fn with_edge_cookie(self, id: &'r str, requires: PermissionSet) -> Self { + Self { + edge_cookie: Some(Gated { + value: id, + requires, + }), + ..self + } + } + + /// The same context carrying what the request's Edge Cookie state holds, + /// being its permissions, consent, location, device signals and Edge + /// Cookie identifier, and the request's services and connection. + /// + /// Each gated value needs what its producer declares. The location needs + /// what the geo module in `services` declares, the device signals what the + /// device module in `services` declares, and the identifier what the Edge + /// Cookie module the state was built with declares. Signals from a device + /// module the adapter builds itself, as Fastly does with none registered, + /// are gated by nothing, so such a module must declare no permission. + #[must_use] + pub fn with_request_state(self, state: &'r EcContext, services: &'r RuntimeServices) -> Self { + let mut context = self + .with_services(services) + .with_client(services.client_info()) + .with_permissions(state.permissions()) + .with_consent(state.consent()); + if let Some(geo) = state.geo_info() { + context = context.with_geo(geo, services.geo().required_permissions()); + } + if let Some(device) = state.device_signals() { + let requires = services + .device_module() + .map_or_else(PermissionSet::none, |module| module.required_permissions()); + context = context.with_device(device, requires); + } + if let (Some(id), Some(module)) = (state.ec_value(), state.selected_module()) { + context = context.with_edge_cookie(id, module.required_permissions()); + } + context + } + + /// What the request resolved to. + #[must_use] + pub fn request(&self) -> ModuleRequest<'r> { + self.request + } + + /// The request's decoded consent signals, when the context carries them. + pub(crate) fn consent(&self) -> Option<&'r ConsentContext> { + self.consent + } + + /// The reader's evidence, when the context carries it. + pub(crate) fn evidence(&self) -> Option<&'r dyn RequestInfo> { + self.evidence + } + + /// The call to one module, named `module`, which declares the permissions + /// in `declared`. + #[must_use] + pub fn call(&self, module: &'static str, declared: PermissionSet) -> ModuleCall<'_> { + ModuleCall { + context: self, + module, + declared, + } + } +} + +/// The method of the empty context. +static EMPTY_METHOD: Method = Method::GET; + +/// A context carrying nothing but a request for `/` on no host, for a module +/// asked outside any request. +static EMPTY: ModuleContext<'static> = + ModuleContext::new(ModuleRequest::new(&EMPTY_METHOD, "", "", "/")); + +impl ModuleContext<'static> { + /// The context of a module asked outside any request, which carries + /// nothing a module could name but an empty request. + /// + /// A caller with no request to resolve, such as a tool that evaluates + /// signals on their own, builds on a copy of it with the builder methods. + #[must_use] + pub fn empty() -> &'static Self { + &EMPTY + } +} + +impl ModuleCall<'static> { + /// A call into [`ModuleContext::empty`], by no name and declaring + /// nothing, for a module asked outside any request, such as by a test. + #[must_use] + pub fn empty() -> Self { + ModuleContext::empty().call("", PermissionSet::none()) + } +} + +impl fmt::Debug for ModuleContext<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ModuleContext") + .field("request", &self.request) + .field("evidence", &self.evidence.is_some()) + .field("permissions", &self.permissions.is_some()) + .field("settings", &self.settings.is_some()) + .field("services", &self.services.is_some()) + .field("geo", &self.geo.is_some()) + .field("device", &self.device.is_some()) + .field("edge_cookie", &self.edge_cookie.is_some()) + .finish_non_exhaustive() + } +} + +/// Core's call to one module, being the request's [`ModuleContext`] and the +/// permissions the module declares. +/// +/// A module's function is handed its parameters through +/// [`inject`](Self::inject). +#[derive(Debug, Clone, Copy)] +pub struct ModuleCall<'c> { + context: &'c ModuleContext<'c>, + module: &'static str, + declared: PermissionSet, +} + +impl<'c> ModuleCall<'c> { + /// The module being called. + #[must_use] + pub fn module(&self) -> &'static str { + self.module + } + + /// The permissions the module declares. + #[must_use] + pub fn declared(&self) -> PermissionSet { + self.declared + } + + /// A gated value, when this module may be passed it. + fn gated( + &self, + value: Option>, + name: &'static str, + ) -> Result<&'c T, Withheld> { + let gated = value.ok_or(Withheld::absent(name))?; + if gated.requires.is_empty() { + return Ok(gated.value); + } + let undeclared = gated.requires.difference(self.declared); + if !undeclared.is_empty() { + return Err(Withheld::new(name, WithheldReason::NotDeclared(undeclared))); + } + let state = self + .context + .permissions + .ok_or(Withheld::new(name, WithheldReason::Unresolved))?; + let unset = gated.requires.difference(state.permissions()); + if !unset.is_empty() { + return Err(Withheld::new(name, WithheldReason::NotGranted(unset))); + } + Ok(gated.value) + } +} + +/// Why a value is not passed to a module. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Withheld { + value: &'static str, + reason: WithheldReason, +} + +/// What kept a value from a module. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WithheldReason { + /// The context carries no such value where this module runs. + Absent, + /// The module does not declare these permissions, which the value's use + /// needs. + NotDeclared(PermissionSet), + /// These permissions, which the value's use needs, are not set on this + /// request. + NotGranted(PermissionSet), + /// The value's use needs permissions, and none are resolved where this + /// module runs. + Unresolved, +} + +impl Withheld { + fn new(value: &'static str, reason: WithheldReason) -> Self { + Self { value, reason } + } + + fn absent(value: &'static str) -> Self { + Self::new(value, WithheldReason::Absent) + } + + /// The value that was withheld, such as `geo`. + #[must_use] + pub fn value(&self) -> &'static str { + self.value + } + + /// Why it was withheld. + #[must_use] + pub fn reason(&self) -> WithheldReason { + self.reason + } +} + +impl fmt::Display for Withheld { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = self.value; + match self.reason { + WithheldReason::Absent => write!(f, "there is no {value} where this module runs"), + WithheldReason::NotDeclared(missing) => write!( + f, + "{value} is withheld because the module does not declare {missing}, which its \ + use needs" + ), + WithheldReason::NotGranted(missing) => write!( + f, + "{value} is withheld because {missing} is not set on this request" + ), + WithheldReason::Unresolved => write!( + f, + "{value} is withheld because its use needs permissions and none are resolved \ + where this module runs" + ), + } + } +} + +impl core::error::Error for Withheld {} + +/// A route whose parameter the request's permissions withhold answers `403`. +/// A value absent where the module runs, or needing permissions that are not +/// resolved there, is a fault in how the module was written or placed, and +/// answers `500`. +impl From for error_stack::Report { + fn from(withheld: Withheld) -> Self { + let message = withheld.to_string(); + error_stack::Report::new(match withheld.reason { + WithheldReason::NotDeclared(_) | WithheldReason::NotGranted(_) => { + TrustedServerError::Forbidden { message } + } + WithheldReason::Absent | WithheldReason::Unresolved => { + TrustedServerError::Configuration { message } + } + }) + } +} + +/// A value a module's function can name as a parameter. +/// +/// Implemented for each value a [`ModuleContext`] carries, for an `Option` of +/// one, which is `None` rather than withheld, and for tuples of up to eight. +pub trait FromModuleContext<'c>: Sized { + /// The value for this call. + /// + /// # Errors + /// + /// [`Withheld`] when the context carries no such value here, or when the + /// value's use needs a permission this module does not declare or is not + /// granted. + fn from_module_context(call: &ModuleCall<'c>) -> Result; +} + +impl<'c, T: FromModuleContext<'c>> FromModuleContext<'c> for Option { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + Ok(T::from_module_context(call).ok()) + } +} + +impl<'c> FromModuleContext<'c> for ModuleRequest<'c> { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + Ok(call.context.request) + } +} + +impl<'c> FromModuleContext<'c> for &'c dyn RequestInfo { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context + .evidence + .ok_or(Withheld::absent("request evidence")) + } +} + +impl<'c> FromModuleContext<'c> for &'c ClientInfo { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.client.ok_or(Withheld::absent("client info")) + } +} + +impl<'c> FromModuleContext<'c> for &'c PermissionState { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context + .permissions + .ok_or(Withheld::absent("permissions")) + } +} + +impl<'c> FromModuleContext<'c> for &'c ConsentContext { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.consent.ok_or(Withheld::absent("consent")) + } +} + +impl<'c> FromModuleContext<'c> for &'c Settings { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.settings.ok_or(Withheld::absent("settings")) + } +} + +impl<'c> FromModuleContext<'c> for &'c RuntimeServices { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.context.services.ok_or(Withheld::absent("services")) + } +} + +impl<'c> FromModuleContext<'c> for &'c GeoInfo { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.gated(call.context.geo, "geo") + } +} + +impl<'c> FromModuleContext<'c> for &'c DeviceSignals { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.gated(call.context.device, "device signals") + } +} + +impl<'c> FromModuleContext<'c> for EdgeCookie<'c> { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + call.gated(call.context.edge_cookie, "the Edge Cookie") + .map(EdgeCookie) + } +} + +/// A tuple of values is passed when every one of them is. +macro_rules! tuple_from_module_context { + ($($value:ident),+) => { + impl<'c, $($value: FromModuleContext<'c>),+> FromModuleContext<'c> for ($($value,)+) { + fn from_module_context(call: &ModuleCall<'c>) -> Result { + Ok(($($value::from_module_context(call)?,)+)) + } + } + }; +} + +tuple_from_module_context!(A1); +tuple_from_module_context!(A1, A2); +tuple_from_module_context!(A1, A2, A3); +tuple_from_module_context!(A1, A2, A3, A4); +tuple_from_module_context!(A1, A2, A3, A4, A5); +tuple_from_module_context!(A1, A2, A3, A4, A5, A6); +tuple_from_module_context!(A1, A2, A3, A4, A5, A6, A7); +tuple_from_module_context!(A1, A2, A3, A4, A5, A6, A7, A8); + +/// Requests for a module's own tests, which have no request to resolve. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support { + use http::Method; + + use super::ModuleRequest; + + static GET: Method = Method::GET; + + /// A `GET` of `path` on `publisher.example` over `https`, with no query. + #[must_use] + pub fn request(path: &'static str) -> ModuleRequest<'static> { + ModuleRequest::new(&GET, "publisher.example", "https", path) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/module_context/inject.rs b/crates/trusted-server-core/src/module_context/inject.rs new file mode 100644 index 0000000000..ba41fed5f0 --- /dev/null +++ b/crates/trusted-server-core/src/module_context/inject.rs @@ -0,0 +1,160 @@ +//! Calling a module's function with its parameters taken from the request's +//! [`ModuleContext`](super::ModuleContext). +//! +//! A trait method takes a [`ModuleCall`] and the implementation hands its own +//! function to [`ModuleCall::inject`], which is generic where the module type +//! is known. +//! +//! The function takes the module first, then up to eight parameters, each a +//! [`FromModuleContext`] value. [`ModuleCall::inject_with`] passes one argument +//! of the caller's own after the module, for the input that is the module's to +//! act on, such as the request a route answers. The function may be `async`, +//! in which case the future it returns is what comes back. +//! +//! ``` +//! use trusted_server_core::evidence::RequestInfo; +//! use trusted_server_core::geo::GeoInfo; +//! use trusted_server_core::module_context::{ModuleCall, Withheld}; +//! +//! struct Greeter; +//! +//! impl Greeter { +//! fn greet(&self, request: &dyn RequestInfo, geo: Option<&GeoInfo>) -> String { +//! let place = geo.map_or("somewhere", |geo| geo.country.as_str()); +//! format!("{} from {place}", request.user_agent()) +//! } +//! +//! fn process(&self, call: ModuleCall<'_>) -> Result { +//! call.inject(self, Self::greet) +//! } +//! } +//! ``` + +use super::{FromModuleContext, ModuleCall, Withheld}; + +/// A function core can call with the module and its parameters taken from a +/// [`ModuleCall`]. +/// +/// Implemented for every function and closure taking `&M` and up to eight +/// [`FromModuleContext`] parameters, so a module never implements it itself. +pub trait Process<'c, M: ?Sized, Args, O> { + /// Calls the function. + /// + /// # Errors + /// + /// [`Withheld`] for the first parameter that cannot be passed, in which case + /// the function is not called. + fn process(self, module: &'c M, call: &ModuleCall<'c>) -> Result; +} + +/// A function core can call with the module, one argument of the caller's +/// own, and its parameters taken from a [`ModuleCall`]. +/// +/// Implemented for every function and closure taking `&M`, the argument, and +/// up to eight [`FromModuleContext`] parameters. +pub trait ProcessWith<'c, M: ?Sized, X, Args, O> { + /// Calls the function. + /// + /// # Errors + /// + /// [`Withheld`] for the first parameter that cannot be passed, in which case + /// the function is not called and the argument is dropped. + fn process_with(self, module: &'c M, argument: X, call: &ModuleCall<'c>) + -> Result; +} + +macro_rules! process { + ($($parameter:ident),*) => { + impl<'c, M, F, O, $($parameter,)*> Process<'c, M, ($($parameter,)*), O> for F + where + M: ?Sized + 'c, + F: FnOnce(&'c M, $($parameter,)*) -> O, + $($parameter: FromModuleContext<'c>,)* + { + #[allow(non_snake_case, reason = "the parameters are named by their types")] + fn process(self, module: &'c M, call: &ModuleCall<'c>) -> Result { + let _ = call; + $(let $parameter = $parameter::from_module_context(call)?;)* + Ok(self(module, $($parameter,)*)) + } + } + + impl<'c, M, X, F, O, $($parameter,)*> ProcessWith<'c, M, X, ($($parameter,)*), O> for F + where + M: ?Sized + 'c, + F: FnOnce(&'c M, X, $($parameter,)*) -> O, + $($parameter: FromModuleContext<'c>,)* + { + #[allow(non_snake_case, reason = "the parameters are named by their types")] + fn process_with( + self, + module: &'c M, + argument: X, + call: &ModuleCall<'c>, + ) -> Result { + let _ = call; + $(let $parameter = $parameter::from_module_context(call)?;)* + Ok(self(module, argument, $($parameter,)*)) + } + } + }; +} + +process!(); +process!(A1); +process!(A1, A2); +process!(A1, A2, A3); +process!(A1, A2, A3, A4); +process!(A1, A2, A3, A4, A5); +process!(A1, A2, A3, A4, A5, A6); +process!(A1, A2, A3, A4, A5, A6, A7); +process!(A1, A2, A3, A4, A5, A6, A7, A8); + +impl<'c> ModuleCall<'c> { + /// Calls `function` with `module` and each of its other parameters taken + /// from the context. + /// + /// A parameter that is withheld skips the call, which is logged at debug + /// level with the reason, and an `Option` parameter that is withheld is + /// `None` instead. See [`module_context`](super) for what is withheld when. + /// + /// # Errors + /// + /// [`Withheld`] when a parameter cannot be passed, and the function is not + /// called. + pub fn inject(&self, module: &'c M, function: F) -> Result + where + M: ?Sized, + F: Process<'c, M, Args, O>, + { + function + .process(module, self) + .inspect_err(|withheld| self.log_skip(withheld)) + } + + /// Calls `function` with `module`, `argument`, and each of its other + /// parameters taken from the context, as [`inject`](Self::inject) does. + /// + /// # Errors + /// + /// [`Withheld`] when a parameter cannot be passed, and the function is not + /// called. + pub fn inject_with( + &self, + module: &'c M, + argument: X, + function: F, + ) -> Result + where + M: ?Sized, + F: ProcessWith<'c, M, X, Args, O>, + { + function + .process_with(module, argument, self) + .inspect_err(|withheld| self.log_skip(withheld)) + } + + fn log_skip(&self, withheld: &Withheld) { + log::debug!("Skipping module `{}`: {withheld}", self.module()); + } +} diff --git a/crates/trusted-server-core/src/module_context/tests.rs b/crates/trusted-server-core/src/module_context/tests.rs new file mode 100644 index 0000000000..dff4fbc923 --- /dev/null +++ b/crates/trusted-server-core/src/module_context/tests.rs @@ -0,0 +1,456 @@ +use std::cell::Cell; +use std::sync::Arc; + +use http::{HeaderMap, HeaderValue, Method, header}; + +use super::*; +use crate::ec::module::HmacModule; +use crate::evidence::OwnedRequestInfo; +use crate::permissions::{Permission, PermissionSet, PermissionState}; +use crate::platform::test_support::{noop_services, noop_services_with_client_ip}; +use crate::redacted::Redacted; + +/// The permission the gated values of these tests need. +fn storage() -> PermissionSet { + PermissionSet::none().with(Permission::StoreOnDevice) +} + +fn geo() -> GeoInfo { + GeoInfo { + city: "Bristol".to_owned(), + country: "GB".to_owned(), + continent: "Europe".to_owned(), + latitude: 51.45, + longitude: -2.58, + metro_code: 0, + region: Some("BST".to_owned()), + asn: None, + } +} + +fn device() -> DeviceSignals { + DeviceSignals::derive_ua_only("Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/146.0") +} + +fn evidence() -> OwnedRequestInfo { + let mut headers = HeaderMap::new(); + headers.insert( + header::USER_AGENT, + HeaderValue::from_static("ExampleAgent/1.0"), + ); + OwnedRequestInfo::new("203.0.113.7".to_owned(), headers) +} + +/// A module that is no more than the functions these tests hand to it. +struct Module; + +#[test] +fn the_request_reaches_every_function_that_names_it() { + let method = Method::POST; + let request = + ModuleRequest::new(&method, "publisher.example", "https", "/page").with_query("a=1"); + let context = ModuleContext::new(request); + let call = context.call("module", PermissionSet::none()); + + let seen = call + .inject(&Module, |_: &Module, request: ModuleRequest<'_>| { + ( + request.method().clone(), + request.host().to_owned(), + request.scheme().to_owned(), + request.path().to_owned(), + request.query().to_owned(), + ) + }) + .expect("the request is in every context"); + + assert_eq!( + seen, + ( + Method::POST, + "publisher.example".to_owned(), + "https".to_owned(), + "/page".to_owned(), + "a=1".to_owned(), + ) + ); +} + +#[test] +fn each_value_the_context_carries_reaches_a_function_that_names_it() { + let method = Method::GET; + let evidence = evidence(); + let client = ClientInfo { + tls_protocol: Some("TLSv1.3".to_owned()), + ..ClientInfo::default() + }; + let permissions = PermissionState::new(storage()); + let consent = ConsentContext::default(); + let mut settings = Settings::default(); + settings.publisher.domain = "publisher.example".to_owned(); + let services_ip: std::net::IpAddr = "203.0.113.9".parse().expect("should parse an IP"); + let services = noop_services_with_client_ip(services_ip); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_evidence(&evidence) + .with_client(&client) + .with_permissions(&permissions) + .with_consent(&consent) + .with_settings(&settings) + .with_services(&services); + let call = context.call("module", PermissionSet::none()); + + let first = call + .inject( + &Module, + |_: &Module, + evidence: &dyn RequestInfo, + client: &ClientInfo, + permissions: &PermissionState, + consent: &ConsentContext| { + ( + evidence.user_agent().to_owned(), + evidence.client_ip().to_owned(), + client.tls_protocol.clone(), + permissions.is_set(Permission::StoreOnDevice), + consent.jurisdiction.clone(), + ) + }, + ) + .expect("every value named is carried"); + assert_eq!( + first, + ( + "ExampleAgent/1.0".to_owned(), + "203.0.113.7".to_owned(), + Some("TLSv1.3".to_owned()), + true, + ConsentContext::default().jurisdiction, + ) + ); + + let second = call + .inject( + &Module, + |_: &Module, settings: &Settings, services: &RuntimeServices| { + ( + settings.publisher.domain.clone(), + services.client_info().client_ip, + ) + }, + ) + .expect("every value named is carried"); + assert_eq!(second, ("publisher.example".to_owned(), Some(services_ip))); +} + +#[test] +fn a_value_the_context_does_not_carry_is_none_or_skips_the_call() { + let method = Method::GET; + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")); + let call = context.call("module", PermissionSet::none()); + + let absent = call + .inject( + &Module, + |_: &Module, + evidence: Option<&dyn RequestInfo>, + settings: Option<&Settings>, + permissions: Option<&PermissionState>| { + ( + evidence.is_none(), + settings.is_none(), + permissions.is_none(), + ) + }, + ) + .expect("an optional parameter never skips the call"); + assert_eq!(absent, (true, true, true)); + + let called = Cell::new(false); + let skipped = call + .inject(&Module, |_: &Module, _settings: &Settings| called.set(true)) + .expect_err("a parameter the context does not carry skips the call"); + assert!(!called.get(), "the function is not called"); + assert_eq!(skipped.value(), "settings"); + assert_eq!(skipped.reason(), WithheldReason::Absent); +} + +/// The three gated values, each set on a context that needs `requires` for +/// it, read back through `Option` parameters by a module declaring +/// `declared`. +fn gated_values( + permissions: Option<&PermissionState>, + requires: PermissionSet, + declared: PermissionSet, +) -> (bool, bool, bool) { + let method = Method::GET; + let geo = geo(); + let device = device(); + let mut context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_geo(&geo, requires) + .with_device(&device, requires) + .with_edge_cookie("an-id", requires); + if let Some(permissions) = permissions { + context = context.with_permissions(permissions); + } + context + .call("module", declared) + .inject( + &Module, + |_: &Module, + geo: Option<&GeoInfo>, + device: Option<&DeviceSignals>, + edge_cookie: Option>| { + (geo.is_some(), device.is_some(), edge_cookie.is_some()) + }, + ) + .expect("an optional parameter never skips the call") +} + +#[test] +fn a_gated_value_reaches_a_module_that_declares_its_permission_and_is_granted_it() { + let granted = PermissionState::new(storage()); + + assert_eq!( + gated_values(Some(&granted), storage(), storage()), + (true, true, true) + ); +} + +#[test] +fn a_gated_value_is_withheld_from_a_module_that_does_not_declare_its_permission() { + let granted = PermissionState::new(storage()); + + assert_eq!( + gated_values(Some(&granted), storage(), PermissionSet::none()), + (false, false, false), + "granted on the request is not enough without the declaration" + ); +} + +#[test] +fn a_gated_value_is_withheld_when_its_permission_is_not_set() { + let refused = PermissionState::default(); + + assert_eq!( + gated_values(Some(&refused), storage(), storage()), + (false, false, false), + "the declaration is not enough without the permission being set" + ); + assert_eq!( + gated_values(None, storage(), storage()), + (false, false, false), + "and nothing is granted where no permissions are resolved" + ); +} + +#[test] +fn a_value_whose_producer_declares_nothing_reaches_every_module() { + assert_eq!( + gated_values(None, PermissionSet::none(), PermissionSet::none()), + (true, true, true), + "evidence is not rationed" + ); +} + +#[test] +fn a_withheld_value_skips_a_call_that_names_it_and_says_why() { + let method = Method::GET; + let geo = geo(); + let refused = PermissionState::default(); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_geo(&geo, storage()) + .with_permissions(&refused); + let called = Cell::new(false); + + let undeclared = context + .call("module", PermissionSet::none()) + .inject(&Module, |_: &Module, _geo: &GeoInfo| called.set(true)) + .expect_err("geo is withheld from a module that does not declare storage"); + let unset = context + .call("module", storage()) + .inject(&Module, |_: &Module, _geo: &GeoInfo| called.set(true)) + .expect_err("geo is withheld while storage is not set"); + + assert!(!called.get(), "a skipped call is never made"); + assert_eq!(undeclared.reason(), WithheldReason::NotDeclared(storage())); + assert_eq!(unset.reason(), WithheldReason::NotGranted(storage())); + assert_eq!( + undeclared.to_string(), + "geo is withheld because the module does not declare necessary.operations.storage, \ + which its use needs" + ); +} + +#[test] +fn an_argument_of_the_callers_own_is_passed_after_the_module() { + let method = Method::GET; + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/path")); + + let answer = context + .call("module", PermissionSet::none()) + .inject_with( + &Module, + String::from("argument"), + |_: &Module, argument: String, request: ModuleRequest<'_>| { + format!("{argument} {}", request.path()) + }, + ) + .expect("nothing is withheld"); + + assert_eq!(answer, "argument /path"); +} + +impl Module { + async fn classify(&self, evidence: &dyn RequestInfo, services: &RuntimeServices) -> String { + format!( + "{} {}", + evidence.user_agent(), + services.client_info().client_ip.is_some() + ) + } + + #[allow( + clippy::too_many_arguments, + reason = "the most parameters a module function may name" + )] + fn eight( + &self, + _request: ModuleRequest<'_>, + _evidence: &dyn RequestInfo, + _permissions: &PermissionState, + _settings: &Settings, + _services: &RuntimeServices, + _geo: Option<&GeoInfo>, + _device: Option<&DeviceSignals>, + _edge_cookie: Option>, + ) -> usize { + 8 + } +} + +#[test] +fn an_async_function_is_called_and_its_future_comes_back() { + let method = Method::GET; + let evidence = evidence(); + let services = noop_services_with_client_ip("203.0.113.9".parse().expect("should parse an IP")); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_evidence(&evidence) + .with_services(&services); + let call = context.call("module", PermissionSet::none()); + + let future = call + .inject(&Module, Module::classify) + .expect("nothing is withheld"); + + assert_eq!(futures::executor::block_on(future), "ExampleAgent/1.0 true"); +} + +#[test] +fn a_function_may_name_eight_values_and_a_tuple_groups_more() { + let method = Method::GET; + let evidence = evidence(); + let permissions = PermissionState::default(); + let deployment = Settings::default(); + let request_services = noop_services(); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_evidence(&evidence) + .with_permissions(&permissions) + .with_settings(&deployment) + .with_services(&request_services); + let call = context.call("module", PermissionSet::none()); + + assert_eq!(call.inject(&Module, Module::eight).expect("all carried"), 8); + let grouped = call + .inject( + &Module, + |_: &Module, (settings, services): (&Settings, &RuntimeServices)| { + ( + std::ptr::eq(settings, &deployment), + std::ptr::eq(services, &request_services), + ) + }, + ) + .expect("a tuple is passed when every value in it is"); + assert_eq!( + grouped, + (true, true), + "the tuple holds the context's own values" + ); +} + +#[test] +fn the_request_state_carries_each_value_gated_by_what_its_producer_declares() { + let method = Method::GET; + let mut state = EcContext::new_for_test(Some("an-id".to_owned()), ConsentContext::default()) + .with_module_for_test(Arc::new(HmacModule::new(Redacted::new( + "test-secret-key-32-bytes-minimum".to_owned(), + )))) + .with_geo_for_test(geo()); + state.set_device_signals(device()); + let services = + noop_services().with_device_module(Arc::new(crate::ec::device::BuiltinDeviceModule)); + let context = ModuleContext::new(ModuleRequest::new(&method, "h", "https", "/")) + .with_request_state(&state, &services); + + let undeclared = context + .call("module", PermissionSet::none()) + .inject( + &Module, + |_: &Module, + geo: Option<&GeoInfo>, + device: Option<&DeviceSignals>, + edge_cookie: Option>, + permissions: &PermissionState| { + ( + geo.map(|geo| geo.country.clone()), + device.is_some(), + edge_cookie.map(|id| id.as_str().to_owned()), + permissions.is_set(Permission::StoreOnDevice), + ) + }, + ) + .expect("nothing named is required"); + assert_eq!( + undeclared, + (Some("GB".to_owned()), true, None, true), + "geo and the device signals come from producers declaring nothing, and the \ + identifier needs what the HMAC module declares" + ); + + let declared = context + .call("module", storage()) + .inject(&Module, |_: &Module, edge_cookie: EdgeCookie<'_>| { + edge_cookie.as_str().to_owned() + }) + .expect("a module declaring storage is passed the identifier"); + assert_eq!(declared, "an-id"); +} + +#[test] +fn a_route_answers_forbidden_only_for_a_value_its_permissions_withhold() { + use crate::error::IntoHttpResponse as _; + + let status = |reason| { + error_stack::Report::::from(Withheld::new("geo", reason)) + .current_context() + .status_code() + }; + + assert_eq!( + status(WithheldReason::NotDeclared(storage())), + http::StatusCode::FORBIDDEN + ); + assert_eq!( + status(WithheldReason::NotGranted(storage())), + http::StatusCode::FORBIDDEN + ); + assert_eq!( + status(WithheldReason::Absent), + http::StatusCode::INTERNAL_SERVER_ERROR, + "a value absent where the route runs is the route's fault, not the reader's" + ); + assert_eq!( + status(WithheldReason::Unresolved), + http::StatusCode::INTERNAL_SERVER_ERROR + ); +} diff --git a/crates/trusted-server-core/src/module_name.rs b/crates/trusted-server-core/src/module_name.rs new file mode 100644 index 0000000000..3e25831a82 --- /dev/null +++ b/crates/trusted-server-core/src/module_name.rs @@ -0,0 +1,247 @@ +//! The name a module is selected by, taken from the folder its crate lives in. +//! +//! A crate at `crates/geo/example` supplies the module `geo.example`: the path +//! below the nearest `crates` directory, with `.` between the parts. A crate +//! supplying several modules names each one after itself, as +//! `middleware.example.hide`. A section may leave its own type folder off, so +//! under `[geo]` the name `example` means `geo.example`. Core's own modules +//! take bare names, such as `builtin`. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock, PoisonError}; + +/// The name of the module the calling crate supplies, or, given an entry, the +/// name of that entry, as `.`. +/// +/// Derived from Cargo's `CARGO_MANIFEST_DIR` when the calling crate is built, +/// so a name cannot drift from its folder. A crate built from outside a +/// `crates` directory, such as a published package, is named by its package +/// name. +#[macro_export] +macro_rules! module_name { + () => { + $crate::module_name::from_manifest(env!("CARGO_MANIFEST_DIR"), env!("CARGO_PKG_NAME"), None) + }; + ($entry:expr) => { + $crate::module_name::from_manifest( + env!("CARGO_MANIFEST_DIR"), + env!("CARGO_PKG_NAME"), + Some($entry), + ) + }; +} + +/// The module name for a crate built at `manifest_dir`, with `entry` after it +/// when the crate supplies several. +/// +/// Each distinct name is made once and kept for the life of the process, so +/// a module can hand out `&'static str` without its crate holding a copy. +#[must_use] +pub fn from_manifest(manifest_dir: &str, package: &str, entry: Option<&str>) -> &'static str { + static NAMES: OnceLock>> = OnceLock::new(); + let mut name = crate_path(manifest_dir).unwrap_or_else(|| package.to_owned()); + if let Some(entry) = entry { + name.push('.'); + name.push_str(entry); + } + let mut names = NAMES + .get_or_init(Mutex::default) + .lock() + .unwrap_or_else(PoisonError::into_inner); + if let Some(existing) = names.get(&name) { + return existing; + } + let leaked: &'static str = Box::leak(name.clone().into_boxed_str()); + names.insert(name, leaked); + leaked +} + +/// The path below the nearest `crates` directory, joined by `.`, or `None` +/// when the folder is not below one. +fn crate_path(manifest_dir: &str) -> Option { + let parts: Vec<&str> = manifest_dir + .split(['/', '\\']) + .filter(|part| !part.is_empty()) + .collect(); + let crates = parts.iter().rposition(|part| *part == "crates")?; + let below = &parts[crates + 1..]; + (!below.is_empty()).then(|| below.join(".")) +} + +/// The longest module name, in bytes. +const MAX_NAME_BYTES: usize = 255; + +/// Whether `name` can name a module: parts joined by `.`, each of lower case +/// letters, digits, `_` or `-`. +/// +/// The same rule serves a crate's derived name and a name written in a +/// section, so anything a crate can be called can also be written. +#[must_use] +pub fn is_valid(name: &str) -> bool { + !name.is_empty() + && name.len() <= MAX_NAME_BYTES + && name.split('.').all(|part| { + !part.is_empty() + && part.bytes().all(|byte| { + byte.is_ascii_lowercase() + || byte.is_ascii_digit() + || matches!(byte, b'_' | b'-') + }) + }) +} + +/// The offered name that `written`, selected in a section of type +/// `type_folder`, means. +/// +/// The name as written wins, which is how a bare name reaches one of core's +/// own modules and a full path reaches a module under another type. Otherwise +/// the type folder is put in front, because a section is named exactly as its +/// folder. +#[must_use] +pub fn resolve<'a>(type_folder: &str, written: &str, offered: &[&'a str]) -> Option<&'a str> { + if let Some(exact) = offered.iter().find(|name| **name == written) { + return Some(exact); + } + offered.iter().copied().find(|name| { + name.split_once('.') + .is_some_and(|(folder, rest)| rest == written && folder == type_folder) + }) +} + +/// The short form of `name` within a section of type `type_folder`, being the +/// name with the type folder taken off, or the name itself when it lives +/// under another type or is one of core's own. +#[must_use] +pub fn short_form<'a>(type_folder: &str, name: &'a str) -> &'a str { + match name.split_once('.') { + Some((folder, rest)) if folder == type_folder => rest, + _ => name, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_crate_is_named_by_its_path_below_crates() { + for (dir, expected) in [ + ( + "/home/runner/work/trusted-server/crates/geo/example", + "geo.example", + ), + ( + "D:\\Workspace\\trusted-server\\crates\\permission-signal\\example-scheme", + "permission-signal.example-scheme", + ), + ( + "/repo/vendor/trusted-server/crates/middleware/example", + "middleware.example", + ), + ("/repo/crates/demo/example", "demo.example"), + ] { + assert_eq!(crate_path(dir).as_deref(), Some(expected), "{dir}"); + } + } + + #[test] + fn the_nearest_crates_directory_counts() { + assert_eq!( + crate_path("/work/crates/tools/checkout/crates/geo/example").as_deref(), + Some("geo.example") + ); + } + + #[test] + fn a_crate_outside_a_crates_directory_takes_its_package_name() { + assert_eq!( + from_manifest("/registry/src/example-geo-0.1.0", "example-geo", None), + "example-geo" + ); + assert_eq!(crate_path("/work/crates"), None); + } + + #[test] + fn an_entry_follows_the_crate_path_and_each_name_is_kept_once() { + let first = from_manifest("/repo/crates/middleware/example", "x", Some("hide")); + let second = from_manifest("/repo/crates/middleware/example", "x", Some("hide")); + + assert_eq!(first, "middleware.example.hide"); + assert!( + std::ptr::eq(first, second), + "the same name is not made twice" + ); + } + + #[test] + fn this_crate_names_itself() { + assert_eq!(crate::module_name!(), "trusted-server-core"); + } + + #[test] + fn a_name_resolves_as_written_then_within_its_type() { + let offered = [ + "builtin", + "geo.example", + "permission-signal.us-privacy", + "testing.mock", + ]; + + assert_eq!(resolve("device", "builtin", &offered), Some("builtin")); + assert_eq!(resolve("geo", "example", &offered), Some("geo.example")); + assert_eq!(resolve("geo", "geo.example", &offered), Some("geo.example")); + assert_eq!( + resolve("permission-signal", "us-privacy", &offered), + Some("permission-signal.us-privacy") + ); + assert_eq!( + resolve("permission_signal", "us-privacy", &offered), + None, + "a section is named exactly as its folder" + ); + assert_eq!( + resolve("ad-server", "testing.mock", &offered), + Some("testing.mock") + ); + assert_eq!( + resolve("device", "example", &offered), + None, + "another type's module" + ); + assert_eq!(resolve("geo", "missing", &offered), None); + } + + #[test] + fn a_name_is_lower_case_parts_joined_by_dots() { + for good in [ + "4example", + "us-privacy", + "host_signals", + "middleware.example.hide", + ] { + assert!(is_valid(good), "{good}"); + } + for bad in [ + "", + "Geo", + "geo..example", + ".geo", + "geo.", + "a b", + "geo/example", + ] { + assert!(!is_valid(bad), "{bad:?}"); + } + } + + #[test] + fn a_short_form_drops_only_the_sections_own_type() { + assert_eq!(short_form("geo", "geo.example"), "example"); + assert_eq!( + short_form("permission-signal", "permission-signal.example-scheme"), + "example-scheme" + ); + assert_eq!(short_form("ad-server", "testing.mock"), "testing.mock"); + assert_eq!(short_form("geo", "platform"), "platform"); + } +} diff --git a/crates/trusted-server-core/src/module_secrets.rs b/crates/trusted-server-core/src/module_secrets.rs new file mode 100644 index 0000000000..24e9653fc6 --- /dev/null +++ b/crates/trusted-server-core/src/module_secrets.rs @@ -0,0 +1,368 @@ +//! The secret settings a module declares in its own table. +//! +//! A module's builder lists the settings in its table that hold the name of +//! a secret, each with a rule for whether the table as written puts the +//! setting to use. As the settings load, the ones not in use are cleared and +//! the others are looked up in the default secret store, and at deploy time +//! each one in use is checked to name a key. None of that needs to know +//! which module declared them. + +use std::borrow::Cow; + +use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; +use serde_json::{Map, Value}; + +use crate::integrations::IntegrationBuilder; +use crate::module_name::short_form; + +/// The section a module's table sits under and the names it can be written +/// at there, being the name with the section's type folder left off and the +/// name in full where the two differ, or a vendor's shared table under +/// `shared`, at the vendor's name alone. +fn written_forms(builder: &IntegrationBuilder) -> Option<(&'static str, Vec<&'static str>)> { + if let Some(vendor) = builder.shared_settings() { + return Some((crate::settings::SHARED_TYPE, vec![vendor])); + } + let name = builder.module_name()?; + let section = builder.section()?; + let short = short_form(section, name); + let mut forms = vec![short]; + if short != name { + forms.push(name); + } + Some((section, forms)) +} + +/// The form `section` selects the module by in a serialized configuration, +/// when it selects it. +fn selected_as<'a>(data: &Value, section: &str, forms: &[&'a str]) -> Option<&'a str> { + let section = data.get(section)?; + let named = |value: &Value| { + let written = value.as_str()?; + forms.iter().copied().find(|form| *form == written) + }; + section.get("module").and_then(named).or_else(|| { + section + .get("modules") + .and_then(Value::as_array) + .and_then(|modules| modules.iter().find_map(named)) + }) +} + +/// The table of a name as written in a section, found by the name's parts. +fn table_mut<'a>( + data: &'a mut Value, + section: &str, + written: &str, +) -> Option<&'a mut Map> { + let mut node = data.get_mut(section)?; + for part in written.split('.') { + node = node.get_mut(part)?; + } + node.as_object_mut() +} + +/// The value at `path` inside a module's table. +#[must_use] +pub(crate) fn value_at<'a>(table: &'a Map, path: &[&str]) -> Option<&'a Value> { + let (first, rest) = path.split_first()?; + let mut node = table.get(*first)?; + for part in rest { + node = node.get(*part)?; + } + Some(node) +} + +fn remove_at(table: &mut Map, path: &[&str]) { + let Some((leaf, parents)) = path.split_last() else { + return; + }; + let mut node = table; + for parent in parents { + let Some(next) = node.get_mut(*parent).and_then(Value::as_object_mut) else { + return; + }; + node = next; + } + node.remove(*leaf); +} + +/// Clears every declared secret setting that is not in use, in a serialized +/// configuration whose secrets have not been looked up yet. +/// +/// A setting is in use when its section selects the module and the module's +/// rule for the setting says its table uses it. A table the section does not +/// select is refused once the settings are deserialized, and clearing its +/// references first means that refusal is what an operator sees, rather than +/// a secret lookup failing. +pub(crate) fn clear_unused(data: &mut Value, builders: &[IntegrationBuilder]) { + for builder in builders { + let secrets = builder.secret_settings(); + if secrets.is_empty() { + continue; + } + let Some((section, forms)) = written_forms(builder) else { + continue; + }; + // A vendor's table runs its builder, so a secret in the table counts + // as in use by the table's own rules. + let selected = match builder.shared_settings() { + Some(vendor) => Some(vendor), + None => selected_as(data, section, &forms), + }; + for form in &forms { + let Some(table) = table_mut(data, section, form) else { + continue; + }; + let runs = selected == Some(*form); + let unused = secrets + .iter() + .filter(|secret| !(runs && (secret.in_use)(table))) + .map(|secret| secret.path) + .collect::>(); + for path in unused { + remove_at(table, path); + } + } + } +} + +/// The secret leaves the builders' modules declare, at each name a module's +/// table can be written under. +/// +/// Every level is optional, so a module that is not configured, or a setting +/// [`clear_unused`] removed, is passed over. +#[must_use] +pub(crate) fn secret_fields(builders: &[IntegrationBuilder]) -> Vec { + let optional = |name: &'static str| SecretPathSegment::OptionalField(Cow::Borrowed(name)); + let mut fields = Vec::new(); + for builder in builders { + let Some((section, forms)) = written_forms(builder) else { + continue; + }; + for secret in builder.secret_settings() { + let Some((leaf, parents)) = secret.path.split_last() else { + continue; + }; + for form in &forms { + let mut path = vec![optional(section)]; + path.extend(form.split('.').map(optional)); + path.extend(parents.iter().copied().map(optional)); + path.push(SecretPathSegment::Field(Cow::Borrowed(*leaf))); + fields.push(SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path, + }); + } + } + } + fields +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::integrations::ModuleSecretSetting; + use crate::integrations::registry_test_support::{probe_registration, validate_nothing}; + + fn uses_key(table: &Map) -> bool { + table.get("lock").and_then(Value::as_bool) == Some(true) + } + + fn uses_spare(table: &Map) -> bool { + uses_key(table) + && table + .get("spare") + .and_then(|spare| spare.get("enabled")) + .and_then(Value::as_bool) + == Some(true) + } + + const SECRETS: &[ModuleSecretSetting] = &[ + ModuleSecretSetting { + path: &["key_name"], + in_use: uses_key, + }, + ModuleSecretSetting { + path: &["spare", "key_name"], + in_use: uses_spare, + }, + ]; + + fn builder() -> IntegrationBuilder { + IntegrationBuilder::new("probe", "seam-probe", probe_registration, validate_nothing) + .with_module_name("testing.probe") + .with_secret_settings(SECRETS) + } + + const VENDOR: &str = "example"; + + fn shared_builder() -> IntegrationBuilder { + IntegrationBuilder::new( + "shared_probe", + "seam-probe", + probe_registration, + validate_nothing, + ) + .with_shared_settings(VENDOR) + .with_secret_settings(SECRETS) + } + + #[test] + fn a_secret_in_a_shared_table_is_kept_when_the_table_is_present_and_in_use() { + let mut data = json!({ "shared": { VENDOR: table(true, false) } }); + + clear_unused(&mut data, &[shared_builder()]); + + assert_eq!( + data.pointer("/shared/example/key_name"), + Some(&json!("main-key")), + "the table runs the builder, so its setting in use stays" + ); + assert_eq!(data.pointer("/shared/example/spare/key_name"), None); + } + + #[test] + fn a_secret_in_a_shared_table_is_declared_under_the_vendors_name() { + let paths: Vec = secret_fields(&[shared_builder()]) + .iter() + .map(|field| { + field + .path + .iter() + .map(|segment| match segment { + SecretPathSegment::Field(name) | SecretPathSegment::OptionalField(name) => { + name.to_string() + } + _ => "*".to_owned(), + }) + .collect::>() + .join(".") + }) + .collect(); + + assert_eq!( + paths, + vec!["shared.example.key_name", "shared.example.spare.key_name"] + ); + } + + fn table(lock: bool, spare: bool) -> Value { + json!({ + "lock": lock, + "key_name": "main-key", + "spare": { "enabled": spare, "key_name": "spare-key" }, + }) + } + + #[test] + fn a_setting_in_use_is_kept_and_one_not_in_use_is_cleared() { + let mut data = json!({ "testing": { "modules": ["probe"], "probe": table(true, false) } }); + + clear_unused(&mut data, &[builder()]); + + assert_eq!( + data.pointer("/testing/probe/key_name"), + Some(&json!("main-key")), + "should keep the setting the table puts to use" + ); + assert_eq!( + data.pointer("/testing/probe/spare/key_name"), + None, + "should clear the setting the table does not put to use" + ); + assert_eq!( + data.pointer("/testing/probe/spare/enabled"), + Some(&json!(false)), + "should leave the rest of the table as written" + ); + } + + #[test] + fn every_setting_of_a_module_no_section_selects_is_cleared() { + let mut data = json!({ "testing": { "probe": table(true, true) } }); + + clear_unused(&mut data, &[builder()]); + + assert_eq!(data.pointer("/testing/probe/key_name"), None); + assert_eq!(data.pointer("/testing/probe/spare/key_name"), None); + } + + #[test] + fn a_module_selected_by_its_full_name_keeps_the_settings_in_the_table_at_that_name() { + let mut data = json!({ + "testing": { + "module": "testing.probe", + "testing": { "probe": table(true, true) }, + "probe": table(true, true), + } + }); + + clear_unused(&mut data, &[builder()]); + + assert_eq!( + data.pointer("/testing/testing/probe/key_name"), + Some(&json!("main-key")), + "should keep the settings of the table the selection names" + ); + assert_eq!( + data.pointer("/testing/testing/probe/spare/key_name"), + Some(&json!("spare-key")) + ); + assert_eq!( + data.pointer("/testing/probe/key_name"), + None, + "should clear the settings of a table under the name not selected" + ); + } + + #[test] + fn the_declared_leaves_are_listed_under_both_written_names() { + let paths = secret_fields(&[builder()]) + .iter() + .map(|field| (field.dotted_path(), field.optional)) + .collect::>(); + + assert_eq!( + paths, + vec![ + ("testing.probe.key_name".to_owned(), true), + ("testing.testing.probe.key_name".to_owned(), true), + ("testing.probe.spare.key_name".to_owned(), true), + ("testing.testing.probe.spare.key_name".to_owned(), true), + ], + "should list each declared leaf at the short name and at the full one" + ); + } + + #[test] + fn a_builder_that_declares_no_secret_setting_lists_none() { + let plain = + IntegrationBuilder::new("probe", "seam-probe", probe_registration, validate_nothing) + .with_module_name("testing.probe"); + let mut data = json!({ "testing": { "probe": table(false, false) } }); + let before = data.clone(); + + clear_unused(&mut data, &[plain]); + + assert_eq!(data, before, "should leave the table alone"); + assert!(secret_fields(&[plain]).is_empty()); + } + + #[test] + fn value_at_reads_a_nested_setting() { + let Value::Object(table) = table(true, true) else { + panic!("should build a table"); + }; + + assert_eq!(value_at(&table, &["key_name"]), Some(&json!("main-key"))); + assert_eq!( + value_at(&table, &["spare", "key_name"]), + Some(&json!("spare-key")) + ); + assert_eq!(value_at(&table, &["absent", "key_name"]), None); + } +} diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 4aded74882..7260d8b134 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -111,24 +111,6 @@ pub struct RegsExt { impl ToExt for RegsExt {} -#[derive(Debug, Serialize)] -pub struct RequestExt { - #[serde(skip_serializing_if = "Option::is_none")] - pub prebid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub trusted_server: Option, -} - -#[derive(Debug, Serialize, serde::Deserialize)] -pub struct PrebidExt { - #[serde(skip_serializing_if = "Option::is_none")] - pub debug: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub returnallbidstatus: Option, -} - -impl ToExt for RequestExt {} - #[derive(Debug, Serialize)] pub struct TrustedServerExt { /// Version of the signing protocol (e.g., "1.1") @@ -147,30 +129,6 @@ pub struct TrustedServerExt { pub ts: Option, } -#[derive(Debug, Serialize)] -pub struct ImpExt { - pub prebid: PrebidImpExt, -} - -impl ToExt for ImpExt {} - -#[derive(Debug, Default, Serialize)] -pub struct PrebidImpExt { - #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] - pub bidder: std::collections::HashMap, - #[serde(skip_serializing_if = "Option::is_none")] - pub storedrequest: Option, -} - -/// PBS imp-level stored request reference. -/// -/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, -/// populating bidder params that are not sent inline. -#[derive(Debug, Serialize)] -pub struct ImpStoredRequest { - pub id: String, -} - #[derive(Debug, Serialize)] pub struct BidExt<'a> { pub trusted_server: BidTrustedServerExt<'a>, diff --git a/crates/trusted-server-core/src/permission_signal/README.md b/crates/trusted-server-core/src/permission_signal/README.md new file mode 100644 index 0000000000..7c2c8d25aa --- /dev/null +++ b/crates/trusted-server-core/src/permission_signal/README.md @@ -0,0 +1,257 @@ +# Permission signals + +A permission is the primitive. This is the seam that decides whether one is +set, and it is open so that a new way of learning a permission can be added +without changing core. + +"Signal" rather than "consent" because consent is only one of the things +arriving here. Global Privacy Control is a setting a browser sends, not an +answer anyone gave to a question, and a jurisdiction rule is neither. Consent +is one kind of signal, so the seam takes the wider name and the consent +subsystem keeps the narrower one. + +## What core holds + +The reasons the schemes live outside core are on the docs page, +docs/guide/permission-signals.md. + +Core holds the trait, the ordering, the country baseline, and the policy +vocabulary for what a deployment decides about the shipped schemes, being +whether a TCF record answers, which signals count as a US-style opt-out and +what an opt-out takes away. It holds no scheme's wire format and no scheme's +meaning. Every scheme lives in its own crate outside core, including the five +that ship by default, so none of them is privileged by being the one that +happens to be built in. Core does not know what a TCF purpose is, because the +mapping from purpose to Data Use lives in the TCF crate, and a deployment +that runs no TCF carries no table of another scheme's numbers. The purposes +are the IAB TCF Europe purposes and what they grant are IAB Tech Lab Privacy +Taxonomy Data Uses, so a reader checks the mapping against the industry's own +documents rather than against us, and docs/guide/permission-model.md lists it +in full. Two purposes have no Data Use yet, so the crate carries a proposed +key for one and the TCF identifier for the other until the taxonomy adds +them. + +A scheme that is not one of the five is added the same way, as a crate that +reads its own signal from the request, without a change to core. The fifth, +Model Terms for Marketing (MTM), is the first terms scheme, where a publisher +and the parties it passes data to agree to be bound by a published set of +terms, and what the module reads is the visitor's answer to a preference +platform, one of three words in the first party cookie +`__mtm_pref`, a name any platform may set. It is one of many terms schemes +rather than the only one, because a publisher, a trade body or a regulator can +each publish terms and each set becomes a module. The five here are a +starting set and not the list. + +## The hierarchy + +Permissions are resolved in layers, each amending the one before. + +```text + country / region rules the baseline: granted, requires-signal, denied + | + v + module 1 (configured order) may amend + | + v + module 2 may amend + | + v + ... may amend + | + v + the permission state for this request +``` + +The baseline comes from `permissions.yaml`, keyed by country and region, with +the top node of the rules tree standing in for a request whose place is +unknown. A geo module supplies the place. No geo module means no country, +so every request resolves at that top node, and only a lookup that failed +resolves at the requires-signal floor, because a place that could not be +determined must not be treated as the declared default. + +Modules are then asked in order. Each sees what the modules before it +settled on and may amend it. A module with no opinion returns `Neutral` and +leaves the prior value standing, which is different from refusing. + +## The order is the policy + +The last module with an opinion decides, so the order is the policy. It is +a deployment's to set, not this code's to assume. + +```toml +[permission-signal] +modules = ["gpc", "gpp", "us-privacy", "tcf", "mtm"] +``` + +A module not on the list does not run, and there is no separate switch. A +publisher who does not want to act on Global Privacy Control removes `"gpc"` +from the list, and the module that reads the header then does not run. One +caveat: core's consent pipeline can also synthesize a US Privacy opt-out from +that header for a visitor in a US state, when the consent settings say to, +which they do by default, and the `us-privacy` module then acts on the +record it produced. A publisher who wants the header to have no effect at all +turns that setting off as well. Leaving `modules` out, or the section +entirely, runs every module the adapter offers, in the order it offers them, +so a signal is never quietly ignored because someone forgot to list it. An +unknown or repeated name is refused at startup, so a typo cannot silently stop +a scheme being honored. + +A module is named by its crate folder below `crates/permission-signal`, so +`gpp` and `us-privacy` are the folders those crates live in, and a name may +also be written in full, as `permission-signal.gpp`. A module that gains +settings will take them in a `[permission-signal.]` +block named for it. None of the five here has settings, so `modules` is the +only key the section accepts, and a block or any other key is refused as an +unknown field rather than ignored. + +The default order asks the signal with no interface of its own first and the +ones carrying a choice made through an interface after. Global Privacy +Control is a browser setting, so it revokes personalization on arrival, +whereas a GPP sale opt-out, a US Privacy string and a TCF record each carry +an answer a person gave, so they are asked later and amend it. A deployment +wanting the browser setting to stand over a later answer puts `gpc` last. + +Trusted Server takes no view on which scheme should win. That is a question +about a jurisdiction and a publisher. + +## A module can see the others + +Amending well sometimes needs to know who set the prior value. A module is +given the whole ordered list and its own position in it, so it can look up a +peer by name, see whether a peer it cares about is configured at all, and ask +a peer directly what that peer makes of a permission. + +That is what makes a rule like "personalization is off, but only because +Global Privacy Control set it, so my answer supersedes it" expressible. The +rule itself belongs to whichever module wants it. This seam only makes the +information available. + +Consulting a peer goes one level deep. A module answering a consultation +cannot consult in turn, so two modules asking each other cannot loop. + +## The terms the data is available under + +A module may also declare the terms documents the request's data is available +under, through `tdls` on the trait, and core carries what every configured +module declared on the permission state. Whoever receives the data reads them +to decide whether those are terms they accept, and whether they may pass the +data on. No declaration means no terms were declared, which is not the same as +terms permitting anything, so a recipient needing a basis and finding none has +none. + +Each entry is the address of a published document a person can read, and the +document must never be edited once published, which is why a version belongs in +its address. A document that can be rewritten tomorrow means a recipient can +never prove what it agreed to, and one edit silently rewrites the basis of every +transaction already sent under it. That is a property of how the document is +published, so the [`Tdl`](crate::tdl::Tdl) type refuses only an address nothing +could fetch. + +Of the five schemes here only MTM carries terms, declaring the versioned Model +Terms document whenever a PMP answer is present, and it is the first of many +rather than the only one. The name matches the `tdl` member the Data Labels work puts on a +node of an `OpenRTB` request, which is where these travel once a bid request +carries them. + +## Who could still answer + +A permission whose baseline requires a signal, and for which every module +answered `Neutral`, is not set, and it is also not refused. Resolution records +it as awaited, and the page reads the list as `awaiting` beside `set`, so a +prompt that has not run yet can be told from a visitor who said no. + +Waiting is only right for a permission some configured module could grant. +So each module declares, through `grants` on the trait, which permissions it +can ever answer `Grant` for under the policy it is given. An opt-out revokes and +declares nothing. TCF declares every Data Use a purpose maps to, and nothing +when the policy silences the record. The assembly keeps as awaited only what +some module declared, and everything else that requires a signal and got none +is simply unset. + +## The signals that were valid + +Each module also says, through `valid_signal` on the trait, which signal it +read from the request and used, as it was received. Core carries what every +configured module vouched for on the permission state, and the page reads +the list as `signals` beside `set`, `awaiting` and `tdls`. A page, a bid +request or a person reading the state relies on exactly those signals and no +other. + +A signal that was absent, could not be read, has expired, or that no +configured module acts on is not in the list, and nothing says which. What +each of those means for the permissions is the decision of the module for +that scheme, taken in `signal` and taken silently. Nothing is logged, because +an unreadable record is a visitor's preference and not an operational fault. +The shipped modules read their own unreadable record as the refusal it +may have carried, on the permissions their scheme covers, and say nothing +about any other scheme. Core answers nothing ahead of the modules, so the +order decides what a readable record from one scheme means beside an +unreadable one from another, as it decides everything else. + +A signal nobody vouched for goes no further. After assembly the consent +context keeps only the raw strings the valid signals name, so a corrupt or +expired record never reaches a bid request or anything else Trusted Server +sends on. + +## Withdrawal is a separate question + +A module may also say that the request explicitly _withdraws_ a permission, +which is different from not granting it. A withdrawal of storage expires the +browser cookie and writes the authoritative tombstone against the identifier. +A permission that is merely not set strips the response headers and leaves an +already-issued identifier alone, so a returning visitor is not permanently +withdrawn before they get to answer. Most schemes have no such notion, a +browser setting and a sale opt-out included, and only TCF answers it. Core +then scopes the answer to the jurisdiction, so a refusal only withdraws where +the storage baseline did not grant storage outright, because where it did +the identifier never depended on the record. + +## Writing a module + +Implement `PermissionSignalModule` in a crate that depends on core, and give +it its name through `module_name!()`, which is the name configuration uses. Answer +`Neutral` for a permission the module has no opinion on, including when the +signal it reads is absent from the request. Returning `Revoke` for an absent +signal turns silence into refusal and would revoke the permission on every +request not carrying that scheme, which is most of them. + +Read the request through `SignalInput::evidence`, which offers headers, +cookies, the path and the query, so a scheme core has never heard of can read +its own signal. The decoded consent record is offered too, for the schemes +core's consent pipeline already decodes, caches against the identifier and +expires. Prefer the record where it exists, because a module that +re-decodes the wire would skip the cached record on a returning visitor and +answer differently from every other reader of the same request. + +Read the policy for what is a deployment's decision rather than the scheme's, +such as which signals count as an opt-out and what an opt-out takes away, so +that a deployment can change those without changing a module. + +Register the crate at the adapter's composition root, where every adapter +builds its `RuntimeServices`. The adapter lists the modules it links, in the +default order, and `build_permission_signal_modules` selects and orders them +from configuration. + +## The modules supplied + +Five crates ship, under `crates/permission-signal/`, and a deployment +configuring nothing gets all five in this order: + +| Name | Crate | Reads | +| ------------ | ------------ | --------------------------------------------------- | +| `gpc` | `gpc` | The `Sec-GPC` header, Global Privacy Control | +| `gpp` | `gpp` | The US sale opt-out in a GPP string | +| `us-privacy` | `us-privacy` | The sale opt-out in a US Privacy string | +| `tcf` | `tcf` | A TCF v2 record, with its purpose mapping in code | +| `mtm` | `mtm` | The PMP answer, under the Model Terms for Marketing | + +The three opt-outs are separate rather than one so that a publisher who does +not act on Global Privacy Control can remove it and keep the other two. + +## What is not a module + +Nothing. Core reads no scheme and answers for none, an unreadable record +included. A publisher chooses which signals to act on by listing modules, +and each module decides for its own scheme what an absent, unreadable or +expired signal means. A scheme that is not listed does not run, whatever the +request carries for it. diff --git a/crates/trusted-server-core/src/permission_signal/mod.rs b/crates/trusted-server-core/src/permission_signal/mod.rs new file mode 100644 index 0000000000..74d7d33b8e --- /dev/null +++ b/crates/trusted-server-core/src/permission_signal/mod.rs @@ -0,0 +1,1196 @@ +#![doc = include_str!("README.md")] + +use std::sync::Arc; + +use error_stack::Report; + +use crate::consent::ConsentContext; +use crate::error::TrustedServerError; +use crate::evidence::RequestInfo; +use crate::module_context::{ModuleCall, ModuleContext}; +use crate::permissions::{ + Acquisition, ConsentSignal, Permission, PermissionSet, SignalPolicy, ValidSignal, +}; +use crate::settings::Settings; +use crate::tdl::Tdl; + +/// What a signal module may read about a request. +/// +/// A struct rather than a parameter list, so a module needing something new +/// does not change every implementation. The fields are what the signal run +/// is deciding. Anything else in the request's module context, such as what +/// the request resolved to or the settings, a module names through +/// [`call`](Self::call). The permissions are what the run produces, so none +/// are resolved yet and no value whose use needs one is passed here. +pub struct SignalInput<'a> { + /// The decoded consent record for this request. + /// + /// Offered because the schemes that ship by default already decode into + /// it, core keeps it against the Edge Cookie identifier between requests, + /// and the rest of the system reads it. A module is not required to use + /// it, and a scheme core has never heard of will not appear in it. Such a + /// module reads [`evidence`](Self::evidence) instead and decodes whatever + /// its scheme needs. + pub consent: &'a ConsentContext, + /// The request itself, as evidence. + /// + /// This is the open half of the seam, and it is [`RequestInfo`], the same + /// abstraction the Edge Cookie and device modules already read. A + /// module asks for the header, cookie, path or query parameter its own + /// scheme uses, so core holds no list of which evidence a scheme may read. + pub evidence: &'a dyn RequestInfo, + /// The policy from `permissions.yaml`, which decides what a signal means + /// rather than leaving each module to invent its own meaning. + pub policy: &'a SignalPolicy, + /// What the country and region rules say about this permission, before any + /// module is asked. A module amends this rather than deciding alone. + pub baseline: Acquisition, + /// What the modules asked before this one settled on. + /// + /// [`ConsentSignal::Neutral`] means none of them had an opinion, so the + /// baseline still stands. + pub settled: ConsentSignal, + /// Every module in configured order, this one included. + modules: &'a [Arc], + /// Where in that order the module being asked sits. + position: usize, + /// Whether this module may consult a peer. + /// + /// False while answering a consultation, which is what stops two modules + /// that consult each other from looping. + may_ask: bool, + /// The request's module context. + context: &'a ModuleContext<'a>, +} + +impl<'a> SignalInput<'a> { + /// An input for a module asked on its own, outside an ordered run, in a + /// context carrying nothing, see [`ModuleContext::empty`]. + #[must_use] + pub fn new( + consent: &'a ConsentContext, + evidence: &'a dyn RequestInfo, + policy: &'a SignalPolicy, + baseline: Acquisition, + ) -> Self { + Self { + consent, + evidence, + policy, + baseline, + settled: ConsentSignal::Neutral, + modules: &[], + position: 0, + may_ask: true, + context: ModuleContext::empty(), + } + } + + /// The same input in the request's module context. + #[must_use] + pub fn with_context(self, context: &'a ModuleContext<'a>) -> Self { + Self { context, ..self } + } + + /// The module's call into the request's module context, which it hands + /// its own function to, naming what it needs. A module asked on its own + /// is called by no name and declares nothing. + #[must_use] + pub fn call(&self) -> ModuleCall<'a> { + match self.modules.get(self.position) { + Some(module) => self + .context + .call(module.id(), module.required_permissions()), + None => self.context.call("", PermissionSet::none()), + } + } + + /// Every module in configured order, this one included. + /// + /// A module consults the list to decide whether a peer it cares about is + /// configured at all, and where it sits relative to this one. + #[must_use] + pub fn modules(&self) -> &[Arc] { + self.modules + } + + /// Where the module being asked sits in that order. + #[must_use] + pub const fn position(&self) -> usize { + self.position + } + + /// Whether a module with this identifier is configured. + #[must_use] + pub fn has(&self, id: &str) -> bool { + self.modules.iter().any(|module| module.id() == id) + } + + /// What a peer makes of `permission`, asked directly. + /// + /// The peer answers as if it were first, so the reply is that peer's own + /// opinion rather than what the run has settled on so far. That is the + /// useful question, because a module wanting to know whether the prior + /// value came from a particular peer asks that peer what it says. + /// + /// Returns `None` when no module carries the identifier, when the caller + /// names itself, and when this input is itself answering a consultation. + /// A module must handle `None` rather than assume a peer is present. + #[must_use] + pub fn ask(&self, id: &str, permission: Permission) -> Option { + if !self.may_ask { + return None; + } + let names: Vec<&str> = self.modules.iter().map(|module| module.id()).collect(); + let name = crate::module_name::resolve(MODULE_TYPE, id, &names)?; + let (position, module) = self + .modules + .iter() + .enumerate() + .find(|(position, module)| module.id() == name && *position != self.position)?; + let input = Self { + consent: self.consent, + evidence: self.evidence, + policy: self.policy, + baseline: self.baseline, + settled: ConsentSignal::Neutral, + modules: self.modules, + position, + may_ask: false, + context: self.context, + }; + Some(module.signal(permission, &input)) + } +} + +/// The type folder of every permission signal crate, which a name written in +/// `[permission-signal] modules` may leave off. +pub const MODULE_TYPE: &str = "permission-signal"; + +/// The name a module reports on the page and in logs, being its name without +/// the type folder, such as `example` for `permission-signal.example`. +#[must_use] +pub fn short_name(module: &dyn PermissionSignalModule) -> &'static str { + crate::module_name::short_form(MODULE_TYPE, module.id()) +} + +/// A module of permission signals. +/// +/// An implementation answers for one signaling scheme. It reads the request, +/// applies whatever the policy says about its own scheme, and returns how it +/// would amend one permission. +/// +/// # Contract +/// +/// Answer [`ConsentSignal::Neutral`] for a permission this module has no +/// opinion on, including when the signal it reads is absent from the request. +/// Returning [`ConsentSignal::Revoke`] for an absent signal would turn silence +/// into refusal and revoke the permission on every request that did not carry +/// this scheme. +pub trait PermissionSignalModule: Send + Sync { + /// The module's name, used in configuration, in logs, and by a peer + /// looking this module up through [`SignalInput::ask`]. + /// + /// A module in a crate takes its crate folder's name through + /// [`crate::module_name!`], such as `permission-signal.example`, which + /// `[permission-signal] modules` may write as `example`. + fn id(&self) -> &'static str; + + /// The permissions this module declares, which decide whether a gated + /// value is passed to it. None by default. Signal modules run before the + /// permissions are resolved, so a declaration here passes nothing yet. + fn required_permissions(&self) -> PermissionSet { + PermissionSet::none() + } + + /// How this module would amend `permission` for this request. + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal; + + /// The permissions this module can ever answer [`ConsentSignal::Grant`] + /// for under `policy`, so a permission a signal could still set can be + /// told from one nothing in this deployment will ever set. + /// + /// Resolution records a permission that requires a signal and got none as + /// awaited, and a page holds what depends on an awaited permission until + /// an answer arrives. That is right only for a permission some configured + /// module could grant. For any other, waiting is waiting for ever, so + /// the assembly narrows the awaited list to the union of these + /// declarations. + /// + /// A scheme that only ever revokes, which is every opt-out, leaves this + /// at its default of nothing. A scheme that grants declares exactly what + /// it maps, under the policy it is given, so a record the policy does not + /// let answer declares nothing either. + fn grants(&self, _policy: &SignalPolicy) -> PermissionSet { + PermissionSet::none() + } + + /// Whether the request carries an explicit withdrawal of `permission` + /// under this scheme, as opposed to merely not granting it. + /// + /// The difference is destructive. A withdrawal of storage expires the + /// browser cookie and writes the authoritative tombstone against the + /// identifier, where a permission that is simply not set for this request + /// strips the response headers and leaves the identifier alone, so that a + /// returning visitor is not permanently withdrawn before they ever get to + /// answer. Most schemes have no such notion, a browser setting and a sale + /// opt-out included, and leave this at its default of `false`. Whether a + /// withdrawal is acted on at all is core's decision from the jurisdiction's + /// baseline, so a refusal only withdraws where the baseline did not grant + /// the permission outright, because where it did the permission never + /// depended on the record. + fn withdraws(&self, _permission: Permission, _input: &SignalInput<'_>) -> bool { + false + } + + /// The signal this module read from the request and used, as it was + /// received, or `None` when there is none it can use. + /// + /// Absent, unreadable, expired and not-acted-on all answer `None`, and + /// nothing here says which. What each of those means for the + /// permissions is this module's decision in [`signal`](Self::signal), + /// taken silently. Core carries what every module vouched for on the + /// permission state, and a signal nobody vouched for is dropped from + /// everything Trusted Server sends on, so a corrupt record never reaches + /// a page or a bid request. See [`crate::module_context`] for what an + /// implementation may name. + fn valid_signal(&self, _call: ModuleCall<'_>) -> Option { + None + } + + /// The terms documents this module says the request's data is + /// available under, empty when it declares none. + /// + /// A locator tells whoever receives the data what terms cover it, so + /// they can decide whether those are terms they accept and whether they + /// may pass the data on. Most schemes carry no terms of their own and + /// leave this at its default, and a module for a terms scheme returns the + /// document that applies to this request. Model Terms for Marketing (MTM) + /// is the first such scheme and one of many rather than the only one. Core + /// does not read the documents, it carries the locators, so what a document + /// says stays between the parties bound by it. + /// + /// A locator must point at a document that is never edited once + /// published, which [`Tdl`] documents and cannot enforce. + fn tdls(&self, _call: ModuleCall<'_>) -> Vec { + Vec::new() + } +} + +/// Asks every module in order and returns what they settle on together. +/// +/// See the module documentation for the layering and why the order is the +/// configuration. +#[must_use] +pub(crate) fn combine( + modules: &[Arc], + permission: Permission, + context: &ModuleContext<'_>, + consent: &ConsentContext, + evidence: &dyn RequestInfo, + policy: &SignalPolicy, + baseline: Acquisition, +) -> ConsentSignal { + let mut settled = ConsentSignal::Neutral; + for (position, module) in modules.iter().enumerate() { + let input = SignalInput { + consent, + evidence, + policy, + baseline, + settled, + modules, + position, + may_ask: true, + context, + }; + // Every module is asked, because a later one may amend what an + // earlier one settled. Stopping at the first answer would make the + // order mean the opposite of what it says. + match module.signal(permission, &input) { + ConsentSignal::Neutral => {} + answer => settled = answer, + } + } + settled +} + +/// Whether the request explicitly withdraws `permission`, scoped to the +/// jurisdiction's baseline for it. +/// +/// A withdrawal only counts where the baseline did not grant the permission +/// outright. Under a `requires_signal` baseline a refusal is the visitor +/// declining the very signal the permission depended on, so it is +/// destructive. Under a `granted` baseline the permission never depended on +/// the record, so the same refusal suppresses use for the request without +/// destroying anything. Any one module answering [`withdraws`] is enough, +/// and no module answering it, or none configured, is never a withdrawal. +/// +/// [`withdraws`]: PermissionSignalModule::withdraws +#[must_use] +pub(crate) fn withdrawn( + modules: &[Arc], + permission: Permission, + context: &ModuleContext<'_>, + consent: &ConsentContext, + evidence: &dyn RequestInfo, + policy: &SignalPolicy, + baseline: Acquisition, +) -> bool { + if matches!(baseline, Acquisition::Granted) { + return false; + } + modules.iter().enumerate().any(|(position, module)| { + let input = SignalInput { + consent, + evidence, + policy, + baseline, + settled: ConsentSignal::Neutral, + modules, + position, + may_ask: true, + context, + }; + module.withdraws(permission, &input) + }) +} + +/// The terms documents the configured modules declare for this request. +/// +/// Asked in the same order the modules answer in, so the list reads the +/// way the deployment is configured, and a document named by two modules +/// is carried once. No module declaring anything leaves the list empty, +/// which says no terms were declared rather than that any terms apply. +#[must_use] +pub(crate) fn tdls( + modules: &[Arc], + context: &ModuleContext<'_>, +) -> Arc<[Tdl]> { + let mut declared: Vec = Vec::new(); + for module in modules { + for tdl in module.tdls(context.call(module.id(), module.required_permissions())) { + if !declared.contains(&tdl) { + declared.push(tdl); + } + } + } + Arc::from(declared) +} + +/// Every permission some module in `modules` declares it can grant under +/// `policy`, which is the most a signal arriving later could still set. +#[must_use] +pub fn answerable( + modules: &[Arc], + policy: &SignalPolicy, +) -> PermissionSet { + modules + .iter() + .map(|module| module.grants(policy)) + .fold(PermissionSet::none(), PermissionSet::union) +} + +/// The signals the configured modules read and found valid, in the order +/// the modules are asked. +/// +/// Each module vouches for at most one signal, its own scheme's, so the +/// list reads the way the deployment is configured. No module vouching +/// for anything leaves the list empty, which says the request carried +/// nothing a configured module could use. +#[must_use] +pub(crate) fn signals( + modules: &[Arc], + context: &ModuleContext<'_>, +) -> Arc<[ValidSignal]> { + modules + .iter() + .filter_map(|module| { + module.valid_signal(context.call(module.id(), module.required_permissions())) + }) + .collect() +} + +/// The modules a deployment named, in the order it named them, drawn from +/// the ones the build makes available. +/// +/// `None` means nothing was configured, which runs every available module +/// in the order the adapter offered them. That is deliberate, so a publisher +/// gets every scheme the build knows about until they say otherwise and a +/// signal is never quietly ignored because someone forgot to list it. An +/// empty list is a publisher acting on no signal at all, and is accepted. +/// +/// # Errors +/// +/// A name matching no available module, or a name given twice, is refused +/// rather than ignored, so a typo cannot silently stop a scheme being honored +/// and a scheme cannot run twice at two places in the order. +pub(crate) fn select( + available: &[Arc], + configured: Option<&[String]>, +) -> Result>, Report> { + let Some(names) = configured else { + return Ok(available.to_vec()); + }; + let mut selected = Vec::with_capacity(names.len()); + for (position, name) in names.iter().enumerate() { + if names[..position].contains(name) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Permission signal module `{name}` is named more than once in \ + [permission-signal] modules. Each module runs once, at one place \ + in the order" + ), + })); + } + let offered = ids(available); + let Some(module) = crate::module_name::resolve(MODULE_TYPE, name, &offered) + .and_then(|resolved| available.iter().find(|module| module.id() == resolved)) + else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Permission signal module `{name}` is not available in this build. \ + Available modules are {}", + offered + .iter() + .map(|id| crate::module_name::short_form(MODULE_TYPE, id)) + .collect::>() + .join(", ") + ), + })); + }; + selected.push(Arc::clone(module)); + } + Ok(selected) +} + +/// The available modules a configured list leaves out, for the startup log. +#[must_use] +pub(crate) fn omitted<'a>( + available: &'a [Arc], + configured: Option<&[String]>, +) -> Vec<&'a str> { + let Some(names) = configured else { + return Vec::new(); + }; + available + .iter() + .map(|module| module.id()) + .filter(|id| !names.iter().any(|name| name == id)) + .collect() +} + +/// The identifiers of `modules`, in order. +#[must_use] +pub(crate) fn ids(modules: &[Arc]) -> Vec<&str> { + modules.iter().map(|module| module.id()).collect() +} + +/// Selects the modules a deployment runs from the ones an adapter makes +/// available, and says so in the log once at startup. +/// +/// This is the adapter's composition point. Core supplies no module of its +/// own, so an adapter hands in every scheme crate it links, in the order that +/// stands when configuration names none, and receives back the ordered list +/// the request path asks. The list is shared rather than owned, so handing it +/// to every request's services is one reference count and no copy. +/// +/// # Errors +/// +/// A configured name that matches nothing available, or a name given twice, +/// fails startup with a message naming what is available, so a typo cannot +/// silently stop a scheme being honored. +pub fn build_permission_signal_modules( + settings: &Settings, + available: &[Arc], +) -> Result]>, Report> { + let configured = settings.permission_signal.modules.as_deref(); + let selected = select(available, configured)?; + match configured { + None => log::info!( + "Permission signals: acting on every module this build offers, [{}], no \ + [permission-signal] modules configured", + ids(&selected).join(", ") + ), + Some([]) => log::info!( + "Permission signals: acting on no module, [permission-signal] modules is \ + empty, so every permission stays at its country and region baseline" + ), + Some(_) => log::info!( + "Permission signals: acting on [{}], asked in that order", + ids(&selected).join(", ") + ), + } + let left_out = omitted(available, configured); + if !left_out.is_empty() { + log::warn!( + "Permission signals: not acting on [{}], which are not in [permission-signal] \ + modules. A signal this deployment does not act on is read from the request \ + and then ignored", + left_out.join(", ") + ); + } + Ok(Arc::from(selected)) +} + +#[cfg(test)] +mod tests { + use http::HeaderMap; + + use super::*; + use crate::evidence::OwnedRequestInfo; + use crate::module_context::test_support; + + /// A module that always answers the same thing, for testing the rule + /// rather than any particular scheme. + struct Fixed(&'static str, ConsentSignal); + + impl PermissionSignalModule for Fixed { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + self.1 + } + } + + /// A module that answers by consulting a peer, which is the behavior the + /// peer visibility exists for. + struct Consulting { + id: &'static str, + peer: &'static str, + } + + impl PermissionSignalModule for Consulting { + fn id(&self) -> &'static str { + self.id + } + + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + match input.ask(self.peer, permission) { + // The peer refused, and this module takes the opposite view + // of the same request, which is the override the seam allows. + Some(ConsentSignal::Revoke) => ConsentSignal::Grant, + _ => ConsentSignal::Neutral, + } + } + } + + /// A module that declares a terms document, which is what a scheme like + /// Model Terms for Marketing does and most schemes do not. + struct Declaring(&'static str, &'static str); + + impl PermissionSignalModule for Declaring { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Neutral + } + + fn tdls(&self, _call: ModuleCall<'_>) -> Vec { + vec![Tdl::new(self.1).expect("should accept the test locator")] + } + } + + /// A module that withdraws storage, for testing the scoping rule. + struct Withdrawing; + + impl PermissionSignalModule for Withdrawing { + fn id(&self) -> &'static str { + "withdrawing" + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Revoke + } + + fn withdraws(&self, permission: Permission, _input: &SignalInput<'_>) -> bool { + permission == Permission::StoreOnDevice + } + } + + fn no_evidence() -> OwnedRequestInfo { + OwnedRequestInfo::new(String::new(), HeaderMap::new()) + } + + /// The signals `modules` vouch for, asked in a context carrying `consent` + /// and no evidence. + fn signals_for( + modules: &[Arc], + consent: &ConsentContext, + ) -> Arc<[ValidSignal]> { + let evidence = no_evidence(); + let context = ModuleContext::new(test_support::request("/")) + .with_consent(consent) + .with_evidence(&evidence); + signals(modules, &context) + } + + /// The terms `modules` declare, asked as [`signals_for`] asks. + fn tdls_for( + modules: &[Arc], + consent: &ConsentContext, + ) -> Arc<[Tdl]> { + let evidence = no_evidence(); + let context = ModuleContext::new(test_support::request("/")) + .with_consent(consent) + .with_evidence(&evidence); + tdls(modules, &context) + } + + fn fixed(id: &'static str, signal: ConsentSignal) -> Arc { + Arc::new(Fixed(id, signal)) + } + + /// A module that declares it can grant one permission, whatever it then + /// answers, standing in for a scheme with a mapping. + struct Granting(&'static str, Permission); + + impl PermissionSignalModule for Granting { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Neutral + } + + fn grants(&self, _policy: &SignalPolicy) -> PermissionSet { + PermissionSet::none().with(self.1) + } + } + + #[test] + fn a_module_declares_nothing_grantable_unless_it_says_otherwise() { + // An opt-out only ever revokes, so the default declaration is empty + // and a deployment of opt-outs alone can grant nothing. + let policy = SignalPolicy::default(); + assert!( + fixed("opt_out", ConsentSignal::Revoke) + .grants(&policy) + .is_empty() + ); + assert!(answerable(&[fixed("a", ConsentSignal::Grant)], &policy).is_empty()); + } + + #[test] + fn what_is_answerable_is_the_union_of_every_declaration() { + let policy = SignalPolicy::default(); + let modules: Vec> = vec![ + Arc::new(Granting("storage", Permission::StoreOnDevice)), + fixed("opt_out", ConsentSignal::Revoke), + Arc::new(Granting("profiling", Permission::CreateAdsProfile)), + ]; + assert_eq!( + answerable(&modules, &policy), + PermissionSet::none() + .with(Permission::StoreOnDevice) + .with(Permission::CreateAdsProfile), + "should collect what every module declares, in any order" + ); + } + + /// A module that vouches for a signal, standing in for a scheme that + /// read its string and could decode it. + struct Vouching(&'static str, &'static str); + + impl PermissionSignalModule for Vouching { + fn id(&self) -> &'static str { + self.0 + } + + fn signal(&self, _permission: Permission, _input: &SignalInput<'_>) -> ConsentSignal { + ConsentSignal::Neutral + } + + fn valid_signal(&self, _call: ModuleCall<'_>) -> Option { + Some(ValidSignal::new(self.0, self.0, self.1)) + } + } + + #[test] + fn the_valid_signals_are_what_each_module_vouched_for_in_order() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Vouching("first", "one")), + fixed("silent", ConsentSignal::Revoke), + Arc::new(Vouching("second", "two")), + ]; + let valid = signals_for(&modules, &consent); + assert_eq!( + &*valid, + &[ + ValidSignal::new("first", "first", "one"), + ValidSignal::new("second", "second", "two"), + ], + "should carry what was vouched for, in configured order, and nothing else" + ); + assert!( + signals_for(&[fixed("silent", ConsentSignal::Grant)], &consent).is_empty(), + "a module vouches for nothing unless it says otherwise" + ); + } + + #[test] + fn a_module_declaring_no_terms_leaves_the_list_empty() { + let consent = ConsentContext::default(); + let declared = tdls_for(&[fixed("quiet", ConsentSignal::Grant)], &consent); + assert!( + declared.is_empty(), + "should declare nothing, because the IAB schemes carry no terms" + ); + } + + #[test] + fn terms_are_collected_in_the_order_the_modules_are_asked() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Declaring("first", "https://terms.example.com/a/1.txt")), + fixed("quiet", ConsentSignal::Neutral), + Arc::new(Declaring("second", "https://terms.example.com/b/1.txt")), + ]; + let declared = tdls_for(&modules, &consent); + let addresses: Vec<&str> = declared.iter().map(Tdl::as_str).collect(); + assert_eq!( + addresses, + vec![ + "https://terms.example.com/a/1.txt", + "https://terms.example.com/b/1.txt" + ], + "should read in the configured order, so the list matches the deployment" + ); + } + + #[test] + fn one_document_named_by_two_modules_is_carried_once() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Declaring("first", "https://terms.example.com/a/1.txt")), + Arc::new(Declaring("second", "https://terms.example.com/a/1.txt")), + ]; + let declared = tdls_for(&modules, &consent); + assert_eq!( + declared.len(), + 1, + "should carry the same document once, not once per module naming it" + ); + } + + #[test] + fn versions_of_one_document_are_both_carried() { + let consent = ConsentContext::default(); + let modules: Vec> = vec![ + Arc::new(Declaring("first", "https://terms.example.com/a/1.txt")), + Arc::new(Declaring("second", "https://terms.example.com/a/2.txt")), + ]; + let declared = tdls_for(&modules, &consent); + assert_eq!( + declared.len(), + 2, + "should keep both, because a recipient agreed to one version and not the other" + ); + } + + #[test] + fn no_modules_declare_nothing() { + let consent = ConsentContext::default(); + assert!( + tdls_for(&[], &consent).is_empty(), + "should declare nothing when no module runs, rather than implying terms" + ); + } + + fn combined(modules: &[Arc]) -> ConsentSignal { + let consent = ConsentContext::default(); + let policy = SignalPolicy::default(); + combine( + modules, + Permission::StoreOnDevice, + ModuleContext::empty(), + &consent, + &no_evidence(), + &policy, + Acquisition::RequiresSignal, + ) + } + + /// The same, for a run whose modules differ only in what they answer. + fn combined_signals(signals: &[ConsentSignal]) -> ConsentSignal { + let modules: Vec> = signals + .iter() + .map(|signal| fixed("test", *signal)) + .collect(); + combined(&modules) + } + + fn withdrawn_under(modules: &[Arc], baseline: Acquisition) -> bool { + let consent = ConsentContext::default(); + let policy = SignalPolicy::default(); + withdrawn( + modules, + Permission::StoreOnDevice, + ModuleContext::empty(), + &consent, + &no_evidence(), + &policy, + baseline, + ) + } + + fn names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_owned()).collect() + } + + // ------------------------------------------------------------------ + // Combining answers in order. + // ------------------------------------------------------------------ + + #[test] + fn no_modules_leaves_the_place_baseline_alone() { + assert_eq!( + combined_signals(&[]), + ConsentSignal::Neutral, + "with no module configured nothing amends the place baseline" + ); + } + + #[test] + fn the_last_module_with_an_opinion_decides() { + assert_eq!( + combined_signals(&[ConsentSignal::Revoke, ConsentSignal::Grant]), + ConsentSignal::Grant, + "a visitor who answers a prompt after arriving with an opt-out header has \ + their answer applied over it, which is why the order is the policy" + ); + assert_eq!( + combined_signals(&[ConsentSignal::Grant, ConsentSignal::Revoke]), + ConsentSignal::Revoke, + "and a deployment that wants the opt-out to win puts it last" + ); + } + + #[test] + fn silence_leaves_an_earlier_answer_standing() { + // The failure this guards is a later module overwriting a settled + // answer with its own absence, which would let adding a module nobody + // uses undo the one that was working. + assert_eq!( + combined_signals(&[ConsentSignal::Grant, ConsentSignal::Neutral]), + ConsentSignal::Grant, + "a later module with no opinion leaves an earlier grant standing" + ); + assert_eq!( + combined_signals(&[ConsentSignal::Revoke, ConsentSignal::Neutral]), + ConsentSignal::Revoke, + "and a later silence leaves an earlier refusal standing too" + ); + } + + #[test] + fn silence_from_every_module_is_not_a_refusal() { + // The failure this guards is a module that reads an absent signal as + // a refusal. It would revoke the permission on every request that did + // not carry that scheme, which is most of them. + assert_eq!( + combined_signals(&[ConsentSignal::Neutral, ConsentSignal::Neutral]), + ConsentSignal::Neutral, + "no module having an opinion is not a refusal" + ); + } + + #[test] + fn a_module_sees_what_the_earlier_ones_settled() { + struct Recording; + + impl PermissionSignalModule for Recording { + fn id(&self) -> &'static str { + "recording" + } + + fn signal(&self, _permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + assert_eq!( + input.settled, + ConsentSignal::Revoke, + "a module is asked with the value the modules before it settled on" + ); + assert_eq!(input.position(), 1, "and with its own place in the order"); + assert_eq!(input.modules().len(), 2, "and with the whole list"); + assert_eq!( + input.baseline, + Acquisition::RequiresSignal, + "and with what the place rules said before anyone was asked" + ); + ConsentSignal::Neutral + } + } + + let modules: Vec> = + vec![fixed("opt-out", ConsentSignal::Revoke), Arc::new(Recording)]; + assert_eq!( + combined(&modules), + ConsentSignal::Revoke, + "the recording module has no opinion, so the opt-out stands" + ); + } + + #[test] + fn a_module_can_override_a_peer_by_consulting_it() { + // The worked example from the README: a later module overrides a + // refusal because of who made it, not merely that one was made. + let modules: Vec> = vec![ + fixed("gpc", ConsentSignal::Revoke), + Arc::new(Consulting { + id: "prompt", + peer: "gpc", + }), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Grant, + "a later module overrides a refusal made by the peer it consulted" + ); + + // The same module leaves the refusal alone when it came from a peer + // it was not told to override. + let modules: Vec> = vec![ + fixed("other", ConsentSignal::Revoke), + Arc::new(Consulting { + id: "prompt", + peer: "gpc", + }), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Revoke, + "and leaves a refusal from any other peer standing" + ); + } + + #[test] + fn asking_reaches_a_peer_wherever_it_sits_in_the_order() { + // A module may consult one configured after it, not only before, so + // a reordering does not silently change what a module can see. + let modules: Vec> = vec![ + Arc::new(Consulting { + id: "prompt", + peer: "gpc", + }), + fixed("gpc", ConsentSignal::Revoke), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Revoke, + "the consultation succeeded, and the later opt-out then settled it" + ); + } + + #[test] + fn asking_for_a_module_that_is_not_configured_answers_nothing() { + struct Absent; + + impl PermissionSignalModule for Absent { + fn id(&self) -> &'static str { + "absent" + } + + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + assert!( + input.ask("not-configured", permission).is_none(), + "a module must be able to tell a missing peer from a silent one" + ); + assert!( + !input.has("not-configured"), + "and sees that the peer is not in the list" + ); + assert!(input.has("absent"), "and can see itself in the list"); + ConsentSignal::Neutral + } + } + + let modules: Vec> = vec![Arc::new(Absent)]; + assert_eq!( + combined(&modules), + ConsentSignal::Neutral, + "a module that finds its peer missing leaves the permission unsettled" + ); + } + + #[test] + fn a_module_cannot_consult_itself() { + struct SelfAsking; + + impl PermissionSignalModule for SelfAsking { + fn id(&self) -> &'static str { + "self-asking" + } + + fn signal(&self, permission: Permission, input: &SignalInput<'_>) -> ConsentSignal { + // Without the guard this recurses until the stack is gone. + assert!( + input.ask("self-asking", permission).is_none(), + "a module asking itself gets no answer" + ); + ConsentSignal::Grant + } + } + + let modules: Vec> = vec![Arc::new(SelfAsking)]; + assert_eq!( + combined(&modules), + ConsentSignal::Grant, + "a module refused its own consultation still answers for itself" + ); + } + + #[test] + fn two_modules_that_consult_each_other_do_not_loop() { + // Each consults the other, and the one answering a consultation is + // refused a consultation of its own, so the pair settles instead of + // recursing. + let modules: Vec> = vec![ + Arc::new(Consulting { + id: "first", + peer: "second", + }), + Arc::new(Consulting { + id: "second", + peer: "first", + }), + ]; + assert_eq!( + combined(&modules), + ConsentSignal::Neutral, + "two modules consulting each other settle instead of looping" + ); + } + + // ------------------------------------------------------------------ + // Withdrawal scoping. + // ------------------------------------------------------------------ + + #[test] + fn a_withdrawal_counts_only_where_the_baseline_did_not_grant() { + let modules: Vec> = vec![Arc::new(Withdrawing)]; + assert!( + withdrawn_under(&modules, Acquisition::RequiresSignal), + "refusing the signal the permission depended on is destructive" + ); + assert!( + withdrawn_under(&modules, Acquisition::Denied), + "and so is refusing under a baseline that never allowed it" + ); + assert!( + !withdrawn_under(&modules, Acquisition::Granted), + "where the permission never depended on the record, the refusal suppresses \ + without destroying" + ); + } + + #[test] + fn a_module_that_merely_revokes_does_not_withdraw() { + // Revoke and withdraw are different questions. A sale opt-out revokes + // and must never destroy an identifier. + let modules: Vec> = + vec![fixed("opt-out", ConsentSignal::Revoke)]; + assert!( + !withdrawn_under(&modules, Acquisition::RequiresSignal), + "a module that only revokes never withdraws" + ); + } + + #[test] + fn no_modules_never_withdraw() { + assert!( + !withdrawn_under(&[], Acquisition::RequiresSignal), + "with no module configured nothing can withdraw" + ); + } + + // ------------------------------------------------------------------ + // Selecting which modules run. + // ------------------------------------------------------------------ + + fn four() -> Vec> { + vec![ + fixed("gpc", ConsentSignal::Neutral), + fixed("gpp", ConsentSignal::Neutral), + fixed("us-privacy", ConsentSignal::Neutral), + fixed("tcf", ConsentSignal::Neutral), + ] + } + + #[test] + fn naming_nothing_runs_every_available_module_in_the_offered_order() { + let selected = select(&four(), None).expect("should accept no configuration"); + assert_eq!( + ids(&selected), + vec!["gpc", "gpp", "us-privacy", "tcf"], + "a publisher who configures nothing acts on every scheme the build knows, so \ + one is never ignored because they forgot to list it" + ); + } + + #[test] + fn the_configured_order_is_the_order_they_run_in() { + let reversed = names(&["tcf", "us-privacy", "gpp", "gpc"]); + let selected = select(&four(), Some(&reversed)).expect("should accept known names"); + assert_eq!( + ids(&selected), + vec!["tcf", "us-privacy", "gpp", "gpc"], + "the list is the order, not merely the membership" + ); + } + + #[test] + fn an_empty_list_is_acting_on_no_signal_and_is_accepted() { + let selected = select(&four(), Some(&[])).expect("should accept an empty list"); + assert!(selected.is_empty(), "an empty list selects no module"); + assert_eq!( + omitted(&four(), Some(&[])).len(), + 4, + "and every module is reported left out" + ); + } + + #[test] + fn a_name_matching_no_available_module_is_refused() { + // Matched rather than `expect_err`, because the success type holds + // trait objects that are deliberately not `Debug`. + let Err(error) = select(&four(), Some(&names(&["gpc", "not-a-module"]))) else { + panic!("should refuse a name this build does not offer"); + }; + let message = format!("{error:?}"); + assert!( + message.contains("not-a-module") && message.contains("gpc, gpp"), + "the refusal names the bad entry and what is available: {message}" + ); + } + + #[test] + fn naming_a_module_twice_is_refused() { + let Err(error) = select(&four(), Some(&names(&["gpc", "tcf", "gpc"]))) else { + panic!("should refuse a module named twice"); + }; + assert!( + format!("{error:?}").contains("more than once"), + "a module runs once, at one place in the order" + ); + } + + #[test] + fn a_left_out_module_is_reported_as_omitted() { + let configured = names(&["gpc", "tcf"]); + assert_eq!( + omitted(&four(), Some(&configured)), + vec!["gpp", "us-privacy"], + "the modules the list leaves out are reported in the offered order" + ); + assert!( + omitted(&four(), None).is_empty(), + "configuring nothing omits nothing" + ); + } +} diff --git a/crates/trusted-server-core/src/permissions.rs b/crates/trusted-server-core/src/permissions.rs new file mode 100644 index 0000000000..77d3dc5867 --- /dev/null +++ b/crates/trusted-server-core/src/permissions.rs @@ -0,0 +1,2552 @@ +//! Module permissions: a technical permission model gating module execution. +//! +//! A module advertises the [`Permission`]s its data use *requires*. Trusted +//! Server resolves which permissions are currently *set* from the session's +//! signals and the country it resolves to, and refuses to run the Edge Cookie +//! module when its required permissions are not set. The device and geo +//! modules declare their requirements through the same method, and the +//! built-in and host ones require none. A device module supplied by an +//! integration module that declares any permission is refused at startup, +//! because no per-request device gate exists yet to honor it. A geo +//! declaration is not consulted, because geo runs before the permissions are +//! assembled and supplies the country they depend on. +//! +//! The vocabulary is the IAB Privacy Taxonomy Data Uses, mapped from the IAB TCF +//! Europe purposes and used **only** as a technical identifier for a permission. +//! Two purposes have no Data Use yet: TCF purpose 1 (device storage) uses a +//! proposed `necessary.operations.storage` key and TCF purpose 11 keeps its TCF +//! identifier. No CMP or TCF *policy* is implemented here. Every TCF purpose +//! is resolved against the session's signals (a present TCF record grants or +//! revokes each purpose, and a US-style opt-out revokes), and the Data Uses +//! with no TCF purpose keep their configured baseline. +//! +//! How a permission is acquired varies by place, so the policy is written as a +//! tree of places: the top of the tree stands for the whole world, countries +//! sit under it keyed by ISO 3166-1 alpha-2 code, and a country's regions sit +//! under that country keyed by ISO 3166-2 subdivision code. +//! [`PermissionMaps::standard`] loads that tree from the embedded +//! `permissions.yaml` (see `DEFAULT_PERMISSION_RULES`). +//! +//! Resolution takes the most specific match and falls back to the node above: +//! the request's region when it is listed, otherwise its country, otherwise +//! the top of the tree. So a request with no country at all (no geo module, +//! or a lookup that resolved nothing), and a request whose country has no rule, +//! both resolve to the top node's group. The top node also declares the +//! `jurisdiction` the consent gates use for a visitor whose place could not be +//! resolved (see [`PermissionMaps::default_jurisdiction`]). A geo module that +//! reports an outright lookup failure is the exception, resolving every +//! permission to the requires-signal floor rather than the top node (see +//! [`PermissionMaps::floor_with`]). + +use std::collections::BTreeMap; +use std::sync::{Arc, OnceLock}; + +use serde::Deserialize; +use serde_yaml_ng::Value; + +use crate::consent::jurisdiction::Jurisdiction; +use crate::tdl::Tdl; + +/// A technical permission a module may require, labeled with its IAB Privacy +/// Taxonomy Data Use, or its IAB TCF Europe purpose where no Data Use exists yet. +/// +/// Only the identifier is used, with no TCF or taxonomy policy implemented. Every +/// named variant with a TCF purpose in `permissions.yaml` is resolved against +/// the session's signals. Only [`Permission::StoreOnDevice`] (and +/// [`Permission::SelectPersonalisedAds`] for sharing) gates a shipped module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Permission { + /// TCF Purpose 1, store and/or access information on a device. It gates + /// every shipped Edge Cookie module. No IAB Privacy Taxonomy Data Use exists + /// for device storage yet, so this uses a proposed `necessary.operations` + /// key pending an upstream addition. + StoreOnDevice, + /// TCF Purpose 2, use limited data to select advertising. + SelectBasicAds, + /// TCF Purpose 3, create profiles for personalised advertising. + CreateAdsProfile, + /// TCF Purpose 4, use profiles to select personalised advertising. + SelectPersonalisedAds, + /// TCF Purpose 5, create profiles to personalise content. + CreateContentProfile, + /// TCF Purpose 6, use profiles to select personalised content. + SelectPersonalisedContent, + /// TCF Purpose 7, measure advertising performance. + MeasureAdPerformance, + /// TCF Purpose 8, measure content performance. + MeasureContentPerformance, + /// TCF Purpose 9, understand audiences through statistics. + MarketResearch, + /// TCF Purpose 10, develop and improve services. + DevelopServices, + /// TCF Purpose 11, use limited data to select content. No IAB Privacy + /// Taxonomy Data Use exists for limited-data content selection yet, so this + /// keeps its TCF identifier and is proposed upstream. No shipped module + /// gates on it. + SelectBasicContent, + /// An IAB Privacy Taxonomy Data Use with no dedicated variant, identified by + /// its index into [`EXTRA_DATA_USES`]. These carry a policy flag in + /// `permissions.yaml` for completeness, and no shipped module gates on them. + Extra(u8), +} + +/// The Data Use identifiers for the named [`Permission`] variants, in variant +/// order (bit index 0..11). +const NAMED_DATA_USES: [&str; 11] = [ + "necessary.operations.storage", + "advertising_marketing.first_party.contextual", + "advertising_marketing.profiling", + "advertising_marketing.first_party.targeted", + "advertising_marketing.personalize.profiling", + "advertising_marketing.personalize.content", + "analytics.ad_reporting.measure_ad_performance", + "analytics.ad_reporting.content_performance", + "analytics.ad_reporting.market_research", + "necessary.operations.improve", + "select-basic-content", +]; + +/// Every other IAB Privacy Taxonomy Data Use, carried so `permissions.yaml` can +/// set a policy flag for the whole taxonomy (bit index 11..). No shipped module +/// gates on these. They exist for completeness, testing, and demonstration. +const EXTRA_DATA_USES: [&str; 53] = [ + "advertising_marketing", + "advertising_marketing.communications", + "advertising_marketing.communications.email", + "advertising_marketing.communications.sms", + "advertising_marketing.first_party", + "advertising_marketing.frequency_capping", + "advertising_marketing.negative_targeting", + "advertising_marketing.personalize", + "advertising_marketing.personalize.system", + "advertising_marketing.serving", + "advertising_marketing.third_party", + "advertising_marketing.third_party.targeted", + "analytics", + "analytics.ad_reporting", + "analytics.ad_reporting.ad_delivery_and_targeting", + "analytics.ad_reporting.ad_fraud_detection", + "analytics.ad_reporting.ad_viewability", + "analytics.ad_reporting.campaign_insights", + "analytics.reporting", + "analytics.reporting.system", + "disclosure", + "disclosure.law_enforcement", + "disclosure.outside_counsel", + "disclosure.sale", + "disclosure.share", + "disclosure.third_party_sale", + "functional", + "functional.performance", + "functional.personalization", + "functional.security", + "necessary", + "necessary.employment", + "necessary.employment.hr", + "necessary.employment.hr.hiring", + "necessary.fraud_detection", + "necessary.legal_obligation", + "necessary.legal_obligation.age_verification", + "necessary.legal_obligation.content_moderation", + "necessary.legal_obligation.dsr", + "necessary.legal_obligation.hold", + "necessary.operations", + "necessary.operations.authentication", + "necessary.operations.debugging", + "necessary.operations.notifications", + "necessary.operations.notifications.email", + "necessary.operations.notifications.sms", + "necessary.operations.payment_processing", + "necessary.operations.quality_assurance", + "necessary.operations.security", + "necessary.operations.support", + "necessary.operations.survey", + "necessary.operations.upgrades", + "necessary.operations.website_use", +]; + +impl Permission { + /// The named permission variants, in bit-index order. + const NAMED: [Permission; 11] = [ + Permission::StoreOnDevice, + Permission::SelectBasicAds, + Permission::CreateAdsProfile, + Permission::SelectPersonalisedAds, + Permission::CreateContentProfile, + Permission::SelectPersonalisedContent, + Permission::MeasureAdPerformance, + Permission::MeasureContentPerformance, + Permission::MarketResearch, + Permission::DevelopServices, + Permission::SelectBasicContent, + ]; + + /// Every modeled permission: the named variants, then every other Privacy + /// Taxonomy Data Use. + pub fn all() -> impl Iterator { + Self::NAMED + .into_iter() + .chain((0..EXTRA_DATA_USES.len() as u8).map(Permission::Extra)) + } + + /// The Data Use identifier for this permission. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Permission::Extra(index) => EXTRA_DATA_USES[index as usize], + named => NAMED_DATA_USES[named.index() as usize], + } + } + + /// The stable bit position for this permission within a [`PermissionSet`]. + #[must_use] + const fn index(self) -> u8 { + match self { + Permission::StoreOnDevice => 0, + Permission::SelectBasicAds => 1, + Permission::CreateAdsProfile => 2, + Permission::SelectPersonalisedAds => 3, + Permission::CreateContentProfile => 4, + Permission::SelectPersonalisedContent => 5, + Permission::MeasureAdPerformance => 6, + Permission::MeasureContentPerformance => 7, + Permission::MarketResearch => 8, + Permission::DevelopServices => 9, + Permission::SelectBasicContent => 10, + Permission::Extra(index) => 11 + index, + } + } + + /// The single-bit mask for this permission within a [`PermissionSet`]. + const fn bit(self) -> u128 { + 1 << self.index() + } + + /// Returns the permission whose Data Use identifier matches `id` (for + /// example `"necessary.operations.storage"`), or `None` when it is unknown. + /// + /// Used to parse permission names from `permissions.yaml`. + #[must_use] + pub fn from_identifier(id: &str) -> Option { + Permission::all().find(|p| p.as_str() == id) + } +} + +impl core::fmt::Display for Permission { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A set of [`Permission`]s, stored as a bitset keyed by each permission's bit +/// index. +/// +/// Used both for what a module requires and for what Trusted Server has set. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PermissionSet(u128); + +impl PermissionSet { + /// The empty set, requiring or containing nothing. + #[must_use] + pub const fn none() -> Self { + Self(0) + } + + /// Returns this set with `permission` added. + #[must_use] + pub const fn with(self, permission: Permission) -> Self { + Self(self.0 | permission.bit()) + } + + /// Whether `permission` is in the set. + #[must_use] + pub const fn contains(self, permission: Permission) -> bool { + self.0 & permission.bit() != 0 + } + + /// Whether the set is empty. + #[must_use] + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Whether every permission in `other` is also in this set. + #[must_use] + pub const fn contains_all(self, other: PermissionSet) -> bool { + self.0 & other.0 == other.0 + } + + /// The permissions in both this set and `other`. + #[must_use] + pub const fn intersect(self, other: PermissionSet) -> Self { + Self(self.0 & other.0) + } + + /// The permissions in either this set or `other`. + #[must_use] + pub const fn union(self, other: PermissionSet) -> Self { + Self(self.0 | other.0) + } + + /// The permissions in this set and not in `other`. + #[must_use] + pub const fn difference(self, other: PermissionSet) -> Self { + Self(self.0 & !other.0) + } + + /// Iterates the permissions in the set, in bit-index order. + /// + /// The built-ins read nothing from the full set; this serves a module or + /// diagnostic path that enumerates what is present. + pub fn iter(self) -> impl Iterator { + Permission::all().filter(move |p| self.contains(*p)) + } +} + +/// The Data Use identifiers in the set, in bit-index order, separated by +/// `, `, or `nothing` for the empty set. +impl core::fmt::Display for PermissionSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + if self.is_empty() { + return f.write_str("nothing"); + } + for (index, permission) in self.iter().enumerate() { + if index > 0 { + f.write_str(", ")?; + } + f.write_str(permission.as_str())?; + } + Ok(()) + } +} + +impl FromIterator for PermissionSet { + fn from_iter>(iter: I) -> Self { + iter.into_iter() + .fold(PermissionSet::none(), PermissionSet::with) + } +} + +/// How a permission is acquired in a given country. +/// +/// This is intentionally country-keyed, not module-keyed: a module only +/// advertises *which* permissions it needs, and the country's rules decide *how* +/// each is obtained. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Acquisition { + /// Set without any signal, exempt or strictly necessary here. + Granted, + /// Set only when the incoming signals grant the matching TCF purpose. + /// The default, matching the floor an unresolved location falls to. + #[default] + RequiresSignal, + /// Never set in this country. + Denied, +} + +/// What a session signal says about a permission, layered on top of the +/// country/region baseline by the consent mapping. +/// +/// This module never reads consent directly. A caller maps its consent model (or +/// any other signal source) to a [`ConsentSignal`] per permission, and the +/// permission model applies it: a [`Grant`](Self::Grant) sets a +/// `RequiresSignal` permission, a [`Revoke`](Self::Revoke) drops a `Granted` one +/// (an opt-out), and [`Neutral`](Self::Neutral) leaves the baseline unchanged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConsentSignal { + /// The signal grants this permission, so a `RequiresSignal` baseline is set. + Grant, + /// The signal withdraws this permission, dropping a `Granted` baseline. + Revoke, + /// The signal says nothing, so the baseline stands. + Neutral, +} + +/// The acquisition rule for each permission in one country or region. +/// +/// A `default` applies to any permission not explicitly overridden, so a rule +/// table stays compact. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CountryRules { + default: Acquisition, + overrides: BTreeMap, +} + +impl CountryRules { + /// Rules with `default` for every permission and no per-permission override. + /// Groups in `permissions.yaml` are built from this plus [`with_rule`]. + /// + /// [`with_rule`]: Self::with_rule + #[must_use] + pub fn with_default(default: Acquisition) -> Self { + Self { + default, + overrides: BTreeMap::new(), + } + } + + /// Sets the acquisition rule for a single permission, overriding the default. + #[must_use] + pub fn with_rule(mut self, permission: Permission, acquisition: Acquisition) -> Self { + self.overrides.insert(permission.index(), acquisition); + self + } + + /// The acquisition rule for `permission`. + #[must_use] + pub fn rule_for(&self, permission: Permission) -> Acquisition { + self.overrides + .get(&permission.index()) + .copied() + .unwrap_or(self.default) + } +} + +/// Which Data Uses a signal revokes. +#[derive(Debug, Clone, Default)] +enum RevokeSet { + /// The signal revokes nothing. + #[default] + None, + /// The signal revokes every Data Use (the map bounds what a revoke drops). + All, + /// The signal revokes only the listed Data Uses. + Set(PermissionSet), +} + +/// What a deployment decides about the shipped signal schemes, parsed from the +/// `signals` section of `permissions.yaml`. +/// +/// The permission model holds this as data so a deployment changes it without +/// changing a module. It is jurisdiction-free, and it carries only the +/// decisions that are a deployment's to make: whether a TCF record answers at +/// all, which signals count as a US-style opt-out, and what an opt-out takes +/// away. What each scheme's own signal means, such as which TCF purpose grants +/// which Data Use, is that scheme's module crate's, not this policy's. The +/// country/region baseline decides the rest. +#[derive(Debug, Clone, Default)] +pub struct SignalPolicy { + /// Whether a present TCF record's grants and revokes apply. Whether a + /// consenting record then stands over an opt-out, or the opt-out over it, + /// is decided by the order the modules are asked in, which is + /// `[permission-signal] modules`, not by this flag. + tcf_authoritative: bool, + /// The signals that constitute a US-style opt-out. + opt_out_sources: Vec, + /// Which Data Uses a US-style opt-out revokes. + opt_out_revokes: RevokeSet, +} + +impl SignalPolicy { + /// Whether a present TCF record's grants and revokes apply. + #[must_use] + pub fn tcf_authoritative(&self) -> bool { + self.tcf_authoritative + } + + /// The signals that constitute a US-style opt-out. + #[must_use] + pub fn opt_out_sources(&self) -> &[OptOutSource] { + &self.opt_out_sources + } + + /// Whether a US-style opt-out revokes `permission`. + #[must_use] + pub fn opt_out_revokes(&self, permission: Permission) -> bool { + match &self.opt_out_revokes { + RevokeSet::None => false, + RevokeSet::All => true, + RevokeSet::Set(set) => set.contains(permission), + } + } +} + +/// Builds a validated [`SignalPolicy`] from the parsed `signals` section, +/// erroring when it names an unknown Data Use or revoke rule. +fn build_signal_policy(spec: &SignalsSpec) -> Result { + let mut policy = SignalPolicy::default(); + if let Some(tcf) = &spec.tcf { + policy.tcf_authoritative = tcf.authoritative; + } + if let Some(opt_out) = &spec.us_opt_out { + policy.opt_out_sources = opt_out.sources.clone(); + policy.opt_out_revokes = match &opt_out.revokes { + RevokeSpec::Keyword(keyword) if keyword == "all" => RevokeSet::All, + RevokeSpec::Keyword(other) => { + return Err(PermissionsError::UnknownRevoke { + value: other.clone(), + }); + } + RevokeSpec::List(names) => { + let mut set = PermissionSet::none(); + for name in names { + let permission = Permission::from_identifier(name).ok_or_else(|| { + PermissionsError::UnknownPermission { name: name.clone() } + })?; + set = set.with(permission); + } + RevokeSet::Set(set) + } + }; + } + Ok(policy) +} + +/// The place tree from `permissions.yaml`, flattened for lookup. +/// +/// `by_country` is keyed on the ISO 3166-1 alpha-2 code a geo module returns +/// (upper-cased). `by_region` keeps the finer rules written under a country, +/// keyed by country and region (for example a US state), which take precedence +/// over the country entry. `default_rules` is the top node's group, the answer +/// for a request whose country and region are both unlisted, and for a request +/// with no place at all. +#[derive(Debug, Clone, Default)] +pub struct PermissionMaps { + by_country: BTreeMap, + by_region: BTreeMap, + jurisdiction_by_country: BTreeMap, + jurisdiction_by_region: BTreeMap, + default_rules: Option, + default_jurisdiction: Option, + signals: SignalPolicy, +} + +/// The default permission rules, compiled into the build from the repository's +/// sample in `config/permissions`. +/// +/// That sample is for testing and evaluation only and is not a production +/// policy. A deployer chooses or replaces the compiled-in file to set the +/// default policy, which is not read at runtime. +const DEFAULT_PERMISSION_RULES: &str = include_str!("../../../config/permissions/sample.yaml"); + +/// Builds the upper-cased `COUNTRY:REGION` key for [`PermissionMaps::by_region`]. +fn region_key(country: &str, region: &str) -> String { + format!( + "{}:{}", + country.to_ascii_uppercase(), + region.to_ascii_uppercase() + ) +} + +impl PermissionMaps { + /// Builds an empty map set with no country or region entries. + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + /// The signal-to-permission policy parsed from the `signals` section of + /// `permissions.yaml`. The consent mapping reads this rather than encoding + /// any signal policy in the code. + #[must_use] + // Public because the permission signal module crates live outside core + // and read the deployment's policy, in their tests and where a module + // needs the shipped decisions rather than a policy built by hand. + pub fn signals(&self) -> &SignalPolicy { + &self.signals + } + + /// Registers explicit rules for an ISO 3166-1 alpha-2 country code. + #[must_use] + pub fn with_country(mut self, iso_code: &str, rules: CountryRules) -> Self { + self.by_country.insert(iso_code.to_ascii_uppercase(), rules); + self + } + + /// Registers explicit rules for a region within a country, keyed by the ISO + /// 3166-1 alpha-2 country and the geo module's region code (for example + /// `US` and `CA`). + /// + /// A region entry takes precedence over the country entry, so a deployer can + /// vary a single state or province on top of the country baseline. + #[must_use] + pub fn with_region(mut self, iso_country: &str, region: &str, rules: CountryRules) -> Self { + self.by_region + .insert(region_key(iso_country, region), rules); + self + } + + /// Registers the top node's rules, which apply to a request whose country + /// and region are both unlisted, and to a request with no place at all. + /// + /// A map built without one resolves such a request at the requires-signal + /// floor, which is what [`PermissionMaps::empty`] does. Every map parsed + /// from a `permissions.yaml` has one, because the top node's `group` is + /// required. + #[must_use] + pub fn with_default_rules(mut self, rules: CountryRules) -> Self { + self.default_rules = Some(rules); + self + } + + /// Registers the top node's jurisdiction, the consent handling for a + /// visitor whose place could not be resolved. + #[must_use] + pub fn with_default_jurisdiction(mut self, jurisdiction: Jurisdiction) -> Self { + self.default_jurisdiction = Some(jurisdiction); + self + } + + /// The jurisdiction the policy declares for a visitor whose place the geo + /// module could not resolve, taken from the top of the `rules:` tree. + /// + /// The consent gates resolve a jurisdiction from the request's place, so + /// with no place they would resolve [`Jurisdiction::Unknown`] and fail + /// closed even where the policy has declared what to do. This is that + /// declaration. A map with no top node (see [`PermissionMaps::empty`]) + /// reports [`Jurisdiction::Unknown`]. + #[must_use] + pub fn default_jurisdiction(&self) -> Jurisdiction { + self.default_jurisdiction.clone().unwrap_or_default() + } + + /// The jurisdiction that applies to `country` and `region`. + /// + /// Walks the tree the same way [`rules_or_default`](Self::rules_or_default) + /// does: the region when it is listed, otherwise the country, otherwise the + /// top node. Inheritance is settled when the file is parsed, so every + /// listed place already carries the jurisdiction it inherits. + #[must_use] + pub fn jurisdiction_for(&self, country: Option<&str>, region: Option<&str>) -> Jurisdiction { + if let (Some(country), Some(region)) = (country, region) + && let Some(jurisdiction) = self + .jurisdiction_by_region + .get(®ion_key(country, region)) + { + return jurisdiction.clone(); + } + if let Some(jurisdiction) = country + .map(str::to_ascii_uppercase) + .and_then(|code| self.jurisdiction_by_country.get(&code)) + { + return jurisdiction.clone(); + } + self.default_jurisdiction() + } + + /// The built-in default rules, parsed from the embedded `permissions.yaml` + /// (see `DEFAULT_PERMISSION_RULES`). + /// + /// The parse runs once per instance and the result is cached. + /// + /// # Panics + /// + /// Panics if the embedded `permissions.yaml` fails to parse. The file is a + /// build-time constant covered by tests, so a panic means the committed file + /// is malformed, not a runtime condition. + #[must_use] + pub fn standard() -> &'static Self { + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| { + Self::from_yaml(DEFAULT_PERMISSION_RULES) + .expect("should parse the embedded default permissions.yaml") + }) + } + + /// Builds the maps from a `permissions.yaml` document: named `groups`, the + /// `rules` tree of places, and the `signals` section that maps each session + /// signal onto Data Uses. + /// + /// # Errors + /// + /// Returns [`PermissionsError`] when the YAML is malformed, the top of the + /// `rules` tree omits `group` or `jurisdiction`, a node below the top is + /// neither a group name nor a block carrying `group`, a node names an + /// unknown group, permission, or acquisition flag, two sibling place codes + /// name the same place, or a signal's revoke list names an unknown Data + /// Use. + pub fn from_yaml(yaml: &str) -> Result { + let file: RulesFile = + serde_yaml_ng::from_str(yaml).map_err(|error| PermissionsError::Parse { + message: error.to_string(), + })?; + + // Build every named group into its CountryRules. + let mut groups: BTreeMap = BTreeMap::new(); + for (name, flags) in &file.groups { + groups.insert(name.clone(), group_rules(name, flags)?); + } + + let mut maps = Self::empty(); + build_rules_tree(&mut maps, &groups, &file.rules)?; + maps.signals = build_signal_policy(&file.signals)?; + Ok(maps) + } + + /// Returns the rules written for `country` and `region` exactly, preferring + /// a region entry, then the country entry, or `None` when neither is listed. + /// + /// This is the literal tree lookup with no fallback to the top node. Use + /// [`rules_or_default`](Self::rules_or_default) for the resolution a + /// request actually gets. + #[must_use] + pub fn rules_for(&self, country: Option<&str>, region: Option<&str>) -> Option<&CountryRules> { + if let (Some(country), Some(region)) = (country, region) + && let Some(rules) = self.by_region.get(®ion_key(country, region)) + { + return Some(rules); + } + country + .map(str::to_ascii_uppercase) + .and_then(|code| self.by_country.get(&code)) + } + + /// The rules a request resolves to: its region, else its country, else the + /// top node of the tree. + /// + /// Returns `None` only for a map with no top node, which the caller treats + /// as the requires-signal floor. Every map parsed from a `permissions.yaml` + /// has one, because the top node's `group` is required, so this is + /// unreachable in a deployment and exists for maps built by hand in tests. + pub(crate) fn rules_or_default( + &self, + country: Option<&str>, + region: Option<&str>, + ) -> Option<&CountryRules> { + self.rules_for(country, region) + .or(self.default_rules.as_ref()) + } + + /// Resolves the permission state for a request: the place baseline + /// augmented by a session signal. + /// + /// `country` and `region` are what a geo module returns (`region` may be + /// `None`). Whatever the tree does not answer falls back to the node above, + /// ending at the top node, so an unlisted country and a request with no + /// country at all both resolve to the top node's group. `signal` maps each + /// permission to a [`ConsentSignal`]; the caller derives it from its consent + /// model so this module stays independent of how a signal is decoded. A + /// `Granted` baseline is set unless the signal is `Revoke`, a + /// `RequiresSignal` baseline is set only on `Grant`, and `Denied` is never + /// set. + #[must_use] + pub fn resolve_with( + &self, + country: Option<&str>, + region: Option<&str>, + signal: impl Fn(Permission, Acquisition) -> ConsentSignal, + ) -> PermissionState { + Self::resolve_rules(self.rules_or_default(country, region), signal) + } + + /// Resolves every permission at the requires-signal floor, whatever the + /// policy tree says. + /// + /// This is the state for a geo module that reported an outright lookup + /// failure. The request's place is unknown in a way the policy's top node + /// must not paper over, so nothing is set unless the session's signals + /// grant it. + #[must_use] + pub fn floor_with( + signal: impl Fn(Permission, Acquisition) -> ConsentSignal, + ) -> PermissionState { + Self::resolve_rules(None, signal) + } + + /// Applies `signal` against `rules`, or against the requires-signal floor + /// when no rules resolved. + fn resolve_rules( + rules: Option<&CountryRules>, + signal: impl Fn(Permission, Acquisition) -> ConsentSignal, + ) -> PermissionState { + let acquisition = + |permission| rules.map_or(Acquisition::RequiresSignal, |r| r.rule_for(permission)); + let mut set = PermissionSet::none(); + let mut awaiting = PermissionSet::none(); + for permission in Permission::all() { + // The baseline is passed to the signal as well as applied to its + // answer, because a source amends the place rules and cannot + // amend what it cannot see. + let baseline = acquisition(permission); + match (baseline, signal(permission, baseline)) { + (Acquisition::Denied, _) => {} + (Acquisition::Granted, ConsentSignal::Revoke) => {} + (Acquisition::Granted, _) => set = set.with(permission), + (Acquisition::RequiresSignal, ConsentSignal::Grant) => set = set.with(permission), + (Acquisition::RequiresSignal, ConsentSignal::Revoke) => {} + // Requires a signal and nobody gave one. Not set, and not + // refused either, which the page has to be able to tell apart. + (Acquisition::RequiresSignal, ConsentSignal::Neutral) => { + awaiting = awaiting.with(permission); + } + } + } + PermissionState::new(set).with_awaiting(awaiting) + } + + /// The baseline permission state for a country and region with no session + /// signal. + /// + /// Permissions exist without a consent model, so this is the set of + /// `Granted` permissions for the place (or the top node it falls back to), + /// and is what a request resolves to when no signal is present. + #[must_use] + pub fn baseline(&self, country: Option<&str>, region: Option<&str>) -> PermissionState { + self.resolve_with(country, region, |_, _| ConsentSignal::Neutral) + } + + /// Convenience over [`resolve_with`](Self::resolve_with) for a boolean + /// signal with no region and no revocation: a `true` grants the permission + /// and a `false` is neutral. + #[must_use] + pub fn resolve( + &self, + country: Option<&str>, + signal: impl Fn(Permission) -> bool, + ) -> PermissionState { + self.resolve_with(country, None, |permission, _| { + if signal(permission) { + ConsentSignal::Grant + } else { + ConsentSignal::Neutral + } + }) + } +} + +/// A signal a module read from the request and found valid, as it was +/// received. +/// +/// The permission state carries one of these for every signal a configured +/// module used, so whatever reads the state can rely on exactly those +/// signals and no other. A signal that was absent, could not be read, or +/// that no configured module acts on is not here, and nothing says why, +/// because the module for that scheme has already decided what its +/// absence means for the permissions. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct ValidSignal { + /// The module that read and used the signal, by its configured id. + pub module: &'static str, + /// The scheme the signal belongs to, for example `tcf` or `gpp`, so a + /// reader can find a scheme without knowing which module read it. + pub scheme: &'static str, + /// The signal as it was received. + pub value: String, +} + +impl ValidSignal { + /// A valid signal of `scheme`, read by `module`, as received. + #[must_use] + pub fn new(module: &'static str, scheme: &'static str, value: impl Into) -> Self { + Self { + module, + scheme, + value: value.into(), + } + } +} + +/// The permissions Trusted Server currently has set for a request. +/// +/// A module executes only when [`all_set`](Self::all_set) of its required +/// permissions returns `true`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PermissionState { + set: PermissionSet, + /// The permissions whose baseline requires a signal and for which no + /// configured module gave one. See [`awaiting`](Self::awaiting). + awaiting: PermissionSet, + /// Whether the request explicitly withdrew device storage, as opposed to + /// storage merely not being set. See + /// [`storage_withdrawn`](Self::storage_withdrawn). + storage_withdrawn: bool, + /// The terms documents the data for this request is available under. See + /// [`tdls`](Self::tdls). Shared rather than owned because the state is + /// cloned along the request path and the list is the same list. + tdls: Arc<[Tdl]>, + /// The signals the configured modules read and found valid. See + /// [`signals`](Self::signals). + signals: Arc<[ValidSignal]>, +} + +impl PermissionState { + /// Builds a state in which exactly the permissions in `set` are set, and + /// nothing is withdrawn, for tests and callers that compute the set + /// directly. + #[must_use] + pub fn new(set: PermissionSet) -> Self { + Self { + set, + awaiting: PermissionSet::none(), + storage_withdrawn: false, + tdls: Arc::default(), + signals: Arc::default(), + } + } + + /// The same state, recording which permissions are still waiting for a + /// signal. Set by resolution from the baseline and the modules' answers. + #[must_use] + pub fn with_awaiting(self, awaiting: PermissionSet) -> Self { + Self { awaiting, ..self } + } + + /// The same state, awaiting only the permissions in `answerable`. + /// + /// Resolution records every permission that requires a signal and got + /// none. Some of those no configured module could ever grant, for + /// example a marketing channel opt-in in a deployment whose modules can + /// grant none, and a page told to wait for them would wait for ever. So + /// the assembly narrows the list to what some module declares it can + /// grant, and the rest are simply not set. + #[must_use] + pub fn awaiting_only(self, answerable: PermissionSet) -> Self { + Self { + awaiting: self.awaiting.intersect(answerable), + ..self + } + } + + /// The permissions whose baseline requires a signal and for which every + /// configured module answered neutral, so nobody has answered yet. + /// + /// This is different from a permission that is not set. A refused + /// permission and one nobody has answered look the same in + /// [`permissions`](Self::permissions), and a page that cannot tell them + /// apart treats a prompt that never ran as a visitor who said no. A + /// permission is never both set and awaited. + #[must_use] + pub const fn awaiting(&self) -> PermissionSet { + self.awaiting + } + + /// Whether a permission is still waiting for a signal. + #[must_use] + pub const fn is_awaited(&self, permission: Permission) -> bool { + self.awaiting.contains(permission) + } + + /// The same state, recording whether device storage was explicitly + /// withdrawn. Set by assembly from what the signal modules answered, + /// scoped to the jurisdiction's storage baseline. + #[must_use] + pub fn with_storage_withdrawn(self, storage_withdrawn: bool) -> Self { + Self { + storage_withdrawn, + ..self + } + } + + /// The same state, carrying the terms documents the data for this request + /// is available under. Set by assembly from what the signal modules + /// declared, in the order they are asked. + #[must_use] + pub fn with_tdls(self, tdls: Arc<[Tdl]>) -> Self { + Self { tdls, ..self } + } + + /// The same state, carrying the signals the modules read and found + /// valid. Set by assembly, in the order the modules are asked. + #[must_use] + pub fn with_signals(self, signals: Arc<[ValidSignal]>) -> Self { + Self { signals, ..self } + } + + /// The signals the configured modules read from the request and + /// found valid, each as it was received. + /// + /// This is the list a page, a bid request or a person reading the state + /// relies on. A signal that is not here was absent, could not be read, or + /// is one no configured module acts on, and the module for its + /// scheme has already decided what that means for the permissions, so + /// nothing downstream needs to reason about the signal itself. + #[must_use] + pub fn signals(&self) -> &[ValidSignal] { + &self.signals + } + + /// The terms documents the data for this request is available under, in + /// the order the modules were asked. + /// + /// Whoever receives the data reads these to decide whether the terms are + /// ones they accept, and whether they may pass the data on. An empty list + /// says no terms were declared, which is not the same as terms that permit + /// anything, so a recipient that needs a basis and finds none has none. + #[must_use] + pub fn tdls(&self) -> &[Tdl] { + &self.tdls + } + + /// Whether the request carries an explicit signal withdrawing device + /// storage, rather than merely lacking the permission. + /// + /// The difference is destructive. A withdrawal expires the browser cookie + /// and writes the authoritative identity-graph tombstone, where a + /// permission that is simply not set strips the Edge Cookie response + /// headers and leaves an already-issued identifier alone, so a returning + /// visitor is not permanently withdrawn before they ever get to answer. + /// Which scheme can withdraw is each module's to say, and only where the + /// jurisdiction's storage baseline did not grant storage outright. + #[must_use] + pub const fn storage_withdrawn(&self) -> bool { + self.storage_withdrawn + } + + /// Whether a single permission is set. + #[must_use] + pub const fn is_set(&self, permission: Permission) -> bool { + self.set.contains(permission) + } + + /// Whether every permission in `required` is set. An empty requirement is + /// always satisfied, so a module that requires nothing always runs. + #[must_use] + pub const fn all_set(&self, required: PermissionSet) -> bool { + self.set.contains_all(required) + } + + /// The full set of permissions that are set, for a module that adapts its + /// behavior to whatever is present. + #[must_use] + pub const fn permissions(&self) -> PermissionSet { + self.set + } + + /// The resolved state as the JSON the page receives in + /// `window.tsjs.permissions`. + /// + /// Names are the [`Permission::as_str`] Data Use identifiers, sorted so the + /// same state always serializes to the same bytes whatever order the set + /// was built in. `awaiting` names the permissions still waiting for a + /// signal, so a page can hold what depends on them rather than read + /// "nobody has answered" as "refused". `tdls` carries the terms documents + /// the data is available under, in the order the modules were asked, so + /// a page module reads the terms alongside the permissions. `signals` + /// carries the signals the modules read and found valid, each as it was + /// received, so a page relies on exactly what the state was built from. + /// An empty state renders as + /// `{"awaiting":[],"set":[],"signals":[],"tdls":[]}`, and all four are + /// answers (nothing is set, nothing is awaited, no signal was valid, no + /// terms were declared) rather than missing values, so page code never + /// has to tell them apart. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::permissions::{ + /// Permission, PermissionSet, PermissionState, + /// }; + /// + /// let state = PermissionState::new( + /// PermissionSet::none().with(Permission::StoreOnDevice), + /// ); + /// assert_eq!( + /// state.page_json(), + /// r#"{"awaiting":[],"set":["necessary.operations.storage"],"signals":[],"tdls":[]}"# + /// ); + /// + /// assert_eq!( + /// PermissionState::default().page_json(), + /// r#"{"awaiting":[],"set":[],"signals":[],"tdls":[]}"# + /// ); + /// ``` + #[must_use] + pub fn page_json(&self) -> String { + self.page_value().to_string() + } + + /// The resolved state in the page shape, as a value, for whatever shows + /// the same resolution somewhere other than a page. + /// + /// See [`page_json`](Self::page_json) for what each key means. This is + /// the only place the page shape is spelled, so no caller writes the JSON + /// by hand. + #[must_use] + pub fn page_value(&self) -> serde_json::Value { + let mut names: Vec<&'static str> = self.set.iter().map(Permission::as_str).collect(); + names.sort_unstable(); + let mut awaiting: Vec<&'static str> = + self.awaiting.iter().map(Permission::as_str).collect(); + awaiting.sort_unstable(); + let tdls: Vec<&str> = self.tdls.iter().map(Tdl::as_str).collect(); + serde_json::json!({ + "set": names, + "awaiting": awaiting, + "signals": &*self.signals, + "tdls": tdls, + }) + } +} + +// --------------------------------------------------------------------------- +// permissions.yaml parsing +// --------------------------------------------------------------------------- + +/// The shape of a `permissions.yaml` document. +#[derive(Debug, Deserialize)] +struct RulesFile { + /// Named permission baselines, keyed by group name. Each group is a flat map + /// of per-permission flags, with an optional `default` shorthand for any + /// permission it omits. + #[serde(default)] + groups: BTreeMap>, + /// The `rules` tree of places. Walked by [`build_rules_tree`] rather than + /// deserialized into a fixed shape, because every key below the reserved + /// words is a place code chosen by the policy owner. + #[serde(default)] + rules: Value, + /// How each session signal maps onto Data Uses. + #[serde(default)] + signals: SignalsSpec, +} + +/// The `signals` section: how each signal source maps onto Data Uses. Parsed +/// into a [`SignalPolicy`] by [`build_signal_policy`]. +#[derive(Debug, Default, Deserialize)] +struct SignalsSpec { + /// The TCF record mapping, or `None` when the file declares no TCF policy. + #[serde(default)] + tcf: Option, + /// The US-style opt-out mapping, or `None` when none is declared. + #[serde(default)] + us_opt_out: Option, +} + +/// The `signals.tcf` block. +/// +/// Only whether a TCF record answers for this deployment. Which purpose grants +/// which Data Use is the TCF scheme's own knowledge and lives in the TCF +/// permission signal module crate, so this file carries no table of another +/// scheme's numbers and a deployment running no TCF configures none. +#[derive(Debug, Deserialize)] +struct TcfSignalSpec { + /// Whether a present TCF record's grants and revokes apply. + #[serde(default = "default_true")] + authoritative: bool, +} + +/// The `signals.us_opt_out` block. +#[derive(Debug, Deserialize)] +struct OptOutSpec { + /// The signals that constitute a US-style opt-out. + #[serde(default)] + sources: Vec, + /// Which Data Uses the opt-out revokes. + #[serde(default)] + revokes: RevokeSpec, +} + +/// A `revokes` value: the keyword `all`, or an explicit list of Data Uses. +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum RevokeSpec { + /// A bare keyword, expected to be `all`. + Keyword(String), + /// An explicit list of Data Use identifiers. + List(Vec), +} + +impl Default for RevokeSpec { + fn default() -> Self { + RevokeSpec::List(Vec::new()) + } +} + +/// A single US-style opt-out signal source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OptOutSource { + /// The `Sec-GPC` request header (Global Privacy Control). + Gpc, + /// A GPP US sale opt-out. + GppSaleOptOut, + /// A US Privacy string sale opt-out. + UsPrivacyOptOut, +} + +/// Default for `#[serde(default = ...)]` on a `bool` field that should be `true`. +fn default_true() -> bool { + true +} + +/// The reserved key naming a node's permission group. +const KEY_GROUP: &str = "group"; + +/// The reserved key naming the top node's jurisdiction. +const KEY_JURISDICTION: &str = "jurisdiction"; + +/// The reserved key holding a node's per-permission overrides. +const KEY_PERMISSIONS: &str = "permissions"; + +/// Resolves an acquisition rule name to its [`Acquisition`]. +fn parse_acquisition(value: &str) -> Result { + match value { + "granted" => Ok(Acquisition::Granted), + "requires_signal" => Ok(Acquisition::RequiresSignal), + "denied" => Ok(Acquisition::Denied), + other => Err(PermissionsError::UnknownAcquisition { + value: other.to_owned(), + }), + } +} + +/// Builds a group's [`CountryRules`] from its flag map. Each key names a +/// permission and its flag; an optional `default` key sets any permission the +/// group omits. A group without a `default` must list every permission, so its +/// meaning is fully explicit (this is how the shipped groups are written). +fn group_rules( + name: &str, + flags: &BTreeMap, +) -> Result { + let default = flags + .get("default") + .map(|value| parse_acquisition(value)) + .transpose()?; + // With no `default`, every permission must be listed, so this placeholder is + // never consulted once completeness is checked below. + let mut rules = CountryRules::with_default(default.unwrap_or(Acquisition::Denied)); + let mut listed = PermissionSet::none(); + for (key, value) in flags { + if key == "default" { + continue; + } + let permission = Permission::from_identifier(key) + .ok_or_else(|| PermissionsError::UnknownPermission { name: key.clone() })?; + rules = rules.with_rule(permission, parse_acquisition(value)?); + listed = listed.with(permission); + } + if default.is_none() { + for permission in Permission::all() { + if !listed.contains(permission) { + return Err(PermissionsError::IncompleteGroup { + group: name.to_owned(), + permission: permission.to_string(), + }); + } + } + } + Ok(rules) +} + +/// Looks up a group by name, erroring when a rule references a group that is not +/// defined. +fn resolve_group( + groups: &BTreeMap, + name: &str, +) -> Result { + groups + .get(name) + .cloned() + .ok_or_else(|| PermissionsError::UnknownGroup { + name: name.to_owned(), + }) +} + +/// Applies a rule's per-permission acquisition overrides on top of its group's +/// rules. Each entry maps a Data Use to `granted`, `requires_signal`, or +/// `denied`, overriding the group's baseline for that Data Use, so any +/// acquisition (not just grant or deny) is expressible per rule. +fn apply_modifications( + mut rules: CountryRules, + modifications: &BTreeMap, +) -> Result { + for (name, value) in modifications { + let permission = Permission::from_identifier(name).ok_or_else(|| { + PermissionsError::UnknownPermission { + name: name.to_owned(), + } + })?; + rules = rules.with_rule(permission, parse_acquisition(value)?); + } + Ok(rules) +} + +/// One node of the `rules` tree, resolved into its rules, the jurisdiction it +/// names (when it names one), and the child place codes written beside them. +struct RuleNode<'a> { + /// The node's group, with any `permissions` overrides applied. + rules: CountryRules, + /// The `jurisdiction:` value written on this node, or `None` when the node + /// inherits the one above it. + jurisdiction: Option<&'a str>, + /// The child place codes, in document order, each still unparsed. + children: Vec<(&'a str, &'a Value)>, +} + +/// Parses one node of the `rules` tree. +/// +/// A node written as a plain string is the shorthand: the string is its group, +/// it names no jurisdiction, and it has no children. A node written as a block +/// must carry a `group:` line, may carry a `jurisdiction:` line and a +/// `permissions:` map of overrides, and may carry child place codes beside +/// them. `path` names the node for error messages (`"US"`, `"US/CA"`, or the +/// top of the tree). +fn parse_rule_node<'a>( + groups: &BTreeMap, + path: &str, + value: &'a Value, +) -> Result, PermissionsError> { + match value { + Value::String(name) => Ok(RuleNode { + rules: resolve_group(groups, name)?, + jurisdiction: None, + children: Vec::new(), + }), + Value::Mapping(map) => { + let group = map.get(KEY_GROUP).and_then(Value::as_str).ok_or_else(|| { + PermissionsError::MissingGroup { + path: path.to_owned(), + } + })?; + let mut rules = resolve_group(groups, group)?; + if let Some(overrides) = map.get(KEY_PERMISSIONS) { + let overrides: BTreeMap = + serde_yaml_ng::from_value(overrides.clone()).map_err(|error| { + PermissionsError::Parse { + message: format!("`{KEY_PERMISSIONS}` under `{path}`: {error}"), + } + })?; + rules = apply_modifications(rules, &overrides)?; + } + let jurisdiction = + match map.get(KEY_JURISDICTION) { + Some(value) => Some(value.as_str().ok_or_else(|| { + PermissionsError::UnknownJurisdiction { + value: format!("{value:?}"), + } + })?), + None => None, + }; + let mut children = Vec::new(); + for (key, child) in map { + let key = key.as_str().ok_or_else(|| PermissionsError::InvalidRule { + path: path.to_owned(), + })?; + if key == KEY_GROUP || key == KEY_PERMISSIONS || key == KEY_JURISDICTION { + continue; + } + children.push((key, child)); + } + Ok(RuleNode { + rules, + jurisdiction, + children, + }) + } + _ => Err(PermissionsError::InvalidRule { + path: path.to_owned(), + }), + } +} + +/// Resolves a `jurisdiction:` value written on the node at `path`. +/// +/// `region` is the node's own ISO 3166-2 code, or `None` for the top of the +/// tree and for a country. `us-state` carries no code of its own, because the +/// node naming it is the state, so it is rejected wherever there is no region +/// to name. +fn parse_jurisdiction( + path: &str, + region: Option<&str>, + value: &str, +) -> Result { + if value == "us-state" && region.is_none() { + return Err(PermissionsError::MisplacedUsState { + path: path.to_owned(), + }); + } + Jurisdiction::from_policy_name(value, region).ok_or_else(|| { + PermissionsError::UnknownJurisdiction { + value: value.to_owned(), + } + }) +} + +/// Walks the `rules` tree into `maps`: the top node, then the country nodes +/// under it, then the region nodes under each country. +/// +/// Each node's group and jurisdiction are settled here, so a node that names +/// neither is stored carrying what it inherits from the node above it. The top +/// node must name both, which is what makes inheritance always terminate. +/// +/// The tree is three levels deep, because a geo module returns a country and +/// a region and nothing finer, so a place written under a region is rejected +/// rather than silently ignored. +fn build_rules_tree( + maps: &mut PermissionMaps, + groups: &BTreeMap, + rules: &Value, +) -> Result<(), PermissionsError> { + const TOP: &str = "the top of the tree"; + + if !matches!(rules, Value::Mapping(_)) { + return Err(PermissionsError::MissingGroup { + path: TOP.to_owned(), + }); + } + let top = parse_rule_node(groups, TOP, rules)?; + let name = top + .jurisdiction + .ok_or(PermissionsError::MissingJurisdiction)?; + let top_jurisdiction = parse_jurisdiction(TOP, None, name)?; + maps.default_rules = Some(top.rules); + maps.default_jurisdiction = Some(top_jurisdiction.clone()); + + // Place codes are matched without regard to case at lookup, so two + // spellings of one place would silently overwrite each other. Reject the + // collision instead, among the siblings at each level. + let mut seen_countries: BTreeMap = BTreeMap::new(); + for (country, node) in top.children { + check_duplicate(&mut seen_countries, country)?; + let node = parse_rule_node(groups, country, node)?; + let jurisdiction = match node.jurisdiction { + Some(name) => parse_jurisdiction(country, None, name)?, + None => top_jurisdiction.clone(), + }; + let code = country.to_ascii_uppercase(); + maps.by_country.insert(code.clone(), node.rules); + maps.jurisdiction_by_country + .insert(code, jurisdiction.clone()); + + let mut seen_regions: BTreeMap = BTreeMap::new(); + for (region, child) in node.children { + let path = format!("{country}/{region}"); + check_duplicate(&mut seen_regions, region)?; + let child = parse_rule_node(groups, &path, child)?; + if !child.children.is_empty() { + return Err(PermissionsError::NestedTooDeep { path }); + } + let child_jurisdiction = match child.jurisdiction { + Some(name) => parse_jurisdiction(&path, Some(region), name)?, + None => jurisdiction.clone(), + }; + let key = region_key(country, region); + maps.by_region.insert(key.clone(), child.rules); + maps.jurisdiction_by_region.insert(key, child_jurisdiction); + } + } + Ok(()) +} + +/// Records a place code among its siblings, erroring when another spelling of +/// the same code is already there. +fn check_duplicate<'a>( + seen: &mut BTreeMap, + key: &'a str, +) -> Result<(), PermissionsError> { + if let Some(first) = seen.insert(key.to_ascii_uppercase(), key) { + return Err(PermissionsError::DuplicateRule { + first: first.to_owned(), + second: key.to_owned(), + }); + } + Ok(()) +} + +/// Parses the compiled `permissions.yaml`, so startup validation can reject a +/// malformed policy with a configuration error instead of panicking on the +/// first lookup. +/// +/// # Errors +/// +/// Returns the same [`PermissionsError`] values [`PermissionMaps::from_yaml`] +/// does, most usefully a top node that omits `group` or `jurisdiction`. +pub fn validate_default_policy() -> Result<(), PermissionsError> { + PermissionMaps::from_yaml(DEFAULT_PERMISSION_RULES).map(|_| ()) +} + +/// An error parsing a `permissions.yaml` document. +#[derive(Debug, derive_more::Display)] +pub enum PermissionsError { + /// Two sibling place codes are the same place spelled differently. + #[display("place codes `{first}` and `{second}` name the same place, so keep one")] + DuplicateRule { first: String, second: String }, + /// A node of the `rules` tree carried no `group`. + #[display( + "the rule for `{path}` needs a `group:` line (or write the group name on its own, as `GB: gdpr-uk`)" + )] + MissingGroup { path: String }, + /// The top of the `rules` tree carried no `jurisdiction`. + #[display( + "the top of the `rules` tree needs a `jurisdiction:` line, naming the consent handling for a visitor whose place cannot be resolved" + )] + MissingJurisdiction, + /// A node with no region of its own named `us-state`. + #[display( + "the rule for `{path}` names `jurisdiction: us-state`, which only a region may name, because the region is the state" + )] + MisplacedUsState { path: String }, + /// A `jurisdiction` value named something the consent model cannot + /// represent. + #[display( + "unknown jurisdiction `{value}` (expected gdpr, us-state, non-regulated, or unknown)" + )] + UnknownJurisdiction { value: String }, + /// A node of the `rules` tree was neither a group name nor a block. + #[display("the rule for `{path}` must be a group name or a block with a `group:` line")] + InvalidRule { path: String }, + /// A place was written under a region, deeper than a geo module resolves. + #[display( + "the rule for `{path}` has places written under it; the tree stops at a region, because that is the finest place a geo module returns" + )] + NestedTooDeep { path: String }, + /// The YAML was malformed or did not match the expected shape. + #[display("failed to parse permission rules: {message}")] + Parse { message: String }, + /// A group without a `default` did not list every permission. + #[display( + "permission group `{group}` has no `default` and is missing a flag for `{permission}` (list every permission, or add a `default`)" + )] + IncompleteGroup { group: String, permission: String }, + /// A rule referenced a group that is not defined. + #[display("unknown permission group `{name}`")] + UnknownGroup { name: String }, + /// A permission flag or modification named an unknown permission. + #[display("unknown permission `{name}`")] + UnknownPermission { name: String }, + /// An acquisition rule was not `granted`, `requires_signal`, or `denied`. + #[display("unknown acquisition rule `{value}` (expected granted, requires_signal, or denied)")] + UnknownAcquisition { value: String }, + /// A `signals` opt-out `revokes` value was neither `all` nor a list. + #[display("unknown revoke rule `{value}` (expected `all` or a list of Data Uses)")] + UnknownRevoke { value: String }, +} + +impl core::error::Error for PermissionsError {} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn permission_set_membership_and_subset() { + let set = PermissionSet::none() + .with(Permission::StoreOnDevice) + .with(Permission::SelectBasicAds); + + assert!(set.contains(Permission::StoreOnDevice)); + assert!(set.contains(Permission::SelectBasicAds)); + assert!( + !set.contains(Permission::SelectPersonalisedAds), + "an absent permission should not be reported as present" + ); + + let required = PermissionSet::none().with(Permission::StoreOnDevice); + assert!(set.contains_all(required), "a subset should be contained"); + assert!( + !required.contains_all(set), + "a superset is not contained in a subset" + ); + assert!( + set.contains_all(PermissionSet::none()), + "the empty requirement is always satisfied" + ); + } + + #[test] + fn permission_set_iterates_in_bit_index_order() { + let set = PermissionSet::none() + .with(Permission::SelectBasicAds) + .with(Permission::StoreOnDevice); + let order: Vec<&str> = set.iter().map(Permission::as_str).collect(); + assert_eq!( + order, + vec![ + "necessary.operations.storage", + "advertising_marketing.first_party.contextual" + ], + "iteration should be in stable bit-index order" + ); + } + + #[test] + fn page_json_lists_set_permissions_sorted_by_name() { + // Arrange: a state whose bit-index order is the reverse of its name + // order, so the sort is what the assertion sees. + let state = PermissionState::new( + PermissionSet::none() + .with(Permission::StoreOnDevice) + .with(Permission::SelectBasicAds), + ); + + // Act + let json = state.page_json(); + + // Assert + assert_eq!( + json, + json!({ + "set": [ + "advertising_marketing.first_party.contextual", + "necessary.operations.storage", + ], + "awaiting": [], + "signals": [], + "tdls": [], + }) + .to_string(), + "should list every set permission by Data Use name, sorted" + ); + } + + #[test] + fn page_json_carries_the_terms_the_data_is_available_under() { + // Arrange: the state a request resolves to when a terms scheme + // declared the document its data is offered under. + let state = PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)) + .with_tdls(Arc::from(vec![ + Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept the test locator"), + ])); + + // Act + let json = state.page_json(); + + // Assert + assert_eq!( + json, + json!({ + "set": ["necessary.operations.storage"], + "awaiting": [], + "signals": [], + "tdls": ["https://terms.example.com/marketing/2.txt"], + }) + .to_string(), + "page code should read the terms alongside the permissions" + ); + } + + #[test] + fn page_json_of_an_empty_state_is_an_empty_set() { + // Arrange + let state = PermissionState::default(); + + // Act + let json = state.page_json(); + + // Assert + assert_eq!( + json, + json!({ "set": [], "awaiting": [], "signals": [], "tdls": [] }).to_string(), + "an empty state should render as an empty set and no declared terms, not as nothing" + ); + } + + #[test] + fn a_permission_requiring_a_signal_that_nobody_gave_is_awaited() { + // Arrange: the requires-signal floor, where every permission needs a + // signal, and modules that answer nothing. + let state = PermissionMaps::floor_with(|_, _| ConsentSignal::Neutral); + + // Assert: nothing is set and everything is awaited, which is the + // state a page must hold on rather than read as a refusal. + assert!( + state.permissions().is_empty(), + "should set nothing without a signal" + ); + assert!( + state.is_awaited(Permission::StoreOnDevice), + "should record that storage is still waiting for a signal" + ); + assert_eq!( + state.page_json(), + json!({ + "set": [], + "awaiting": Permission::all().map(Permission::as_str).collect::>(), + "signals": [], + "tdls": [], + }) + .to_string(), + "the page should be told which permissions are still awaited" + ); + } + + #[test] + fn awaiting_is_narrowed_to_what_a_module_could_still_grant() { + // Arrange: the floor, where everything is awaited, and a deployment + // whose modules can only ever grant storage. + let state = PermissionMaps::floor_with(|_, _| ConsentSignal::Neutral) + .awaiting_only(PermissionSet::none().with(Permission::StoreOnDevice)); + + // Assert: storage is still awaited, and a permission nobody could + // grant is not, because no page should wait for it. + assert!(state.is_awaited(Permission::StoreOnDevice)); + assert!( + !state.is_awaited(Permission::SelectPersonalisedAds), + "should not await a permission no configured module can grant" + ); + assert!( + state.permissions().is_empty(), + "narrowing what is awaited should set nothing" + ); + } + + #[test] + fn an_answered_or_baselined_permission_is_never_awaited() { + // Arrange: the floor again, with a module that grants storage and + // revokes personalised advertising. + let state = PermissionMaps::floor_with(|permission, _| match permission { + Permission::StoreOnDevice => ConsentSignal::Grant, + Permission::SelectPersonalisedAds => ConsentSignal::Revoke, + _ => ConsentSignal::Neutral, + }); + + // Assert: a grant is set and not awaited, a refusal is neither set + // nor awaited, and a permission nobody answered is awaited. + assert!(state.is_set(Permission::StoreOnDevice)); + assert!( + !state.is_awaited(Permission::StoreOnDevice), + "a grant is an answer" + ); + assert!(!state.is_set(Permission::SelectPersonalisedAds)); + assert!( + !state.is_awaited(Permission::SelectPersonalisedAds), + "a refusal is an answer too, and must not look like no answer" + ); + assert!(state.is_awaited(Permission::SelectBasicAds)); + } + + #[test] + fn the_floor_sets_a_permission_only_when_a_signal_grants_it() { + // Empty maps and no top node: every permission is the requires-signal + // floor, set only when a signal grants it. + let maps = PermissionMaps::default(); + + let denied = maps.resolve(Some("GB"), |_| false); + assert!( + !denied.is_set(Permission::StoreOnDevice), + "the floor should not set necessary.operations.storage without a signal" + ); + + let granted = maps.resolve(Some("GB"), |p| p == Permission::StoreOnDevice); + assert!( + granted.is_set(Permission::StoreOnDevice), + "the floor should set necessary.operations.storage once a signal grants it" + ); + } + + #[test] + fn the_floor_ignores_the_top_node_even_where_one_grants() { + // A policy whose top node grants everything, so the only way the floor + // can leave the permission unset is by never consulting the tree. This + // is the geo lookup failure case. + let maps = PermissionMaps::empty() + .with_default_rules(CountryRules::with_default(Acquisition::Granted)); + assert!( + maps.baseline(None, None).is_set(Permission::StoreOnDevice), + "the top node grants storage, or this test proves nothing" + ); + assert!( + !PermissionMaps::floor_with(|_, _| ConsentSignal::Neutral) + .is_set(Permission::StoreOnDevice), + "the floor must not fall back to the top node" + ); + } + + #[test] + fn an_unlisted_country_uses_the_top_node() { + // A map whose top node grants, so an unlisted country is answered by + // it rather than by the floor. + let maps = PermissionMaps::empty() + .with_default_rules(CountryRules::with_default(Acquisition::Granted)); + assert!( + maps.resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "an unlisted country should resolve at the top node" + ); + // No country at all resolves the same way. + assert!( + maps.resolve(None, |_| false) + .is_set(Permission::StoreOnDevice), + "no country should resolve at the top node too" + ); + // With no top node the requires-signal floor sets nothing. + assert!( + !PermissionMaps::empty() + .resolve(None, |_| false) + .is_set(Permission::StoreOnDevice), + "with no top node, an unlisted country sets nothing without a signal" + ); + } + + #[test] + fn a_matching_country_is_used_over_the_top_node() { + // US grants; the top node requires a signal. + let maps = PermissionMaps::empty() + .with_country("us", CountryRules::with_default(Acquisition::Granted)) + .with_default_rules(CountryRules::with_default(Acquisition::RequiresSignal)); + assert!( + maps.resolve(Some("US"), |_| false) + .is_set(Permission::StoreOnDevice), + "a country with its own rule uses it, not the top node" + ); + assert!( + !maps + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "an unlisted country falls back to the top node" + ); + } + + #[test] + fn per_permission_override_beats_the_country_default() { + // Granted by default, but deny necessary.operations.storage specifically. + let rules = CountryRules::with_default(Acquisition::Granted) + .with_rule(Permission::StoreOnDevice, Acquisition::Denied); + let maps = PermissionMaps::empty().with_country("zz", rules); + let state = maps.resolve(Some("ZZ"), |_| true); + + assert!( + !state.is_set(Permission::StoreOnDevice), + "an explicit Denied override should beat the granted default" + ); + assert!( + state.is_set(Permission::SelectBasicAds), + "other permissions should still follow the granted default" + ); + } + + #[test] + fn all_set_gates_on_the_required_set() { + let state = PermissionState::new(PermissionSet::none().with(Permission::StoreOnDevice)); + + assert!( + state.all_set(PermissionSet::none()), + "a module requiring nothing always runs" + ); + assert!( + state.all_set(PermissionSet::none().with(Permission::StoreOnDevice)), + "a set requirement is satisfied" + ); + assert!( + !state.all_set(PermissionSet::none().with(Permission::SelectPersonalisedAds)), + "an unset requirement is not satisfied" + ); + } + + #[test] + fn with_default_sets_the_baseline_acquisition() { + // A granted default is set with no signal required. + let granted = PermissionMaps::empty() + .with_country("zz", CountryRules::with_default(Acquisition::Granted)); + assert!( + granted + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "a granted default should set with no signal" + ); + // A requires-signal default is set only once a signal grants it. + let opt_in = PermissionMaps::empty().with_country( + "zz", + CountryRules::with_default(Acquisition::RequiresSignal), + ); + assert!( + !opt_in + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "a requires-signal default should not be set without a signal" + ); + assert!( + opt_in + .resolve(Some("ZZ"), |p| p == Permission::StoreOnDevice) + .is_set(Permission::StoreOnDevice), + "a requires-signal default should set once a signal grants it" + ); + } + + #[test] + fn granted_and_denied_rules_ignore_signals() { + // A Granted rule is set even when no signal is present. + let granted = CountryRules::with_default(Acquisition::RequiresSignal) + .with_rule(Permission::StoreOnDevice, Acquisition::Granted); + assert!( + PermissionMaps::empty() + .with_country("zz", granted) + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "a Granted rule is set with no signal" + ); + + // A Denied rule is never set, even when every signal grants. + let denied = CountryRules::with_default(Acquisition::Granted) + .with_rule(Permission::StoreOnDevice, Acquisition::Denied); + assert!( + !PermissionMaps::empty() + .with_country("zz", denied) + .resolve(Some("ZZ"), |_| true) + .is_set(Permission::StoreOnDevice), + "a Denied rule is never set even with a signal" + ); + } + + #[test] + fn standard_maps_eu_requires_signal_and_uk_grants_storage() { + let maps = PermissionMaps::standard(); + assert!( + !maps + .resolve(Some("DE"), |_| false) + .is_set(Permission::StoreOnDevice), + "an EU country should not set necessary.operations.storage without a signal" + ); + assert!( + maps.resolve(Some("DE"), |p| p == Permission::StoreOnDevice) + .is_set(Permission::StoreOnDevice), + "an EU country should set necessary.operations.storage once a signal grants it" + ); + assert!( + maps.resolve(Some("GB"), |_| false) + .is_set(Permission::StoreOnDevice), + "the UK should grant necessary.operations.storage without a signal" + ); + } + + #[test] + fn standard_maps_us_and_australia_grant_storage_by_default() { + let maps = PermissionMaps::standard(); + for code in ["US", "AU"] { + assert!( + maps.resolve(Some(code), |_| false) + .is_set(Permission::StoreOnDevice), + "{code} should grant necessary.operations.storage by default" + ); + } + // No default configured, so an unmapped country hits the requires-signal + // floor and sets nothing without a signal. + assert!( + !maps + .resolve(Some("ZZ"), |_| false) + .is_set(Permission::StoreOnDevice), + "an unmapped country with no default sets nothing without a signal" + ); + } + + #[test] + fn resolve_with_revokes_a_granted_permission_on_opt_out() { + // US grants necessary.operations.storage by default; an opt-out signal revokes it. + let maps = PermissionMaps::standard(); + assert!( + maps.baseline(Some("US"), None) + .is_set(Permission::StoreOnDevice), + "the US baseline should set necessary.operations.storage" + ); + let revoked = maps.resolve_with(Some("US"), None, |p, _| { + if p == Permission::StoreOnDevice { + ConsentSignal::Revoke + } else { + ConsentSignal::Neutral + } + }); + assert!( + !revoked.is_set(Permission::StoreOnDevice), + "an opt-out signal should revoke a granted permission" + ); + } + + #[test] + fn a_region_entry_overrides_the_country_baseline() { + // US grants by default; a state can require a signal instead. + let maps = PermissionMaps::standard().clone().with_region( + "US", + "CA", + CountryRules::with_default(Acquisition::RequiresSignal), + ); + assert!( + !maps + .baseline(Some("US"), Some("CA")) + .is_set(Permission::StoreOnDevice), + "the CA region rule should require a signal, overriding the US baseline" + ); + assert!( + maps.baseline(Some("US"), Some("NY")) + .is_set(Permission::StoreOnDevice), + "a state with no region entry should follow the US baseline" + ); + } + + #[test] + fn from_yaml_parses_groups_rules_and_modifications() { + let yaml = r#" +groups: + eu: + default: requires_signal + us: + default: granted +rules: + group: eu + jurisdiction: gdpr + FR: eu + US: + group: us + jurisdiction: non-regulated + CA: + group: eu + jurisdiction: us-state + permissions: + necessary.operations.storage: granted + advertising_marketing.first_party.contextual: denied +"#; + let maps = PermissionMaps::from_yaml(yaml).expect("should parse the rules"); + + // Bare group references. + assert!( + !maps + .baseline(Some("FR"), None) + .is_set(Permission::StoreOnDevice), + "FR (eu) requires a signal for device storage" + ); + assert!( + maps.baseline(Some("US"), None) + .is_set(Permission::StoreOnDevice), + "US (us) grants device storage" + ); + + // CA references the eu group, but its permissions map grants + // necessary.operations.storage, overriding the eu baseline. + assert!( + maps.baseline(Some("US"), Some("CA")) + .is_set(Permission::StoreOnDevice), + "the permissions map grants necessary.operations.storage for CA, overriding the eu baseline" + ); + // The permissions map denies advertising_marketing.first_party.contextual, + // so it is not set even when a signal grants it. + assert!( + !maps + .resolve_with(Some("US"), Some("CA"), |_, _| ConsentSignal::Grant) + .is_set(Permission::SelectBasicAds), + "the permissions map denies advertising_marketing.first_party.contextual even when a signal grants it" + ); + + // An unlisted country falls back to the top node (eu, requires signal). + assert!( + !maps + .baseline(Some("ZZ"), None) + .is_set(Permission::StoreOnDevice), + "an unlisted country should use the top node" + ); + } + + /// A two-group policy the tree tests below vary, with `g` granting + /// everything and `strict` requiring a signal for everything. + const TEST_GROUPS: &str = "groups:\n g:\n default: granted\n \ + strict:\n default: requires_signal\n"; + + #[test] + fn a_shorthand_node_is_a_group_with_nothing_below_it() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: strict + jurisdiction: gdpr + GB: g +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("a shorthand node should parse"); + assert_eq!( + maps.rules_for(Some("GB"), None) + .expect("GB should have a rule") + .rule_for(Permission::StoreOnDevice), + Acquisition::Granted, + "the shorthand string should name the node's group" + ); + assert_eq!( + maps.jurisdiction_for(Some("GB"), None), + Jurisdiction::Gdpr, + "a shorthand node should inherit the top node's jurisdiction" + ); + } + + #[test] + fn a_block_node_carries_children_beside_its_group() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: g + jurisdiction: gdpr + US: + group: g + jurisdiction: non-regulated + CA: + group: strict + jurisdiction: us-state +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("a block node should parse"); + assert_eq!( + maps.rules_for(Some("US"), Some("CA")) + .expect("US/CA should have a rule") + .rule_for(Permission::StoreOnDevice), + Acquisition::RequiresSignal, + "the region child should carry its own group" + ); + assert_eq!( + maps.rules_or_default(Some("US"), Some("NV")) + .expect("an unlisted region should fall back to its country") + .rule_for(Permission::StoreOnDevice), + Acquisition::Granted, + "an unlisted region should resolve at its country node" + ); + } + + /// Every way `PermissionMaps::from_yaml` refuses a policy, each with the + /// error it must report. + #[test] + fn from_yaml_rejects_each_malformed_policy() { + type Refusal = fn(&PermissionsError) -> bool; + // A policy given one YAML line to an element, with or without the test + // groups ahead of it. + let tree = |lines: &[&str]| format!("{TEST_GROUPS}{}\n", lines.join("\n")); + let policy = |lines: &[&str]| format!("{}\n", lines.join("\n")); + let cases: [(&str, String, Refusal); 15] = [ + ( + "a block without a group", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US:", + " jurisdiction: non-regulated", + ]), + |err| matches!(err, PermissionsError::MissingGroup { path } if path == "US"), + ), + ( + "a top node without a group", + tree(&["rules:", " jurisdiction: gdpr", " US: g"]), + |err| matches!(err, PermissionsError::MissingGroup { .. }), + ), + ( + "a top node without a jurisdiction", + tree(&["rules:", " group: g", " US: g"]), + |err| matches!(err, PermissionsError::MissingJurisdiction), + ), + ( + "an unknown jurisdiction", + tree(&["rules:", " group: g", " jurisdiction: ccpa"]), + |err| matches!(err, PermissionsError::UnknownJurisdiction { .. }), + ), + ( + // `us-state` names no code of its own, so only a region can + // carry it. + "us-state at the top of the tree", + tree(&["rules:", " group: g", " jurisdiction: us-state"]), + |err| matches!(err, PermissionsError::MisplacedUsState { .. }), + ), + ( + "us-state on a country", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US:", + " group: g", + " jurisdiction: us-state", + ]), + |err| matches!(err, PermissionsError::MisplacedUsState { .. }), + ), + ( + "a place written under a region", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US:", + " group: g", + " CA:", + " group: g", + " LA:", + " group: g", + ]), + |err| matches!(err, PermissionsError::NestedTooDeep { .. }), + ), + ( + "a rule naming a group that is not defined", + tree(&["rules:", " group: missing", " jurisdiction: gdpr"]), + |err| matches!(err, PermissionsError::UnknownGroup { name } if name == "missing"), + ), + ( + "a rule that is neither a group name nor a block", + tree(&[ + "rules:", + " group: g", + " jurisdiction: gdpr", + " US: [1, 2]", + ]), + |err| matches!(err, PermissionsError::InvalidRule { path } if path == "US"), + ), + ( + // A group with no `default` must list every permission, so this + // one, naming only storage, is rejected rather than silently + // leaving every other permission unset. + "an incomplete group without a default", + policy(&[ + "groups:", + " g:", + " necessary.operations.storage: granted", + "rules: {}", + ]), + |err| matches!(err, PermissionsError::IncompleteGroup { .. }), + ), + ( + "an unknown permission", + policy(&[ + "groups:", + " g:", + " default: granted", + " not-a-permission: denied", + "rules: {}", + ]), + |err| matches!(err, PermissionsError::UnknownPermission { .. }), + ), + ( + "an unknown acquisition", + policy(&["groups:", " g:", " default: maybe", "rules: {}"]), + |err| matches!(err, PermissionsError::UnknownAcquisition { .. }), + ), + ( + "an override that is not an acquisition", + policy(&[ + "groups:", + " g:", + " default: granted", + "rules:", + " group: g", + " jurisdiction: unknown", + " US:", + " group: g", + " permissions:", + " necessary.operations.storage: enabled", + ]), + |err| matches!(err, PermissionsError::UnknownAcquisition { .. }), + ), + ( + "two spellings of one country", + policy(&[ + "groups:", + " g:", + " default: granted", + "rules:", + " group: g", + " jurisdiction: unknown", + " us: g", + " US: g", + ]), + |err| matches!(err, PermissionsError::DuplicateRule { .. }), + ), + ( + "an unknown revoke keyword", + policy(&[ + "groups:", + " g:", + " default: requires_signal", + "rules:", + " group: g", + " jurisdiction: gdpr", + " FR: g", + "signals:", + " us_opt_out:", + " sources: [gpc]", + " revokes: everything", + ]), + |err| matches!(err, PermissionsError::UnknownRevoke { .. }), + ), + ]; + for (case, yaml, refusal) in cases { + let Err(err) = PermissionMaps::from_yaml(&yaml) else { + panic!("{case} should be rejected"); + }; + assert!(refusal(&err), "{case}: wrong refusal, got {err:?}"); + } + } + + #[test] + fn a_jurisdiction_is_inherited_and_can_be_overridden_at_each_level() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: g + jurisdiction: gdpr + GB: g + US: + group: g + jurisdiction: non-regulated + CA: + group: g + jurisdiction: us-state + NY: g +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("should parse the tree"); + assert_eq!( + maps.jurisdiction_for(Some("GB"), None), + Jurisdiction::Gdpr, + "a country naming none should inherit the top node" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), None), + Jurisdiction::NonRegulated, + "a country naming its own should override the top node" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), Some("NY")), + Jurisdiction::NonRegulated, + "a region naming none should inherit its country" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), Some("CA")), + Jurisdiction::UsState("CA".to_owned()), + "a region naming us-state should name itself as the state" + ); + assert_eq!( + maps.jurisdiction_for(Some("ZZ"), None), + Jurisdiction::Gdpr, + "an unlisted country should inherit the top node" + ); + assert_eq!( + maps.default_jurisdiction(), + Jurisdiction::Gdpr, + "no place at all should resolve the top node" + ); + } + + #[test] + fn place_codes_are_matched_without_regard_to_case() { + let yaml = format!( + "{TEST_GROUPS}\ +rules: + group: strict + jurisdiction: gdpr + us: + group: strict + jurisdiction: non-regulated + ca: + group: g + jurisdiction: us-state +" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("should parse lower-case place codes"); + assert!( + maps.baseline(Some("US"), Some("CA")) + .is_set(Permission::StoreOnDevice), + "an upper-case request should match a lower-case rule" + ); + assert_eq!( + maps.jurisdiction_for(Some("US"), Some("CA")), + Jurisdiction::UsState("CA".to_owned()), + "the state code should be upper-cased whatever case the file uses" + ); + } + + #[test] + fn the_settled_example_resolves_as_documented() { + // The example in the permissions.yaml header, kept here so the file's + // own teaching example is proven rather than asserted. + let yaml = r#" +groups: + gdpr-eu: + default: requires_signal + gdpr-uk: + default: requires_signal + us-notice: + default: granted + us-opt-out: + default: granted +rules: + group: gdpr-eu + jurisdiction: gdpr + GB: gdpr-uk + US: + group: us-notice + jurisdiction: non-regulated + CA: + group: us-opt-out + jurisdiction: us-state + NY: us-notice +"#; + let maps = PermissionMaps::from_yaml(yaml).expect("the header example should parse"); + for (country, region, jurisdiction) in [ + (Some("GB"), None, Jurisdiction::Gdpr), + (Some("US"), None, Jurisdiction::NonRegulated), + (Some("US"), Some("NY"), Jurisdiction::NonRegulated), + ( + Some("US"), + Some("CA"), + Jurisdiction::UsState("CA".to_owned()), + ), + (Some("JP"), None, Jurisdiction::Gdpr), + (None, None, Jurisdiction::Gdpr), + ] { + assert_eq!( + maps.jurisdiction_for(country, region), + jurisdiction, + "{country:?}/{region:?} should resolve as the header documents" + ); + } + assert!( + !maps + .baseline(Some("GB"), None) + .is_set(Permission::StoreOnDevice), + "GB takes the gdpr-uk group, which requires a signal in this example" + ); + assert!( + maps.baseline(Some("US"), Some("NY")) + .is_set(Permission::StoreOnDevice), + "NY takes us-notice, which grants" + ); + } + + #[test] + fn the_shipped_policy_reproduces_the_retired_consent_lists() { + // The policy tree carries the 31 GDPR countries and the 20 US + // privacy states, in place of compiled defaults in the consent + // configuration, so the shipped file must resolve each to its regime. + let maps = PermissionMaps::standard(); + let gdpr_countries = [ + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", + "LI", "NO", "GB", + ]; + assert_eq!( + gdpr_countries.len(), + 31, + "the retired list held 27 EU, 3 EEA and the UK" + ); + for country in gdpr_countries { + assert_eq!( + maps.jurisdiction_for(Some(country), None), + Jurisdiction::Gdpr, + "`{country}` should resolve the GDPR jurisdiction" + ); + } + + let privacy_states = [ + "CA", "VA", "CO", "CT", "UT", "MT", "OR", "TX", "FL", "DE", "IA", "NE", "NH", "NJ", + "TN", "MN", "MD", "IN", "KY", "RI", + ]; + assert_eq!( + privacy_states.len(), + 20, + "the retired list held 20 states with a comprehensive privacy law" + ); + for state in privacy_states { + assert_eq!( + maps.jurisdiction_for(Some("US"), Some(state)), + Jurisdiction::UsState(state.to_owned()), + "`US/{state}` should resolve its own state jurisdiction" + ); + } + + // A state with no law of its own inherits the US node, and the US with + // no region does the same. + for region in [Some("WY"), Some("NY"), None] { + assert_eq!( + maps.jurisdiction_for(Some("US"), region), + Jurisdiction::NonRegulated, + "`US/{region:?}` should inherit the country node" + ); + } + assert_eq!( + maps.jurisdiction_for(Some("AU"), None), + Jurisdiction::NonRegulated, + "Australia should carry its own non-regulated jurisdiction" + ); + assert_eq!( + maps.default_jurisdiction(), + Jurisdiction::Gdpr, + "an unresolved place should take the top node's GDPR jurisdiction" + ); + } + + #[test] + fn a_us_state_keeps_the_country_permission_baseline() { + // Each listed state carries the same group as the country node, so + // adding the states for the jurisdiction changed no permission. + let maps = PermissionMaps::standard(); + for state in ["CA", "TX", "DE", "RI", "WY"] { + assert!( + maps.baseline(Some("US"), Some(state)) + .is_set(Permission::StoreOnDevice), + "`US/{state}` should keep the US opt-out storage baseline" + ); + } + } + + #[test] + fn from_yaml_accepts_an_explicit_group_listing_every_permission() { + // The shipped style: no `default`, every permission spelled out. + let mut group = String::from("groups:\n everything:\n"); + for permission in Permission::all() { + group.push_str(&format!(" {permission}: granted\n")); + } + let yaml = format!( + "{group}rules:\n group: everything\n jurisdiction: unknown\n US: everything\n" + ); + let maps = PermissionMaps::from_yaml(&yaml).expect("an explicit group should parse"); + assert!( + maps.baseline(Some("US"), None) + .is_set(Permission::MarketResearch), + "every listed permission should take its flag" + ); + } + + #[test] + fn a_detailed_rule_can_set_requires_signal_per_permission() { + let yaml = "groups:\n g:\n default: granted\nrules:\n group: g\n jurisdiction: unknown\n US:\n group: g\n permissions:\n necessary.operations.storage: requires_signal\n"; + let maps = PermissionMaps::from_yaml(yaml).expect("should parse the override map"); + let rules = maps + .rules_for(Some("US"), None) + .expect("should resolve the US rule"); + assert_eq!( + rules.rule_for(Permission::StoreOnDevice), + Acquisition::RequiresSignal, + "the per-permission map should express requires_signal" + ); + assert_eq!( + rules.rule_for(Permission::SelectPersonalisedAds), + Acquisition::Granted, + "an unlisted permission should keep the group default" + ); + } + + #[test] + fn every_eu_and_eea_member_requires_a_signal_for_storage() { + // The shipped permissions.yaml must cover all 27 EU member states and + // the three EEA members, each resolving storage as requires-signal, so + // no member state silently falls to the top of the tree. + let maps = PermissionMaps::standard(); + for country in [ + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", + "LI", "NO", + ] { + let rules = maps + .rules_for(Some(country), None) + .unwrap_or_else(|| panic!("`{country}` should have a rule")); + assert_eq!( + rules.rule_for(Permission::StoreOnDevice), + Acquisition::RequiresSignal, + "storage in `{country}` should require a signal" + ); + } + } + + #[test] + fn from_yaml_parses_the_signals_section_into_a_policy() { + let yaml = "\ +groups: + g: + default: requires_signal +rules: + group: g + jurisdiction: gdpr + FR: g +signals: + tcf: + authoritative: true + us_opt_out: + sources: [gpc] + revokes: [advertising_marketing.first_party.targeted] +"; + let maps = PermissionMaps::from_yaml(yaml).expect("should parse the signals section"); + let signals = maps.signals(); + assert!(signals.tcf_authoritative(), "tcf should be authoritative"); + assert!( + signals.opt_out_revokes(Permission::SelectPersonalisedAds), + "a listed Data Use is revoked by the opt-out" + ); + assert!( + !signals.opt_out_revokes(Permission::StoreOnDevice), + "an unlisted Data Use is not revoked by the opt-out" + ); + } +} diff --git a/crates/trusted-server-core/src/platform/geo.rs b/crates/trusted-server-core/src/platform/geo.rs new file mode 100644 index 0000000000..c8077215dd --- /dev/null +++ b/crates/trusted-server-core/src/platform/geo.rs @@ -0,0 +1,30 @@ +use std::net::IpAddr; + +use error_stack::Report; + +use super::{GeoInfo, PlatformError, PlatformGeo, RuntimeServices}; + +/// A geo module that resolves nothing. +/// +/// Installed when the `[geo] module` selector is unset, and when it is set to +/// `"none"`, which spells the same choice explicitly. A client IP is then never +/// sent to any host geo service, so a default deployment is not tied to any host +/// geo capability. Every geo consumer already treats [`GeoInfo`] as optional, so +/// a `None` result degrades gracefully: the permission baseline falls back to +/// the top of the `permissions.yaml` rules tree, the auction omits geo, and so on. +/// +/// Adapter crates should use this type rather than defining their own stub so +/// the behavior is the same on every platform. +#[derive(Debug, Default, Clone, Copy)] +pub struct DisabledGeo; + +#[async_trait::async_trait(?Send)] +impl PlatformGeo for DisabledGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &RuntimeServices, + ) -> Result, Report> { + Ok(None) + } +} diff --git a/crates/trusted-server-core/src/platform/loading_cache.rs b/crates/trusted-server-core/src/platform/loading_cache.rs new file mode 100644 index 0000000000..8677381e1e --- /dev/null +++ b/crates/trusted-server-core/src/platform/loading_cache.rs @@ -0,0 +1,354 @@ +//! A cache that loads a missing value once, however many callers ask for it +//! at the same time. +//! +//! The first caller for a key that no layer holds runs the [`Loader`]. Every +//! caller that asks for the key while it runs waits for that load, and all of +//! them continue with its value or its error. A failed load is not stored, so +//! the next caller loads again. +//! +//! Core declares the interface and [`NoLoadingCache`], which holds nothing. A +//! module outside core supplies the implementation through +//! [`IntegrationBuilder::with_loading_cache`](crate::integrations::IntegrationBuilder::with_loading_cache), +//! keeping a brief copy in memory over the host's shared store, being its +//! value cache where it has one and its key-value store otherwise. +//! +//! Every key is a [`StoreKey`], which starts with the publisher's +//! [`ServerKeyPrefix`], so two publishers sharing a store never share an +//! entry. + +use core::fmt; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; + +use super::{RuntimeServices, ServerKeyPrefix}; + +/// A value a [`Loader`] produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LoadedValue { + /// The value. + pub bytes: Bytes, + /// How long the value may be used from now, or `None` for as long as the + /// cache's own lifetimes allow. Every copy of it ends by then. + pub lifetime: Option, +} + +impl LoadedValue { + /// A value with no lifetime of its own. + #[must_use] + pub fn new(bytes: impl Into) -> Self { + Self { + bytes: bytes.into(), + lifetime: None, + } + } + + /// Sets how long the value may be used from now. + #[must_use] + pub fn with_lifetime(mut self, lifetime: Duration) -> Self { + self.lifetime = Some(lifetime); + self + } +} + +/// Why a load produced no value. +/// +/// Every caller waiting on the load receives it, and nothing is stored, so the +/// next caller loads again. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LoadError { + message: std::sync::Arc, +} + +impl LoadError { + /// An error saying `message`. + #[must_use] + pub fn new(message: impl Into) -> Self { + Self { + message: message.into().into(), + } + } + + /// What went wrong. + #[must_use] + pub fn message(&self) -> &str { + &self.message + } +} + +impl fmt::Display for LoadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl core::error::Error for LoadError {} + +/// The work a [`Loader`] does. +/// +/// Not `Send`, because the platform's own futures are not. It is made on the +/// thread that runs it. +pub type LoadFuture = Pin>>>; + +/// What a cache runs when no layer holds a key. +/// +/// It owns everything it uses, and is `Send`, so a host that can run work on +/// its own can start it on another thread and keep it running after the +/// caller that started it has gone. +pub type Loader = Box LoadFuture + Send>; + +/// How long one cache's copies last. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheLifetimes { + /// How long the brief copy in memory is used. + pub memory: Duration, + /// How long the copy in the shared store is used, at most. A value with a + /// shorter lifetime of its own ends sooner. + pub store: Duration, + /// How long the copy in the shared store lasts unused. + pub store_idle: Duration, +} + +/// One cache: its name, the lifetimes its copies get, and how its loads may +/// run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheSpec { + /// The cache's name, which is part of every key it writes. + pub name: &'static str, + /// How long its copies last. + pub lifetimes: CacheLifetimes, + /// The most values the brief copy in memory holds. + pub memory_entries: usize, + /// Whether a load may run apart from the caller that started it, on a + /// host that can run work on its own. False for a load that blocks the + /// thread it runs on, which only the caller's own thread can afford. + pub detachable: bool, +} + +/// A key in a shared store, being the publisher's prefix and the cache's name +/// as a namespace, and the caller's own key below it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoreKey { + namespace: String, + key: String, +} + +impl StoreKey { + /// The key `key` in the cache `cache`, for the publisher `prefix` names. + /// + /// The caller's key should be short, such as a digest, because a store + /// refuses a long key. + #[must_use] + pub fn new(prefix: &ServerKeyPrefix, cache: &str, key: impl Into) -> Self { + Self { + namespace: format!("{}/{cache}", prefix.as_str()), + key: key.into(), + } + } + + /// The publisher's prefix and the cache's name, as `/`. + #[must_use] + pub fn namespace(&self) -> &str { + &self.namespace + } + + /// The caller's own key. + #[must_use] + pub fn key(&self) -> &str { + &self.key + } +} + +impl fmt::Display for StoreKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.namespace, self.key) + } +} + +/// A cache that loads a missing value once per key at a time. +/// +/// `Send + Sync` on the trait and `?Send` on the futures, as for the other +/// platform services, because [`RuntimeServices`] crosses threads while the +/// platform layer's futures never do. +#[async_trait::async_trait(?Send)] +pub trait PlatformLoadingCache: Send + Sync { + /// The value for `key` in the cache `spec` describes, from a copy that is + /// still fresh or from `load`. + /// + /// `load` runs at most once per key at a time, and a caller arriving while + /// it runs waits for it. The copies are kept in the stores `services` + /// supplies. + /// + /// # Errors + /// + /// The error of the load this caller waited on, which every caller waiting + /// on that load receives. + async fn get_or_load( + &self, + services: &RuntimeServices, + spec: &CacheSpec, + key: &StoreKey, + load: Loader, + ) -> Result; + + /// Removes `key` from the memory copy and the shared store. + async fn remove(&self, services: &RuntimeServices, spec: &CacheSpec, key: &StoreKey); + + /// Whether this cache keeps what it loads. A caller whose load pays off only + /// when its value is kept does without it on a cache that keeps nothing. + fn keeps_values(&self) -> bool { + true + } +} + +impl fmt::Debug for dyn PlatformLoadingCache { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PlatformLoadingCache") + } +} + +/// The loading cache of a deployment that links no module supplying one. +/// +/// It holds nothing and runs every load, so each caller asks the source +/// itself. +#[derive(Debug, Default, Clone, Copy)] +pub struct NoLoadingCache; + +#[async_trait::async_trait(?Send)] +impl PlatformLoadingCache for NoLoadingCache { + async fn get_or_load( + &self, + _services: &RuntimeServices, + _spec: &CacheSpec, + _key: &StoreKey, + load: Loader, + ) -> Result { + load().await.map(|loaded| loaded.bytes) + } + + async fn remove(&self, _services: &RuntimeServices, _spec: &CacheSpec, _key: &StoreKey) {} + + fn keeps_values(&self) -> bool { + false + } +} + +/// The loading cache each `[cache.]` table selected, by the name a +/// [`CacheSpec`] carries. A cache whose table selected none loads through +/// [`NoLoadingCache`]. +#[derive(Default)] +pub struct LoadingCaches { + by_feature: HashMap<&'static str, Arc>, + /// What a cache whose table selected nothing loads through, when a test + /// gives every caller one cache. + unselected: Option>, +} + +impl LoadingCaches { + /// Every caller loading through `cache`, for a test of one cache. + #[must_use] + pub fn every(cache: Arc) -> Self { + Self { + by_feature: HashMap::new(), + unselected: Some(cache), + } + } + + /// Keeps `cache` as the one the table `name` selected. + pub fn insert(&mut self, name: &'static str, cache: Arc) { + self.by_feature.insert(name, cache); + } + + /// The cache named `name` loads through. + #[must_use] + pub fn for_feature(&self, name: &str) -> &dyn PlatformLoadingCache { + match self.by_feature.get(name).or(self.unselected.as_ref()) { + Some(cache) => &**cache, + None => &NoLoadingCache, + } + } + + /// Whether no table selected a cache. + #[must_use] + pub fn is_empty(&self) -> bool { + self.by_feature.is_empty() && self.unselected.is_none() + } +} + +impl fmt::Debug for LoadingCaches { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut selected: Vec<&&str> = self.by_feature.keys().collect(); + selected.sort(); + f.debug_struct("LoadingCaches") + .field("selected", &selected) + .field("every", &self.unselected.is_some()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + #[test] + fn two_publishers_never_share_a_key() { + crate::prefix::test_support::assert_keys_follow_the_publisher(|domain| { + StoreKey::new(&ServerKeyPrefix::new("ts", domain), "page", "abc").to_string() + }); + } + + #[test] + fn a_key_names_the_publisher_then_the_cache() { + let key = StoreKey::new( + &ServerKeyPrefix::new("ts", "www.example.com"), + "robots-txt", + "k1", + ); + + assert_eq!(key.namespace(), "ts.example.com/robots-txt"); + assert_eq!(key.key(), "k1"); + assert_eq!(key.to_string(), "ts.example.com/robots-txt/k1"); + } + + #[test] + fn the_default_cache_runs_every_load() { + let services = super::super::test_support::noop_services(); + let spec = CacheSpec { + name: "test", + lifetimes: CacheLifetimes { + memory: Duration::from_secs(1), + store: Duration::from_secs(1), + store_idle: Duration::from_secs(1), + }, + memory_entries: 1, + detachable: false, + }; + let key = StoreKey::new(&ServerKeyPrefix::new("ts", "example.com"), "test", "k"); + let runs = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let counted = Arc::clone(&runs); + let value = futures::executor::block_on(NoLoadingCache.get_or_load( + &services, + &spec, + &key, + Box::new(move || { + Box::pin(async move { + counted.fetch_add(1, Ordering::SeqCst); + Ok(LoadedValue::new(Bytes::from_static(b"v"))) + }) + }), + )) + .expect("the load should succeed"); + assert_eq!(value, Bytes::from_static(b"v")); + } + + assert_eq!(runs.load(Ordering::SeqCst), 2); + } +} diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 63f5c2a94e..36231dfc6c 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -14,6 +14,8 @@ //! - [`PlatformGeo`] — geographic information lookup //! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! - [`PlatformTemplateCache`] — shared transformed-template caching +//! - [`PlatformValueCache`], a shared cache of small values with lifetimes +//! - [`PlatformLoadingCache`], a cache that loads a missing value once per key //! //! ## Platform-Agnostic Components //! @@ -36,15 +38,26 @@ use std::time::Duration; mod backend_naming; mod error; +mod geo; mod http; mod image_optimizer; mod kv; +mod loading_cache; +mod server_key; mod template_assembly; mod template_cache; -#[cfg(test)] -pub(crate) mod test_support; +#[cfg(any(test, feature = "test-utils"))] +#[allow( + clippy::must_use_candidate, + clippy::missing_panics_doc, + clippy::missing_errors_doc, + clippy::new_without_default, + reason = "test doubles, offered to other crates' tests under the test-utils feature" +)] +pub mod test_support; mod traits; mod types; +mod value_cache; pub use backend_naming::{ AuctionTargetCapabilities, AuctionTargetDescriptor, AuctionTargetId, BackendNamingError, @@ -52,6 +65,7 @@ pub use backend_naming::{ }; pub use edgezero_core::key_value_store::{KvError, KvHandle, KvStore as PlatformKvStore}; pub use error::PlatformError; +pub use geo::DisabledGeo; pub use http::{ PlatformCacheIntent, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, UnavailableHttpClient, @@ -61,6 +75,11 @@ pub use image_optimizer::{ PlatformImageOptimizerParams, PlatformImageOptimizerRegion, }; pub use kv::UnavailableKvStore; +pub use loading_cache::{ + CacheLifetimes, CacheSpec, LoadError, LoadFuture, LoadedValue, Loader, LoadingCaches, + NoLoadingCache, PlatformLoadingCache, StoreKey, +}; +pub use server_key::ServerKeyPrefix; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, contains_publisher_esi_directive, @@ -77,10 +96,42 @@ pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, StoreName, }; +pub use value_cache::{PlatformValueCache, ValueLookup, ValueReservation}; /// Default first-byte timeout for platform backends. pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +use std::sync::Arc; + +use crate::settings::Settings; + +/// Selects the geo module named by the `[geo] module` selector. +/// +/// Returns [`DisabledGeo`] when no module is selected, so a default +/// deployment makes no host geo call and the permission baseline comes from +/// the top of the `permissions.yaml` rules tree. `module = "none"` spells the +/// same choice explicitly. The host platform's own geo lookup is opt-in: +/// `module = "platform"` returns `host_default`, which the adapter passes +/// as its platform geo implementation. +/// +/// Any other value names an integration module that declares a geo module. +/// This function returns [`DisabledGeo`] for one, because it cannot see the +/// registry, and the adapter then replaces it with the module's module from +/// `IntegrationRegistry::geo_module`. So the value returned here is the base +/// the adapter starts from, not necessarily what serves the request. A selector +/// naming a module that supplies no geo module is rejected when the registry +/// is built, which is the only layer that can tell. +#[must_use] +pub fn build_geo_module( + settings: &Settings, + host_default: Arc, +) -> Arc { + match settings.geo.module.as_deref() { + Some("platform") => host_default, + _ => Arc::new(DisabledGeo), + } +} + #[cfg(test)] mod tests { use std::net::{IpAddr, Ipv4Addr}; @@ -148,6 +199,14 @@ mod tests { { } + #[test] + fn disabled_geo_requires_no_permissions() { + assert!( + DisabledGeo.required_permissions().is_empty(), + "the default disabled geo module requires no permissions" + ); + } + #[test] fn runtime_services_can_be_constructed_and_cloned() { let services = noop_services(); @@ -163,16 +222,63 @@ mod tests { ); } - #[test] - fn runtime_services_geo_lookup_returns_none_for_no_ip() { + #[tokio::test] + async fn runtime_services_geo_lookup_returns_none_for_no_ip() { let services = noop_services(); let result = services .geo() - .lookup(services.client_info().client_ip) + .lookup(services.client_info().client_ip, &noop_services()) + .await .expect("should not fail for noop geo with no ip"); assert!(result.is_none(), "should return None when no IP is present"); } + #[tokio::test] + async fn build_geo_module_defaults_to_no_geo() { + let settings = Settings::default(); + let host: Arc = Arc::new(test_support::NoopGeo); + let selected = build_geo_module(&settings, Arc::clone(&host)); + assert!( + !Arc::ptr_eq(&host, &selected), + "default settings should not use the host geo" + ); + assert!( + selected + .lookup( + Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7))), + &noop_services() + ) + .await + .expect("disabled geo lookup should not fail") + .is_none(), + "the default geo module should resolve nothing" + ); + } + + #[test] + fn build_geo_module_none_selects_no_geo_explicitly() { + let mut settings = Settings::default(); + settings.geo.module = Some("none".to_owned()); + let host: Arc = Arc::new(test_support::NoopGeo); + let selected = build_geo_module(&settings, Arc::clone(&host)); + assert!( + !Arc::ptr_eq(&host, &selected), + "module none should not use the host geo" + ); + } + + #[test] + fn build_geo_module_uses_host_geo_when_platform_is_selected() { + let mut settings = Settings::default(); + settings.geo.module = Some("platform".to_owned()); + let host: Arc = Arc::new(test_support::NoopGeo); + let selected = build_geo_module(&settings, Arc::clone(&host)); + assert!( + Arc::ptr_eq(&host, &selected), + "the platform selector should use the host geo" + ); + } + #[test] fn runtime_services_with_kv_store_replaces_only_the_new_clone() { let services = noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 7))); diff --git a/crates/trusted-server-core/src/platform/server_key.rs b/crates/trusted-server-core/src/platform/server_key.rs new file mode 100644 index 0000000000..435dc36dc5 --- /dev/null +++ b/crates/trusted-server-core/src/platform/server_key.rs @@ -0,0 +1,88 @@ +//! The start of every key a publisher's entries take in a shared store. + +use core::fmt; + +/// The start of every key a publisher's entries take in a shared store, so +/// two publishers sharing a store never share a key. +/// +/// Written `.`, where the head names who keeps the store, such +/// as `ts`, and the domain is the publisher's. The selected prefix module +/// gives both, see [`crate::prefix`], taking the domain from the publisher the +/// request resolved to and never from the raw request, because a `Host` +/// header can say anything and a prefix built from one would let any request +/// start a key space of its own. +/// +/// A single leading `www.` is folded, so `www.example.com` and `example.com` +/// share keys, and every other label is kept, so `clients.example.com` and +/// `public.example.com` do not. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerKeyPrefix { + text: String, + domain_start: usize, +} + +impl ServerKeyPrefix { + /// The prefix `head` and the publisher's `domain` make. Every shared store + /// key's prefix is made here. + #[must_use] + pub fn new(head: &str, domain: &str) -> Self { + let head = head.trim(); + let domain = domain.trim().trim_end_matches('.').to_ascii_lowercase(); + let folded = domain.strip_prefix("www.").unwrap_or(&domain); + Self { + text: format!("{head}.{folded}"), + domain_start: head.len() + 1, + } + } + + /// The prefix as text. + #[must_use] + pub fn as_str(&self) -> &str { + &self.text + } + + /// The publisher's domain the prefix carries, lowercased and with a + /// leading `www.` folded. + #[must_use] + pub fn domain(&self) -> &str { + &self.text[self.domain_start..] + } +} + +impl fmt::Display for ServerKeyPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_leading_www_is_folded_and_every_other_label_kept() { + for (domain, prefix) in [ + ("example.com", "ts.example.com"), + ("www.example.com", "ts.example.com"), + ("WWW.Example.COM", "ts.example.com"), + ("example.com.", "ts.example.com"), + ("clients.example.com", "ts.clients.example.com"), + ("public.example.com", "ts.public.example.com"), + ("www.www.example.com", "ts.www.example.com"), + ] { + assert_eq!( + ServerKeyPrefix::new("ts", domain).as_str(), + prefix, + "{domain}" + ); + } + } + + #[test] + fn the_domain_is_the_folded_one() { + let prefix = ServerKeyPrefix::new("ts", "www.example.com"); + + assert_eq!(prefix.domain(), "example.com"); + assert_eq!(prefix.to_string(), "ts.example.com"); + } +} diff --git a/crates/trusted-server-core/src/platform/template_cache.rs b/crates/trusted-server-core/src/platform/template_cache.rs index 203bba1625..e50bfe5032 100644 --- a/crates/trusted-server-core/src/platform/template_cache.rs +++ b/crates/trusted-server-core/src/platform/template_cache.rs @@ -59,7 +59,7 @@ pub struct TemplateCacheKey { /// mutate first. pub url: String, /// Host and scheme. The post-processed output is host-dependent by construction: - /// both reach `IntegrationHtmlContext` and drive URL rewriting. + /// both are handed to every middleware and drive URL rewriting. pub request_host: String, /// See [`Self::request_host`]. pub request_scheme: String, @@ -819,17 +819,6 @@ pub trait PlatformTemplateCache: Send + Sync { /// Purge every cached variant for one publisher URL. async fn purge_url(&self, key: &TemplateCacheKey) -> Result<(), TemplateCacheError>; - - /// Purge one already-derived surrogate key. - /// - /// Exists because [`Self::purge_url`] needs a whole [`TemplateCacheKey`], which a - /// purge caller cannot build: an operator or a CMS webhook has a URL, not an origin - /// identity, a template fingerprint, or the origin's `Vary` values. Pair it with - /// [`reader_url_surrogate_key`]. - async fn purge_url_surrogate_key(&self, key: &str) -> Result<(), TemplateCacheError>; - - /// Purge every stored template. The rollback lever. - async fn purge_all(&self) -> Result<(), TemplateCacheError>; } /// A template read from the cache. @@ -874,17 +863,6 @@ impl PlatformTemplateCache for UnavailableTemplateCache { async fn purge_url(&self, _key: &TemplateCacheKey) -> Result<(), TemplateCacheError> { Err(TemplateCacheError::Unsupported) } - - /// Reports unsupported rather than succeeding. A purge surface that silently does - /// nothing is worse than one that refuses: an operator mid-incident would read the - /// success and stop looking. - async fn purge_url_surrogate_key(&self, _key: &str) -> Result<(), TemplateCacheError> { - Err(TemplateCacheError::Unsupported) - } - - async fn purge_all(&self) -> Result<(), TemplateCacheError> { - Err(TemplateCacheError::Unsupported) - } } #[cfg(test)] @@ -1197,19 +1175,6 @@ mod tests { } } - #[test] - fn an_adapter_without_a_template_cache_refuses_a_surrogate_purge() { - let outcome = futures::executor::block_on( - UnavailableTemplateCache.purge_url_surrogate_key("ts-template-readerurl-abc"), - ); - - assert!( - matches!(outcome, Err(TemplateCacheError::Unsupported)), - "a purge surface that silently does nothing is worse than one that refuses: \ - an operator mid-incident would read the success and stop looking" - ); - } - #[test] fn the_reader_facing_key_ignores_origin_rewriting() { // `url` is the origin-rewritten target URI, not what an operator types. Two diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 17813c77dc..ee2ba8b1d7 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -11,13 +11,11 @@ use rand::rngs::OsRng; use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, - StoreName, + PlatformImageOptimizerOptions, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, + PlatformSelectResult, RuntimeServices, StoreId, StoreName, }; -use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; -pub(crate) struct NoopConfigStore; +pub struct NoopConfigStore; impl PlatformConfigStore for NoopConfigStore { fn get(&self, _store_name: &StoreName, _key: &str) -> Result> { @@ -38,7 +36,7 @@ impl PlatformConfigStore for NoopConfigStore { } } -pub(crate) struct NoopSecretStore; +pub struct NoopSecretStore; impl PlatformSecretStore for NoopSecretStore { fn get_bytes( @@ -63,12 +61,12 @@ impl PlatformSecretStore for NoopSecretStore { } } -pub(crate) struct HashMapConfigStore { +pub struct HashMapConfigStore { data: HashMap, } impl HashMapConfigStore { - pub(crate) fn new(data: HashMap) -> Self { + pub fn new(data: HashMap) -> Self { Self { data } } } @@ -95,12 +93,12 @@ impl PlatformConfigStore for HashMapConfigStore { } } -pub(crate) struct HashMapSecretStore { +pub struct HashMapSecretStore { data: HashMap>, } impl HashMapSecretStore { - pub(crate) fn new(data: HashMap>) -> Self { + pub fn new(data: HashMap>) -> Self { Self { data } } } @@ -131,7 +129,7 @@ impl PlatformSecretStore for HashMapSecretStore { } } -pub(crate) struct NoopBackend; +pub struct NoopBackend; impl PlatformBackend for NoopBackend { fn naming_policy(&self) -> super::BackendNamingPolicy { @@ -147,7 +145,7 @@ impl PlatformBackend for NoopBackend { } } -pub(crate) struct NoopHttpClient; +pub struct NoopHttpClient; // ?Send matches PlatformHttpClient. Body wraps LocalBoxStream which is !Send // by design; see http.rs for the full rationale. @@ -181,7 +179,7 @@ impl PlatformHttpClient for NoopHttpClient { /// Test stub for [`PlatformBackend`] that returns `"stub-backend"` for any /// spec, allowing callers to proceed past backend registration. -pub(crate) struct StubBackend; +pub struct StubBackend; impl PlatformBackend for StubBackend { fn naming_policy(&self) -> super::BackendNamingPolicy { @@ -197,6 +195,82 @@ impl PlatformBackend for StubBackend { } } +// --------------------------------------------------------------------------- +// NamingBackend +// --------------------------------------------------------------------------- + +/// Test stub for [`PlatformBackend`] that names a backend the way the given +/// policy does, so two demand sources get two names, and records what it was +/// asked. +pub struct NamingBackend { + pub policy: super::BackendNamingPolicy, + /// How many names were predicted. + pub predicted: std::sync::atomic::AtomicUsize, + /// How many backends were registered or refused. + pub ensured: std::sync::atomic::AtomicUsize, + /// The specification of every backend registered, in order. + pub specs: Mutex>, + /// The discriminators whose registration fails. + pub fail_ensure_for: Mutex>, +} + +impl NamingBackend { + pub fn new(policy: super::BackendNamingPolicy) -> Self { + Self { + policy, + predicted: std::sync::atomic::AtomicUsize::new(0), + ensured: std::sync::atomic::AtomicUsize::new(0), + specs: Mutex::new(Vec::new()), + fail_ensure_for: Mutex::new(std::collections::HashSet::new()), + } + } + + /// Makes registration fail for the source `provider_id`. + pub fn fail_ensure_for(&self, provider_id: &str) { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .insert(provider_id.to_string()); + } + + fn name(&self, spec: &PlatformBackendSpec) -> Result> { + self.policy + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) + } +} + +impl PlatformBackend for NamingBackend { + fn naming_policy(&self) -> super::BackendNamingPolicy { + self.policy + } + + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { + self.predicted + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.name(spec) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.ensured + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if spec.discriminator.as_deref().is_some_and(|provider_id| { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .contains(provider_id) + }) { + return Err(Report::new(PlatformError::Backend)); + } + self.specs + .lock() + .expect("should lock planned backend specs") + .push(spec.clone()); + self.name(spec) + } +} + // --------------------------------------------------------------------------- // StubHttpClient // --------------------------------------------------------------------------- @@ -226,7 +300,7 @@ struct StubPendingResponse { const MAX_RECORDED_BODY_BYTES: usize = 64 * 1024 * 1024; type RecordedHeaderBytes = Vec)>>; -pub(crate) struct StubHttpClient { +pub struct StubHttpClient { calls: Mutex>, responses: Mutex>, // Headers captured per send call, stored as (name, value) string pairs. @@ -302,7 +376,7 @@ impl StubHttpClient { } /// Make `has_enforceable_total_request_deadline()` report the given value. - pub(crate) fn set_enforceable_total_request_deadline(&self, supported: bool) { + pub fn set_enforceable_total_request_deadline(&self, supported: bool) { self.enforceable_total_request_deadline .store(supported, std::sync::atomic::Ordering::Relaxed); } @@ -370,7 +444,7 @@ impl StubHttpClient { } /// Make the next `select()` complete successfully before a later queued error. - pub(crate) fn push_select_success(&self) { + pub fn push_select_success(&self) { self.select_errors .lock() .expect("should lock select_errors") @@ -379,7 +453,7 @@ impl StubHttpClient { /// Override backend metadata on the next pending handle returned by /// [`Self::send_async`]. `None` removes the metadata entirely. - pub(crate) fn push_pending_backend_name_override(&self, backend_name: Option<&str>) { + pub fn push_pending_backend_name_override(&self, backend_name: Option<&str>) { self.pending_backend_name_overrides .lock() .expect("should lock pending backend name overrides") @@ -391,7 +465,7 @@ impl StubHttpClient { /// This is test-only timing control for deadline behavior. It deliberately /// uses a caller-selected, generous contrast with the tested budget rather /// than relying on scheduler races. - pub(crate) fn push_select_delay(&self, delay: Duration) { + pub fn push_select_delay(&self, delay: Duration) { self.select_delays .lock() .expect("should lock select_delays") @@ -399,7 +473,7 @@ impl StubHttpClient { } /// Queue a wall-clock delay before the next direct pending-request `wait()`. - pub(crate) fn push_wait_delay(&self, delay: Duration) { + pub fn push_wait_delay(&self, delay: Duration) { self.wait_delays .lock() .expect("should lock wait_delays") @@ -432,7 +506,7 @@ impl StubHttpClient { /// Return raw request header values captured per request, in order. /// /// Unlike [`Self::recorded_request_headers`], this includes malformed bytes. - pub(crate) fn recorded_request_header_bytes(&self) -> Vec)>> { + pub fn recorded_request_header_bytes(&self) -> Vec)>> { self.request_header_bytes .lock() .expect("should lock request_header_bytes") @@ -448,7 +522,7 @@ impl StubHttpClient { } /// Return the cache intent captured per `send` or `send_async` call, in order. - pub(crate) fn recorded_cache_intents(&self) -> Vec { + pub fn recorded_cache_intents(&self) -> Vec { self.cache_intents .lock() .expect("should lock cache intents") @@ -857,10 +931,15 @@ fn build_stub_pending_response( builder.body(body).change_context(PlatformError::HttpClient) } -pub(crate) struct NoopGeo; +pub struct NoopGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for NoopGeo { - fn lookup(&self, _client_ip: Option) -> Result, Report> { + async fn lookup( + &self, + _client_ip: Option, + _services: &crate::platform::RuntimeServices, + ) -> Result, Report> { Ok(None) } } @@ -869,7 +948,7 @@ impl PlatformGeo for NoopGeo { /// /// Use this when a test exercises code that reads from config AND secret stores, /// such as `request_signing::signing` and `request_signing::rotation`. -pub(crate) fn build_services_with_config_and_secret( +pub fn build_services_with_config_and_secret( config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, ) -> RuntimeServices { @@ -884,7 +963,7 @@ pub(crate) fn build_services_with_config_and_secret( .build() } -pub(crate) fn build_services_with_config_and_secret_and_client_ip( +pub fn build_services_with_config_and_secret_and_client_ip( config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, client_ip: IpAddr, @@ -903,7 +982,7 @@ pub(crate) fn build_services_with_config_and_secret_and_client_ip( .build() } -pub(crate) fn build_request_signing_services() -> RuntimeServices { +pub fn build_request_signing_services() -> RuntimeServices { let signing_key = SigningKey::generate(&mut OsRng); let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); let x_b64 = general_purpose::URL_SAFE_NO_PAD.encode(signing_key.verifying_key().as_bytes()); @@ -923,7 +1002,7 @@ pub(crate) fn build_request_signing_services() -> RuntimeServices { ) } -pub(crate) fn build_services_with_config( +pub fn build_services_with_config( config_store: impl PlatformConfigStore + 'static, ) -> RuntimeServices { RuntimeServices::builder() @@ -937,14 +1016,153 @@ pub(crate) fn build_services_with_config( .build() } -pub(crate) fn noop_services() -> RuntimeServices { +pub fn noop_services() -> RuntimeServices { build_services_with_config(NoopConfigStore) } +/// Build a [`RuntimeServices`] with an injected geo module, so a test can +/// drive a geo outcome through the [`PlatformGeo`] seam rather than +/// constructing the resolved status by hand. +/// +/// This is the only way to reach the lookup-failure path, because the seam is +/// what turns an `Err` into the requires-signal floor. +pub fn build_services_with_geo(geo: Arc) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(geo) + .client_info(ClientInfo::default()) + .build() +} + +/// Build a [`RuntimeServices`] carrying a device module, so a test can drive +/// the device seam rather than setting the resolved attributes by hand. +/// +/// Setting the attributes directly would prove the `OpenRTB` mapping and +/// nothing about whether any module is ever asked, which is the half of the +/// chain that breaks. +pub fn noop_services_with_device_module( + device_module: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build() + .with_device_module(device_module) +} + +/// Build a [`RuntimeServices`] carrying an Edge Cookie module, threaded the +/// way a composition root threads the module it resolved, so a test can +/// exercise the seam a vendor module reaches core through and check that the +/// request path reuses that instance. +pub fn noop_services_with_ec_module( + ec_module: Arc, +) -> RuntimeServices { + // A fixed client IP, so a module that reads one (the built-in HMAC + // module does) can run. + noop_services_with_ec_module_and_ip( + ec_module, + Some("203.0.113.10".parse().expect("should parse test client IP")), + ) +} + +/// Build a [`RuntimeServices`] with an injected Edge Cookie module and no +/// client IP, modeling a host that cannot determine one. +/// +/// Whether that matters is the module's decision, so this exists to test +/// both answers: a module reading other evidence still creates an +/// identifier, and one that needs the IP refuses. +/// A config store that answers one known key, so a test can prove a module +/// reached the config store it was handed rather than a value it already held. +#[derive(Debug)] +pub struct FixedConfigStore { + pub store: &'static str, + pub key: &'static str, + pub value: &'static str, +} + +impl PlatformConfigStore for FixedConfigStore { + fn get(&self, store_name: &StoreName, key: &str) -> Result> { + if store_name.as_ref() == self.store && key == self.key { + Ok(self.value.to_owned()) + } else { + Err(Report::new(PlatformError::ConfigStore)) + } + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +/// Build a [`RuntimeServices`] carrying both an Edge Cookie module and a +/// config store the module is expected to read through, so a test can prove +/// the services reaching the module are the ones the caller supplied. +pub fn services_with_ec_module_and_config_store( + ec_module: Arc, + config_store: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(config_store) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: Some("203.0.113.10".parse().expect("should parse test client IP")), + ..ClientInfo::default() + }) + .resolved_ec_module(ec_module) + .build() +} + +pub fn noop_services_with_ec_module_without_client_ip( + ec_module: Arc, +) -> RuntimeServices { + noop_services_with_ec_module_and_ip(ec_module, None) +} + +fn noop_services_with_ec_module_and_ip( + ec_module: Arc, + client_ip: Option, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip, + ..ClientInfo::default() + }) + .resolved_ec_module(ec_module) + .build() +} + /// Build a [`RuntimeServices`] whose auction telemetry sink is the supplied /// recording (or otherwise custom) sink, so tests can assert which terminal /// auction events were emitted. -pub(crate) fn noop_services_with_telemetry_sink( +pub fn noop_services_with_telemetry_sink( auction_telemetry_sink: Arc, ) -> RuntimeServices { RuntimeServices::builder() @@ -966,14 +1184,14 @@ pub(crate) fn noop_services_with_telemetry_sink( /// both make HTTP calls and resolve backends don't need two separate service /// setups. If your test must verify that a missing backend returns an error, /// use [`noop_services`] directly. -pub(crate) fn build_services_with_http_client( +pub fn build_services_with_http_client( http_client: Arc, ) -> RuntimeServices { build_services_with_secret_and_http_client(NoopSecretStore, http_client) } /// Build test services that dispatch HTTP requests with an attested client IP. -pub(crate) fn build_services_with_http_client_and_client_ip( +pub fn build_services_with_http_client_and_client_ip( http_client: Arc, client_ip: IpAddr, ) -> RuntimeServices { @@ -991,7 +1209,7 @@ pub(crate) fn build_services_with_http_client_and_client_ip( .build() } -pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { +pub fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(NoopConfigStore)) .secret_store(Arc::new(NoopSecretStore)) @@ -1017,7 +1235,7 @@ pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { dead_code, reason = "retained for target-specific transport-timeout tests" )] -pub(crate) fn build_services_with_backend_and_http_client( +pub fn build_services_with_backend_and_http_client( backend: Arc, http_client: Arc, ) -> RuntimeServices { @@ -1038,7 +1256,7 @@ pub(crate) fn build_services_with_backend_and_http_client( } /// Build a [`RuntimeServices`] with a custom secret store, [`StubBackend`], and HTTP client. -pub(crate) fn build_services_with_config_secret_and_http_client( +pub fn build_services_with_config_secret_and_http_client( config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, http_client: Arc, @@ -1054,7 +1272,7 @@ pub(crate) fn build_services_with_config_secret_and_http_client( .build() } -pub(crate) fn build_services_with_secret_http_client_and_client_ip( +pub fn build_services_with_secret_http_client_and_client_ip( secret_store: impl PlatformSecretStore + 'static, http_client: Arc, client_ip: Option, @@ -1076,16 +1294,100 @@ pub(crate) fn build_services_with_secret_http_client_and_client_ip( } /// Build test services with a custom secret store and the standard test config store. -pub(crate) fn build_services_with_secret_and_http_client( +pub fn build_services_with_secret_and_http_client( secret_store: impl PlatformSecretStore + 'static, http_client: Arc, ) -> RuntimeServices { build_services_with_config_secret_and_http_client(NoopConfigStore, secret_store, http_client) } +/// A loading cache for tests that cannot link the module supplying the real +/// one. +/// +/// It keeps each value until the lifetime its load gave runs out, records the +/// key and cache of every call, and runs a load for every key it does not +/// hold. Loads are not joined, because these tests ask one at a time, and +/// the module's own tests prove the joining. +#[derive(Default)] +pub struct MemoryLoadingCache { + entries: Mutex)>>, + calls: Mutex>, +} + +impl MemoryLoadingCache { + /// The cache name and key of every call, in order. + pub fn calls(&self) -> Vec<(&'static str, String)> { + self.calls.lock().expect("should lock the calls").clone() + } + + /// Every key held, sorted. + pub fn keys(&self) -> Vec { + let mut keys: Vec = self + .entries + .lock() + .expect("should lock the entries") + .keys() + .cloned() + .collect(); + keys.sort(); + keys + } +} + +#[async_trait::async_trait(?Send)] +impl super::PlatformLoadingCache for MemoryLoadingCache { + async fn get_or_load( + &self, + _services: &RuntimeServices, + spec: &super::CacheSpec, + key: &super::StoreKey, + load: super::Loader, + ) -> Result { + let key = key.to_string(); + self.calls + .lock() + .expect("should lock the calls") + .push((spec.name, key.clone())); + let held = self + .entries + .lock() + .expect("should lock the entries") + .get(&key) + .filter(|(_, end)| end.is_none_or(|end| web_time::Instant::now() < end)) + .map(|(bytes, _)| bytes.clone()); + if let Some(bytes) = held { + return Ok(bytes); + } + let loaded = load().await?; + let end = loaded + .lifetime + .map(|lifetime| web_time::Instant::now() + lifetime); + if loaded.lifetime.is_none_or(|lifetime| !lifetime.is_zero()) { + self.entries + .lock() + .expect("should lock the entries") + .insert(key, (loaded.bytes.clone(), end)); + } + Ok(loaded.bytes) + } + + async fn remove( + &self, + _services: &RuntimeServices, + _spec: &super::CacheSpec, + key: &super::StoreKey, + ) { + self.entries + .lock() + .expect("should lock the entries") + .remove(&key.to_string()); + } +} + #[cfg(test)] mod tests { - use crate::platform::DEFAULT_FIRST_BYTE_TIMEOUT; + use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformImageOptimizerParams}; + use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; use edgezero_core::body::Body; use edgezero_core::http::request_builder; diff --git a/crates/trusted-server-core/src/platform/traits.rs b/crates/trusted-server-core/src/platform/traits.rs index 6c31cd2796..47d7d9cf57 100644 --- a/crates/trusted-server-core/src/platform/traits.rs +++ b/crates/trusted-server-core/src/platform/traits.rs @@ -2,7 +2,10 @@ use std::net::IpAddr; use error_stack::Report; -use super::{BackendNamingPolicy, GeoInfo, PlatformBackendSpec, PlatformError, StoreId, StoreName}; +use super::{ + BackendNamingPolicy, GeoInfo, PlatformBackendSpec, PlatformError, RuntimeServices, StoreId, + StoreName, +}; /// Synchronous, object-safe access to a key-value config store. /// @@ -136,13 +139,44 @@ pub trait PlatformBackend: Send + Sync { } } -/// Synchronous, object-safe geo lookup. +/// Object-safe geo lookup. +/// +/// Asynchronous because a geo module may reach a backend, a key-value store +/// or a secret to resolve a location, and a module that cannot make those +/// calls cannot be written at all. A module that resolves from data already +/// in hand still declares an async method and returns immediately. +/// +/// Uses `#[async_trait(?Send)]` for the same reason as +/// [`PlatformHttpClient`](super::PlatformHttpClient): the trait object stays +/// `Send + Sync` so it can be shared and run multi-threaded, while the future +/// it returns is pinned to one thread because the host SDKs produce `!Send` +/// futures on wasm32. +#[async_trait::async_trait(?Send)] pub trait PlatformGeo: Send + Sync { /// Look up geographic information for the given client IP address. /// + /// An implementation must return [`GeoInfo`] with the country as an + /// ISO 3166-1 alpha-2 code (for example `US`) and the region as the + /// ISO 3166-2 subdivision code without the country prefix (for example + /// `CA`). The permission model keys its country and region rules on these + /// codes, matched case-insensitively, so the Fastly and other geo + /// modules feed the same rules without translation. + /// /// # Errors /// /// Returns [`PlatformError::Geo`] when the platform geo lookup fails /// unexpectedly. Returns `Ok(None)` when no data is available for the IP. - fn lookup(&self, client_ip: Option) -> Result, Report>; + async fn lookup( + &self, + client_ip: Option, + services: &RuntimeServices, + ) -> Result, Report>; + + /// The permissions this module's data use requires. + /// + /// The default is empty, so the default (disabled) geo module requires no + /// permission. + fn required_permissions(&self) -> crate::permissions::PermissionSet { + crate::permissions::PermissionSet::none() + } } diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 3570bdc102..8b2102fa0f 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -9,6 +9,10 @@ use super::{ PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformHttpClient, PlatformKvStore, PlatformSecretStore, }; +use crate::ec::device::DeviceModule; +use crate::ec::module::EdgeCookieModule; +use crate::evidence::HostSignals; +use crate::permission_signal::PermissionSignalModule; /// Geographic information extracted from a request. /// @@ -18,7 +22,7 @@ use super::{ pub struct GeoInfo { /// City name. pub city: String, - /// Two-letter country code. + /// ISO 3166-1 alpha-2 country code, for example `US` or `GB`. pub country: String, /// Continent name. pub continent: String, @@ -28,7 +32,8 @@ pub struct GeoInfo { pub longitude: f64, /// DMA (Designated Market Area) / metro code. pub metro_code: i64, - /// Region code. + /// ISO 3166-2 subdivision code without the country prefix, for example `CA` + /// for California, or `None` when no region resolves. pub region: Option, /// Autonomous System Number (e.g. `7922` = Comcast). /// Used to distinguish home ISP vs. corporate VPN. @@ -168,6 +173,16 @@ pub struct RuntimeServices { /// per-request basis by cloning [`RuntimeServices`] with /// [`RuntimeServices::with_kv_store`]. pub(crate) kv_store: Arc, + /// The host's shared cache of small values, or `None` where the host has + /// none, in which case a caller holds what it pays for in the key-value + /// store instead. + pub(crate) value_cache: Option>, + /// The loading cache each `[cache.]` table selected, which loads a + /// missing value once per key and keeps its copies in + /// [`value_cache`](Self::value_cache) or the key-value store. A cache whose + /// table selected none loads through [`super::NoLoadingCache`], which holds + /// nothing. + pub(crate) loading_caches: Arc, /// Shared transformed-template cache. Defaults to /// [`UnavailableTemplateCache`], so adapters without one degrade to transforming /// per request rather than failing. See @@ -188,6 +203,29 @@ pub struct RuntimeServices { pub(crate) auction_telemetry_sink: Arc, /// Per-request client metadata extracted at the entry point. pub(crate) client_info: ClientInfo, + /// Host-computed client signals (TLS JA4, HTTP/2), when the host + /// supplies them. `None` on a host that exposes none, so a module that + /// requires them cannot be built and the request stops. + pub(crate) host_signals: Option>, + /// The Edge Cookie module this deployment already resolved from + /// `[ec] module` while it built application state. + /// + /// `None` when the adapter resolved nothing here, in which case the request + /// path resolves the selection itself, which is what a deployment that + /// selects no module, the Axum adapter, and the core tests all do. + pub(crate) resolved_ec_module: Option>, + pub(crate) device_module: Option>, + /// The permission signal modules this deployment runs, in the order + /// they are asked, selected at the composition root from the scheme + /// crates the adapter links. Empty when the adapter offers none, in which + /// case every permission stays at its country and region baseline. Shared, + /// so building the services for a request bumps a reference count rather + /// than copying the list, and cloning the services does the same. + pub(crate) permission_signal_modules: Arc<[Arc]>, + /// The prefixes of the publisher this request was resolved to, from the + /// selected prefix module, which start every name the browser holds and + /// every key in a shared store. See [`crate::prefix`]. + pub(crate) prefixes: crate::prefix::Prefixes, } impl RuntimeServices { @@ -233,6 +271,37 @@ impl RuntimeServices { &*self.kv_store } + /// The host's shared cache of small values, or `None` where it has none. + #[must_use] + pub fn value_cache(&self) -> Option<&dyn super::PlatformValueCache> { + self.value_cache.as_deref() + } + + /// The loading cache the table `[cache.]` selected, by the name a + /// [`super::CacheSpec`] carries. + #[must_use] + pub fn loading_cache_for(&self, name: &str) -> &dyn super::PlatformLoadingCache { + self.loading_caches.for_feature(name) + } + + /// The value for `key` in the cache `spec` describes, from a fresh copy + /// or from `load`, kept in this request's stores. See + /// [`super::PlatformLoadingCache::get_or_load`]. + /// + /// # Errors + /// + /// The error of the load this caller waited on. + pub async fn load_cached( + &self, + spec: &super::CacheSpec, + key: &super::StoreKey, + load: super::Loader, + ) -> Result { + self.loading_cache_for(spec.name) + .get_or_load(self, spec, key, load) + .await + } + /// The shared transformed-template cache. #[must_use] pub fn template_cache(&self) -> &dyn super::PlatformTemplateCache { @@ -275,6 +344,66 @@ impl RuntimeServices { &self.client_info } + /// Returns the host-computed client signals, when the host supplies + /// them. + /// + /// A module that derives identity from the TLS JA4 or HTTP/2 signals + /// takes these as an injected service. The result is `None` on a host that + /// exposes none, so such a module cannot be built there and the request + /// stops rather than creating a degraded identifier. + #[must_use] + pub fn host_signals(&self) -> Option> { + self.host_signals.clone() + } + + /// Returns the Edge Cookie module the composition root already resolved, + /// when the adapter threaded one through. + /// + /// Resolving `[ec] module` reads no request data, so the answer is the + /// same for every request and an adapter that resolves it once while it + /// builds application state can hand the result here instead of the + /// request path resolving the same settings again. `None` means nothing was + /// threaded, so the request path resolves for itself. Read this through + /// [`request_module`](crate::ec::module::request_module) rather than + /// directly, so both answers are handled in one place. + #[must_use] + pub fn resolved_ec_module(&self) -> Option> { + self.resolved_ec_module.clone() + } + + /// The permission signal modules this deployment runs, in order. + #[must_use] + pub fn permission_signal_modules(&self) -> &[Arc] { + &self.permission_signal_modules + } + + /// The device module a module supplied, when `[device] module` selected + /// one. `None` leaves core's own device classification in place. + #[must_use] + pub fn device_module(&self) -> Option> { + self.device_module.clone() + } + + /// The prefixes of the publisher this request was resolved to, being the + /// start of every name the browser holds and of every key in a shared + /// store. See [`crate::prefix`]. + #[must_use] + pub fn prefixes(&self) -> &crate::prefix::Prefixes { + &self.prefixes + } + + /// The start of every key the publisher's entries take in a shared store. + #[must_use] + pub fn server_key_prefix(&self) -> &super::ServerKeyPrefix { + self.prefixes.server_key() + } + + /// The start of every name the browser holds for the publisher. + #[must_use] + pub fn public_prefix(&self) -> &crate::prefix::PublicPrefix { + self.prefixes.public() + } + /// Wrap the KV store in a [`super::KvHandle`] for ergonomic access to /// JSON helpers, pagination, and validation. #[must_use] @@ -295,6 +424,108 @@ impl RuntimeServices { } } + /// Returns a clone of this instance with the resolved Edge Cookie module + /// replaced. + /// + /// Adapters that build their per-request services through a shared helper + /// with no application state in hand use this to thread the module the + /// composition root resolved. `None` leaves the request path to resolve + /// `[ec] module` for itself, which is what the Axum adapter and a + /// deployment selecting no module both do. + #[must_use] + pub fn with_resolved_ec_module( + self, + resolved_ec_module: Option>, + ) -> Self { + Self { + resolved_ec_module, + ..self + } + } + + /// Returns a clone of this instance with the geo module replaced by + /// `geo`. + /// + /// Adapters use this to apply the module-supplied geo module selected by + /// `[geo] module`, so a vendor module can resolve location in place of + /// the host's own lookup without the rest of the runtime services graph + /// being rebuilt. + #[must_use] + pub fn with_geo(self, geo: Arc) -> Self { + Self { geo, ..self } + } + + /// Returns a clone of this instance with the per-request client metadata + /// replaced. + /// + /// A response-side path that builds the rest of the services graph once and + /// varies only the client metadata per request uses this to apply the real + /// client data without rebuilding the graph. A module called from such a + /// path still needs that data. + #[must_use] + pub fn with_client_info(self, client_info: ClientInfo) -> Self { + Self { + client_info, + ..self + } + } + + /// Returns a clone of this instance with the Edge Cookie module replaced. + /// + /// Adapters use this to apply the module-supplied module selected by + /// `[ec] module`, the same way they apply a module's geo module, so a + /// vendor declares identity on its registration rather than through a + /// second extension mechanism. The registry already matched the selector + /// against the registered modules, so the module's module is the + /// resolved answer and lands on the one seam the request path reads. + #[must_use] + pub fn with_ec_module(self, ec_module: Arc) -> Self { + Self { + resolved_ec_module: Some(ec_module), + ..self + } + } + + /// Returns a clone of this instance with the device module replaced. + /// + /// Adapters use this to apply the module-supplied module selected by + /// `[device] module`. + #[must_use] + pub fn with_device_module(self, device_module: Arc) -> Self { + Self { + device_module: Some(device_module), + ..self + } + } + + /// Returns a clone of this instance carrying the prefixes of the publisher + /// the request was resolved to. + /// + /// Adapters apply what the selected prefix module gives, through + /// [`IntegrationRegistry::prefixes`](crate::integrations::IntegrationRegistry::prefixes), + /// alongside the other modules a request's services carry. + #[must_use] + pub fn with_prefixes(self, prefixes: crate::prefix::Prefixes) -> Self { + Self { prefixes, ..self } + } + + /// Returns a clone of this instance with the loading caches replaced. + #[must_use] + pub fn with_loading_caches(self, caches: Arc) -> Self { + Self { + loading_caches: caches, + ..self + } + } + + /// Returns a clone of this instance with every caller loading through + /// `cache`, for a test of one cache. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn with_loading_cache(self, cache: Arc) -> Self { + self.with_loading_caches(Arc::new(super::LoadingCaches::every(cache))) + } + /// Returns a clone of this instance with the template cache replaced. #[must_use] pub fn with_template_cache(self, cache: Arc) -> Self { @@ -333,6 +564,8 @@ pub struct RuntimeServicesBuilder { config_store: Option>, secret_store: Option>, kv_store: Option>, + value_cache: Option>, + loading_caches: Option>, template_cache: Option>, template_assembler: Option>, backend: Option>, @@ -340,6 +573,11 @@ pub struct RuntimeServicesBuilder { geo: Option>, auction_telemetry_sink: Option>, client_info: Option, + host_signals: Option>, + resolved_ec_module: Option>, + device_module: Option>, + prefixes: Option, + permission_signal_modules: Arc<[Arc]>, } impl RuntimeServicesBuilder { @@ -348,6 +586,8 @@ impl RuntimeServicesBuilder { config_store: None, secret_store: None, kv_store: None, + value_cache: None, + loading_caches: None, template_cache: None, template_assembler: None, backend: None, @@ -355,6 +595,11 @@ impl RuntimeServicesBuilder { geo: None, auction_telemetry_sink: None, client_info: None, + host_signals: None, + resolved_ec_module: None, + device_module: None, + prefixes: None, + permission_signal_modules: Arc::default(), } } @@ -372,6 +617,20 @@ impl RuntimeServicesBuilder { self } + /// Set the host's shared cache of small values. + #[must_use] + pub fn value_cache(mut self, cache: Arc) -> Self { + self.value_cache = Some(cache); + self + } + + /// Set the loading cache each `[cache.]` table selected. + #[must_use] + pub fn loading_caches(mut self, caches: Arc) -> Self { + self.loading_caches = Some(caches); + self + } + /// Set the shared transformed-template cache. #[must_use] pub fn template_cache(mut self, cache: Arc) -> Self { @@ -434,6 +693,61 @@ impl RuntimeServicesBuilder { self } + /// Set the host-computed client signals service. + /// + /// Optional: a host that exposes no TLS or HTTP/2 signals leaves this + /// unset, so a module that requires them cannot be built and the request + /// stops. + #[must_use] + pub fn host_signals(mut self, host_signals: Arc) -> Self { + self.host_signals = Some(host_signals); + self + } + + /// Set the Edge Cookie module the composition root already resolved. + /// + /// Optional. This is the module the selector actually chose, so setting + /// it keeps the request path from resolving the same settings a second + /// time. It is the single seam through which a vendor or host Edge Cookie + /// module reaches the request path. + #[must_use] + pub fn resolved_ec_module(mut self, module: Arc) -> Self { + self.resolved_ec_module = Some(module); + self + } + + /// Set the permission signal modules this deployment runs, in the order + /// they are asked. + /// + /// Optional, and empty when unset. An adapter hands in the shared list + /// [`build_permission_signal_modules`] selected from the scheme crates it + /// links, so the request path asks exactly the modules configuration + /// named, in that order, and core supplies none of its own. + /// + /// [`build_permission_signal_modules`]: + /// crate::permission_signal::build_permission_signal_modules + #[must_use] + pub fn permission_signal_modules( + mut self, + modules: Arc<[Arc]>, + ) -> Self { + self.permission_signal_modules = modules; + self + } + + /// Set the prefixes of the publisher the request was resolved to. + /// + /// Optional in the builder, because services built for no request, such + /// as a test's, have no publisher to resolve, and + /// [`Prefixes::default`](crate::prefix::Prefixes::default), carrying no + /// publisher domain, stands when it is not set. Every adapter sets it on + /// the services of a request. + #[must_use] + pub fn prefixes(mut self, prefixes: crate::prefix::Prefixes) -> Self { + self.prefixes = Some(prefixes); + self + } + /// Construct [`RuntimeServices`] from the accumulated configuration. /// /// # Panics @@ -451,6 +765,10 @@ impl RuntimeServicesBuilder { kv_store: self .kv_store .expect("should set kv_store before building RuntimeServices"), + value_cache: self.value_cache, + // Defaulted rather than required: a deployment that links no + // module supplying one asks each source itself. + loading_caches: self.loading_caches.unwrap_or_default(), // Defaulted rather than required: an adapter with no template cache // should degrade to transforming per request, not fail to build. template_cache: self @@ -474,6 +792,11 @@ impl RuntimeServicesBuilder { client_info: self .client_info .expect("should set client_info before building RuntimeServices"), + host_signals: self.host_signals, + resolved_ec_module: self.resolved_ec_module, + device_module: self.device_module, + prefixes: self.prefixes.unwrap_or_default(), + permission_signal_modules: self.permission_signal_modules, } } } diff --git a/crates/trusted-server-core/src/platform/value_cache.rs b/crates/trusted-server-core/src/platform/value_cache.rs new file mode 100644 index 0000000000..0e617b77fb --- /dev/null +++ b/crates/trusted-server-core/src/platform/value_cache.rs @@ -0,0 +1,80 @@ +//! A shared cache of small values with lifetimes, which a host supplies when it +//! has one. +//! +//! A module that pays for a remote answer holds it here, so the next +//! equivalent request, or a second lookup in the same request, is answered +//! without paying again. An entry can go at any time, which costs the caller one +//! fetch and never an error, so nothing durable belongs here. + +use std::time::Duration; + +use bytes::Bytes; + +/// Values keyed by text, shared by every request the host serves from the same +/// place. The caller owns its key space, so a key starts with a prefix naming +/// the module that wrote it. +/// +/// `Send + Sync` on the trait and `?Send` on the futures, as for the other +/// platform services, because [`super::RuntimeServices`] crosses threads while +/// the platform layer's futures never do. +#[async_trait::async_trait(?Send)] +pub trait PlatformValueCache: Send + Sync { + /// The value held under `key`, or `None` when there is none or it cannot be + /// read. + async fn get(&self, key: &str) -> Option; + + /// Holds `value` under `key` for `lifetime`. A failure is the host's to + /// log and is not returned, because the caller's only cost is fetching + /// again. + async fn put(&self, key: &str, value: Bytes, lifetime: Duration); + + /// The value held under `key`, or, when there is none and the host can + /// make other callers for the key wait, a [`ValueReservation`] obliging + /// this caller to fill it. + /// + /// The default never reserves, for a host whose cache cannot make callers + /// wait. + async fn lookup_or_reserve(&self, key: &str) -> ValueLookup { + match self.get(key).await { + Some(value) => ValueLookup::Hit(value), + None => ValueLookup::Miss, + } + } + + /// Removes the value held under `key`. The default does nothing, for a + /// host whose cache cannot remove an entry before its lifetime ends. + async fn remove(&self, _key: &str) {} +} + +/// What [`PlatformValueCache::lookup_or_reserve`] found. +pub enum ValueLookup { + /// The value held for the key. + Hit(Bytes), + /// Nothing held, and this caller is the one to fill the key. The host + /// makes other callers for the key wait until the reservation is filled + /// or dropped. + Reserved(Box), + /// Nothing held, and no reservation. + Miss, +} + +impl std::fmt::Debug for ValueLookup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Hit(value) => f.debug_tuple("Hit").field(&value.len()).finish(), + Self::Reserved(_) => f.write_str("Reserved"), + Self::Miss => f.write_str("Miss"), + } + } +} + +/// The obligation to fill a key that other callers are waiting for. +/// +/// Dropping it unfilled releases the callers waiting on it, and one of them +/// is given the obligation, so a failed or canceled load never leaves them +/// waiting. +pub trait ValueReservation { + /// Holds `value` for `lifetime` and releases the callers waiting on it. A + /// failure is the host's to log, as for [`PlatformValueCache::put`]. + fn fill(self: Box, value: Bytes, lifetime: Duration); +} diff --git a/crates/trusted-server-core/src/prefix.rs b/crates/trusted-server-core/src/prefix.rs new file mode 100644 index 0000000000..357b442c85 --- /dev/null +++ b/crates/trusted-server-core/src/prefix.rs @@ -0,0 +1,456 @@ +//! The start of every name the browser holds for a publisher, and of every key +//! the publisher's entries take in a shared store. +//! +//! A deployment selects the module that gives both with `[prefix] module`. Core +//! holds the extension point, [`PrefixModule`], and its own module, +//! [`TrustedServerPrefix`], which gives `ts` and `ts.` and runs when +//! nothing is selected. +//! +//! The two prefixes answer different questions. A public prefix starts the +//! names the browser holds, being cookies and first-party storage keys, so it +//! carries no domain, because a browser already keeps each site's apart from +//! every other site's. A server key prefix starts every key in a store +//! publishers share, so it carries the publisher's domain, see +//! [`ServerKeyPrefix`]. +//! +//! The template cache is the exception. Its surrogate keys, `ts-template` and +//! `ts-template-url-`, and its store key, +//! `ts-template-cache-v-`, keep `ts` under every module, so +//! `ts cache purge` names one key whatever the deployment. The store key's +//! digest covers the request's host and the public prefix, so no two +//! publishers share an entry. +//! +//! A module gives the parts, and each use writes its own form from them, +//! because what is allowed differs by use: +//! +//! | Use | Form | `ts` | `_ex`, `www.example.com` | +//! | --- | --- | --- | --- | +//! | cookie | [`PublicPrefix::cookie_name`] | `ts-ec` | `_ex.www.example.com-ec` | +//! | store key | [`ServerKeyPrefix::as_str`] | `ts.example.com` | as its module gives it | +//! +//! The page's client script is told the public prefix on its bundle tag rather +//! than holding a copy, see [`PUBLIC_PREFIX_ATTRIBUTE`]. Its own globals, such +//! as `window.tsjs`, are what page code calls it by and keep their names under +//! every module. + +use core::fmt; +use std::sync::Arc; + +use crate::module_context::{ModuleCall, ModuleContext, ModuleRequest, Withheld}; +use crate::permissions::PermissionSet; +use crate::platform::ServerKeyPrefix; +use crate::settings::Settings; + +/// The type folder, and the section, that selects the prefix module. +pub const MODULE_TYPE: &str = "prefix"; + +/// The name core's own module is called by when it runs. +pub const TRUSTED_SERVER_MODULE: &str = "ts"; + +/// The head of every name core's own module gives. +const TRUSTED_SERVER_HEAD: &str = "ts"; + +/// The bundle tag attribute that tells the client script the public prefix, +/// so the script writes the deployment's names and holds no copy of its own. +pub const PUBLIC_PREFIX_ATTRIBUTE: &str = "data-ts-public-prefix"; + +/// The start of every name the browser holds for a publisher, such as `_ex`. +/// +/// Made of parts, such as `_ex` and `www.example.com`, joined by `.`. Every +/// part holds only ASCII letters, digits, `_`, `-` and `.`, so the prefix is a +/// valid cookie name and attribute value. The first part starts with a letter +/// or a single `_`, so no cookie name starts with `__`, where browsers give +/// names such as `__Host-` rules of their own and drop a cookie that breaks +/// them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicPrefix { + dotted: String, +} + +impl PublicPrefix { + /// The prefix made of `parts`, in order, or `None` when there are none, + /// when a part is empty or holds anything else, or when the first part + /// starts with anything but a letter or a single `_`. + #[must_use] + pub fn new(parts: I) -> Option + where + I: IntoIterator, + S: AsRef, + { + let parts: Vec = parts + .into_iter() + .map(|part| part.as_ref().to_owned()) + .collect(); + let well_formed = parts.iter().all(|part| { + !part.is_empty() + && part + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.')) + }); + let first_is_a_name = parts.first().is_some_and(|first| { + first.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') + && !first.starts_with("__") + }); + if !well_formed || !first_is_a_name { + return None; + } + Some(Self { + dotted: parts.join("."), + }) + } + + /// The parts joined by `.`, such as `_ex.www.example.com`, which every + /// cookie name and storage key starts with. + #[must_use] + pub fn as_str(&self) -> &str { + &self.dotted + } + + /// The name of the cookie `name`, written `-`, + /// such as `ts-ec` or `_ex-ec`. + #[must_use] + pub fn cookie_name(&self, name: &str) -> String { + format!("{}-{name}", self.dotted) + } + + /// Whether `cookie` is in the namespace this prefix starts, which is every + /// name [`Self::cookie_name`] gives, compared without regard to case. + #[must_use] + pub fn owns_cookie_name(&self, cookie: &[u8]) -> bool { + let length = self.dotted.len(); + cookie + .get(..length) + .is_some_and(|head| head.eq_ignore_ascii_case(self.dotted.as_bytes())) + && cookie.get(length) == Some(&b'-') + } +} + +/// Core's own, `ts`, for a value no request resolved, such as a default +/// Edge Cookie state. +impl Default for PublicPrefix { + fn default() -> Self { + TrustedServerPrefix::public() + } +} + +impl fmt::Display for PublicPrefix { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.dotted) + } +} + +/// Gives the start of the names the browser holds for a publisher and of the +/// keys its entries take in shared stores. +/// +/// Selected with `[prefix] module`, and core's [`TrustedServerPrefix`] runs +/// when nothing is selected. Each method is handed a [`ModuleCall`] carrying +/// the request and the deployment's settings, and +/// nothing else, and an implementation hands its own function to +/// [`ModuleCall::inject`] naming what it needs. +/// +/// The domain a server key prefix carries comes from the publisher the request +/// resolved to, being the loaded settings' `publisher.domain`, and never from +/// the raw request. +pub trait PrefixModule: Send + Sync + fmt::Debug { + /// The start of every cookie name and first-party storage key for the + /// publisher this call was resolved to. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value the call does not carry. + fn public_prefix(&self, call: ModuleCall<'_>) -> Result; + + /// The start of every key the publisher's entries take in a shared store. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value the call does not carry. + fn server_key_prefix(&self, call: ModuleCall<'_>) -> Result; +} + +/// Core's own prefix module, which gives `ts`, the prefix of the cookie names +/// Trusted Server writes, and `ts.`. It runs when `[prefix]` +/// selects nothing. +#[derive(Debug, Default, Clone, Copy)] +pub struct TrustedServerPrefix; + +impl TrustedServerPrefix { + /// `ts`. + #[must_use] + pub fn public() -> PublicPrefix { + PublicPrefix { + dotted: TRUSTED_SERVER_HEAD.to_owned(), + } + } + + fn server_key(&self, settings: &Settings) -> ServerKeyPrefix { + ServerKeyPrefix::new(TRUSTED_SERVER_HEAD, &settings.publisher.domain) + } +} + +impl PrefixModule for TrustedServerPrefix { + fn public_prefix(&self, _call: ModuleCall<'_>) -> Result { + Ok(Self::public()) + } + + fn server_key_prefix(&self, call: ModuleCall<'_>) -> Result { + call.inject(self, Self::server_key) + } +} + +/// Both prefixes for one request, as its services carry them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prefixes { + public: PublicPrefix, + server_key: ServerKeyPrefix, +} + +impl Prefixes { + /// The two prefixes. + #[must_use] + pub fn new(public: PublicPrefix, server_key: ServerKeyPrefix) -> Self { + Self { public, server_key } + } + + /// What `module`, called `name`, gives for the publisher `request` was + /// resolved to under `settings`. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value other than the request and + /// the settings. + pub fn resolve( + module: &dyn PrefixModule, + name: &'static str, + settings: &Settings, + request: ModuleRequest<'_>, + ) -> Result { + let context = ModuleContext::new(request).with_settings(settings); + let call = context.call(name, PermissionSet::none()); + Ok(Self { + public: module.public_prefix(call)?, + server_key: module.server_key_prefix(call)?, + }) + } + + /// What core's own module gives for the publisher `settings` describe. + #[must_use] + pub fn trusted_server(settings: &Settings) -> Self { + Self::new( + TrustedServerPrefix::public(), + TrustedServerPrefix.server_key(settings), + ) + } + + /// The start of every name the browser holds. + #[must_use] + pub fn public(&self) -> &PublicPrefix { + &self.public + } + + /// The start of every key in a shared store. + #[must_use] + pub fn server_key(&self) -> &ServerKeyPrefix { + &self.server_key + } +} + +/// The prefixes of services built for no request, such as a test's or the +/// operator command line's, being core's own names with no publisher domain. +impl Default for Prefixes { + fn default() -> Self { + Self::new( + TrustedServerPrefix::public(), + ServerKeyPrefix::new(TRUSTED_SERVER_HEAD, ""), + ) + } +} + +/// The selected prefix module and the name it runs under. +#[derive(Debug, Clone)] +pub struct SelectedPrefixModule { + name: &'static str, + module: Arc, +} + +impl SelectedPrefixModule { + /// `module`, which runs under `name`. + #[must_use] + pub fn new(name: &'static str, module: Arc) -> Self { + Self { name, module } + } + + /// Core's own module, which runs when nothing is selected. + #[must_use] + pub fn trusted_server() -> Self { + Self::new(TRUSTED_SERVER_MODULE, Arc::new(TrustedServerPrefix)) + } + + /// The name the module runs under. + #[must_use] + pub fn name(&self) -> &'static str { + self.name + } + + /// What the module gives for the publisher `request` was resolved to. + /// + /// # Errors + /// + /// [`Withheld`] when the module names a value other than the request and + /// the settings. + pub fn prefixes( + &self, + settings: &Settings, + request: ModuleRequest<'_>, + ) -> Result { + Prefixes::resolve(&*self.module, self.name, settings, request) + } +} + +impl Default for SelectedPrefixModule { + fn default() -> Self { + Self::trusted_server() + } +} + +/// Prefixes and checks for a module's own tests of its store keys. +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support { + use super::{Prefixes, ServerKeyPrefix, TrustedServerPrefix}; + + /// The prefixes core's own module gives a publisher on `domain`. + #[must_use] + pub fn trusted_server_for(domain: &str) -> Prefixes { + Prefixes::new( + TrustedServerPrefix::public(), + ServerKeyPrefix::new(super::TRUSTED_SERVER_HEAD, domain), + ) + } + + /// Checks that the keys `key_for` makes for a publisher's domain follow + /// the publisher: two publishers never share one, `www.example.com` and + /// `example.com` do, and `clients.example.com` and `public.example.com` + /// do not. + /// + /// # Panics + /// + /// When any of the three does not hold, naming it. + pub fn assert_keys_follow_the_publisher(key_for: impl Fn(&str) -> String) { + assert_ne!( + key_for("first.example"), + key_for("second.example"), + "two publishers should never share a key" + ); + assert_eq!( + key_for("www.example.com"), + key_for("example.com"), + "a publisher's www. and bare names should share a key" + ); + assert_ne!( + key_for("clients.example.com"), + key_for("public.example.com"), + "two subdomains other than www. should not share a key" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::module_context::test_support::request; + + fn settings_for(domain: &str) -> Settings { + let mut settings = Settings::default(); + settings.publisher.domain = domain.to_owned(); + settings + } + + #[test] + fn no_module_selected_gives_exactly_ts() { + let prefixes = SelectedPrefixModule::default() + .prefixes(&settings_for("www.example.com"), request("/")) + .expect("core's module should name only what the call carries"); + + assert_eq!(prefixes.public().as_str(), "ts"); + assert_eq!(prefixes.server_key().as_str(), "ts.example.com"); + for (name, cookie) in [ + ("ec", "ts-ec"), + ("ecr", "ts-ecr"), + ("eids", "ts-eids"), + ("tester", "ts-tester"), + ] { + assert_eq!(prefixes.public().cookie_name(name), cookie); + } + } + + #[test] + fn each_use_writes_its_own_form_from_the_parts() { + let prefix = + PublicPrefix::new(["_ex", "www.example.com"]).expect("should accept the parts"); + + assert_eq!(prefix.as_str(), "_ex.www.example.com"); + assert_eq!(prefix.cookie_name("ec"), "_ex.www.example.com-ec"); + } + + #[test] + fn a_part_that_could_leave_a_name_is_refused() { + for parts in [ + vec![], + vec![""], + vec!["9ex"], + vec!["-ex"], + vec![".ex"], + vec!["__ex"], + vec!["__Host"], + vec!["_ex", ""], + vec!["_ex", "a b"], + vec!["_ex", "a;b"], + vec!["_ex", "a=b"], + vec!["_ex", "a\"b"], + vec!["_ex", "a/b"], + ] { + assert!(PublicPrefix::new(parts.clone()).is_none(), "{parts:?}"); + } + } + + #[test] + fn a_prefix_owns_the_cookie_names_it_gives_and_no_others() { + let prefix = TrustedServerPrefix::public(); + + for owned in ["ts-ec", "TS-EIDS", "ts-anything", "ts-"] { + assert!(prefix.owns_cookie_name(owned.as_bytes()), "{owned}"); + } + for other in ["ts", "tsx-ec", "_ex-ec", "sharedId"] { + assert!(!prefix.owns_cookie_name(other.as_bytes()), "{other}"); + } + } + + /// The client script reads the prefix from the attribute the server + /// writes, on the bundle tag, so both names are the same in the two + /// places. + #[test] + fn the_client_script_reads_the_attribute_the_server_writes() { + let script = include_str!("../../trusted-server-js/lib/src/shared/public_prefix.ts"); + + assert!( + script.contains(&format!( + "export const PUBLIC_PREFIX_ATTRIBUTE = '{PUBLIC_PREFIX_ATTRIBUTE}'" + )), + "the client script should read the attribute the server writes" + ); + assert!( + script.contains("const BUNDLE_TAG_ID = 'trustedserver-js'") + && crate::tsjs::tsjs_script_tag_with_attributes(&[], &[]) + .contains(r#"id="trustedserver-js""#), + "the client script should read the tag the server writes" + ); + } + + #[test] + fn the_default_carries_no_publisher_domain() { + let prefixes = Prefixes::default(); + + assert_eq!(prefixes.public().as_str(), "ts"); + assert_eq!(prefixes.server_key().domain(), ""); + } +} diff --git a/crates/trusted-server-core/src/provider_table.rs b/crates/trusted-server-core/src/provider_table.rs new file mode 100644 index 0000000000..9715accdc7 --- /dev/null +++ b/crates/trusted-server-core/src/provider_table.rs @@ -0,0 +1,886 @@ +//! The table the auction's `[demand]` and `[ad-server]` share. +//! +//! Each is one top-level table. Its `modules` key, or `module` where one +//! runs, selects what runs, and every other key is one named provider's +//! settings table: +//! +//! ```toml +//! [demand] +//! modules = ["pbs_demo"] +//! +//! [demand.pbs_demo] +//! implementation = "auction.prebid-server" +//! endpoint = "https://prebid.example.com/openrtb2/auction" +//! ``` +//! +//! A table's `implementation` line names the implementation it configures by +//! its module path, which lets several providers share one implementation +//! under names of their own. Without the line the table's name stands for +//! the implementation, which only a section named by the implementation's +//! type can match, as `mock` in `[ad-server]` is `ad-server.mock`. A provider +//! with nothing to set needs no table. [`ProviderTable::validate`] checks the +//! rules that hold for every type, and the code that reads a type checks what +//! only that type knows, such as which implementations exist. + +use std::collections::BTreeMap; +use std::fmt; + +use serde::de::{self, MapAccess, Visitor}; +use serde::ser::SerializeMap as _; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::{Map, Value}; + +/// The line naming the implementation a table configures when the table's +/// name is a label rather than the implementation. +pub const IMPLEMENTATION_KEY: &str = "implementation"; + +/// The longest provider name, in bytes. +const MAX_NAME_BYTES: usize = 63; + +/// A provider type whose `modules` names any number of providers. +pub type ProviderList = ProviderTable>; + +/// A provider type whose `module` names at most one provider. +pub type ProviderChoice = ProviderTable>; + +/// How a type's selection is written. +pub trait ProviderSelection: Default + Clone + PartialEq + fmt::Debug { + /// The key the selection is written under, `module` for a type that runs + /// one and `modules` for a type that runs several. + const KEY: &'static str; + + /// The selected names, in the order they were written. + fn names(&self) -> Vec<&str>; + + /// Reads the selection. + /// + /// # Errors + /// + /// Returns a message when the value has the wrong shape for this type. + fn from_value(value: Value) -> Result; + + /// Writes the selection, or `None` when nothing is selected. + fn to_value(&self) -> Option; +} + +impl ProviderSelection for Vec { + const KEY: &'static str = "modules"; + + fn names(&self) -> Vec<&str> { + self.iter().map(String::as_str).collect() + } + + fn from_value(value: Value) -> Result { + match value { + Value::Array(items) => items + .into_iter() + .map(|item| match item { + Value::String(name) => Ok(name), + other => Err(format!("`modules` must list names, found `{other}`")), + }) + .collect(), + Value::String(name) => Err(format!( + "`modules` is a list for this type, so write [\"{name}\"]" + )), + other => Err(format!( + "`modules` must be a list of names, found `{other}`" + )), + } + } + + fn to_value(&self) -> Option { + (!self.is_empty()).then(|| Value::Array(self.iter().cloned().map(Value::String).collect())) + } +} + +impl ProviderSelection for Option { + const KEY: &'static str = "module"; + + fn names(&self) -> Vec<&str> { + self.iter().map(String::as_str).collect() + } + + fn from_value(value: Value) -> Result { + match value { + Value::String(name) => Ok(Some(name)), + Value::Array(_) => { + Err("`module` names one provider for this type, not a list".to_owned()) + } + other => Err(format!("`module` must be a name, found `{other}`")), + } + } + + fn to_value(&self) -> Option { + self.clone().map(Value::String) + } +} + +/// One provider type's table. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ProviderTable { + selected: S, + tables: BTreeMap>, +} + +impl ProviderTable { + /// Builds a table from its selection and its named settings tables. + #[must_use] + pub fn new(selected: S, tables: BTreeMap>) -> Self { + Self { selected, tables } + } + + /// The selected provider names, in the order they were written. + #[must_use] + pub fn selected(&self) -> Vec<&str> { + self.selected.names() + } + + /// Whether no provider is selected. + #[must_use] + pub fn is_empty(&self) -> bool { + self.selected.names().is_empty() + } + + /// Whether the table sets nothing at all, neither a selection nor any named + /// table, so a serialized configuration can leave it out. + #[must_use] + pub fn is_unset(&self) -> bool { + self.is_empty() && self.tables.is_empty() + } + + /// Every named settings table, selected or not. + #[must_use] + pub fn tables(&self) -> &BTreeMap> { + &self.tables + } + + /// The implementation a selected name uses, being its `implementation` + /// line or else the name itself. + #[must_use] + pub fn implementation_of<'a>(&'a self, name: &'a str) -> &'a str { + self.tables + .get(name) + .and_then(|table| table.get(IMPLEMENTATION_KEY)) + .and_then(Value::as_str) + .unwrap_or(name) + } + + /// A name's settings, without its `implementation` line. A name with no + /// table has no settings. + #[must_use] + pub fn settings_of(&self, name: &str) -> Map { + let mut settings = self.tables.get(name).cloned().unwrap_or_default(); + settings.remove(IMPLEMENTATION_KEY); + settings + } + + /// Checks the rules every provider type shares, naming the type in each + /// message. + /// + /// # Errors + /// + /// Returns a message for a name that is not `snake_case`, a name selected + /// twice, an `implementation` line that is not a module name, or a table + /// the selection does not name. + pub fn validate(&self, type_name: &str) -> Result<(), String> { + let selected = self.selected.names(); + for (position, name) in selected.iter().enumerate() { + check_name(type_name, "module name", name)?; + if selected[..position].contains(name) { + return Err(format!( + "[{type_name}] {} names `{name}` more than once", + S::KEY + )); + } + } + for (name, table) in &self.tables { + check_name(type_name, "table name", name)?; + if let Some(implementation) = table.get(IMPLEMENTATION_KEY) { + let valid = implementation + .as_str() + .is_some_and(crate::module_name::is_valid); + if !valid { + return Err(format!( + "[{type_name}.{name}] implementation must be a module name, its \ + crate's parts joined by `.`, each in lower case letters, digits, \ + `_` or `-`, such as `auction.example`" + )); + } + } + if !selected.contains(&name.as_str()) { + return Err(format!( + "[{type_name}.{name}] is configured, but [{type_name}] {key} does not select `{name}`. Add it to {key}, or remove the table", + key = S::KEY + )); + } + } + Ok(()) + } +} + +/// Checks that a module's name within its type is `snake_case`. +fn check_name(type_name: &str, what: &str, name: &str) -> Result<(), String> { + let mut bytes = name.bytes(); + let valid = name.len() <= MAX_NAME_BYTES + && bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_'); + if valid { + Ok(()) + } else { + Err(format!( + "[{type_name}] {what} `{name}` must be snake_case: a lowercase letter followed by up to {} lowercase letters, digits or underscores", + MAX_NAME_BYTES - 1 + )) + } +} + +impl<'de, S: ProviderSelection> Deserialize<'de> for ProviderTable { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct TableVisitor(core::marker::PhantomData); + + impl<'de, S: ProviderSelection> Visitor<'de> for TableVisitor { + type Value = ProviderTable; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a provider table with its selection key and named tables") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut table = ProviderTable::::default(); + while let Some(key) = map.next_key::()? { + let value = map.next_value::()?; + if key == S::KEY { + table.selected = S::from_value(value).map_err(de::Error::custom)?; + continue; + } + if key == "provider" { + return Err(de::Error::custom(format!( + "`provider` is `{}` here. Name what runs with `{}`", + S::KEY, + S::KEY + ))); + } + match value { + Value::Object(settings) => { + table.tables.insert(key, settings); + } + _ => { + return Err(de::Error::custom(format!( + "`{key}` is not a setting of this table. Only `{}` and named provider tables belong here", + S::KEY + ))); + } + } + } + Ok(table) + } + } + + deserializer.deserialize_map(TableVisitor(core::marker::PhantomData)) + } +} + +impl Serialize for ProviderTable { + fn serialize(&self, serializer: Z) -> Result + where + Z: Serializer, + { + let selection = self.selected.to_value(); + let mut map = + serializer.serialize_map(Some(self.tables.len() + usize::from(selection.is_some())))?; + if let Some(selection) = selection { + map.serialize_entry(S::KEY, &selection)?; + } + for (name, settings) in &self.tables { + map.serialize_entry(name, settings)?; + } + map.end() + } +} + +/// Which key a section's selection was written under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SelectionKey { + Module, + Modules, +} + +impl SelectionKey { + const fn as_str(self) -> &'static str { + match self { + Self::Module => "module", + Self::Modules => "modules", + } + } +} + +/// The modules one section selects, and each one's settings table. +/// +/// `modules` selects several, in the order they run, and `module` one, and a +/// section carries at most one of the two. Every other table in the section is +/// the settings of a selected module, at the name written in the selection +/// with `.` between its parts, so `modules = ["testing.example"]` reads its +/// settings from `[
.testing.example]`. +#[derive(Clone, Default, PartialEq)] +pub struct SectionModules { + key: Option, + selected: Vec, + tables: Map, +} + +impl fmt::Debug for SectionModules { + // The tables can hold secrets, so only the selection and the tables' + // names are shown. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SectionModules") + .field("selected", &self.selected) + .field("tables", &self.tables.keys().collect::>()) + .finish() + } +} + +impl SectionModules { + /// Reads a section's entries, in a section that holds nothing else. + /// + /// # Errors + /// + /// Returns a message for a selection of the wrong shape, for both keys at + /// once, and for an entry that is neither a selection nor a table. + pub fn from_entries(entries: Map) -> Result { + let mut section = Self::default(); + for (key, value) in entries { + match key.as_str() { + "module" | "modules" => { + let key = if key == "module" { + SelectionKey::Module + } else { + SelectionKey::Modules + }; + if section.key.is_some_and(|written| written != key) { + return Err( + "carries both `module` and `modules`. Write `module` for one or \ + `modules` for several" + .to_owned(), + ); + } + section.key = Some(key); + section.selected = match key { + SelectionKey::Module => { + as ProviderSelection>::from_value(value)? + .into_iter() + .collect() + } + SelectionKey::Modules => { + as ProviderSelection>::from_value(value)? + } + }; + } + _ => match value { + Value::Object(_) => { + section.tables.insert(key, value); + } + _ => { + return Err(format!( + "has `{key}`, which is not a setting it reads. Only `module` or \ + `modules` and the settings tables of the modules they select \ + belong there" + )); + } + }, + } + } + Ok(section) + } + + /// The selected names, as written, in the order they run. + #[must_use] + pub fn selected(&self) -> &[String] { + &self.selected + } + + /// Whether the section selects nothing and holds no table. + #[must_use] + pub fn is_unset(&self) -> bool { + self.selected.is_empty() && self.tables.is_empty() + } + + /// The settings table of a name as written in the selection, found by + /// its parts. + #[must_use] + pub fn settings_of(&self, written: &str) -> Option<&Map> { + let mut parts = written.split('.'); + let mut table = self.tables.get(parts.next()?)?.as_object()?; + for part in parts { + table = table.get(part)?.as_object()?; + } + Some(table) + } + + /// The settings table of a name as written in the selection, to change. + pub fn settings_of_mut(&mut self, written: &str) -> Option<&mut Map> { + let mut parts = written.split('.'); + let mut table = self.tables.get_mut(parts.next()?)?.as_object_mut()?; + for part in parts { + table = table.get_mut(part)?.as_object_mut()?; + } + Some(table) + } + + /// Selects `written` and stores its settings table, which is what writing + /// both in a document does. + pub fn insert(&mut self, written: &str, settings: Map) { + if !self.selected.iter().any(|name| name == written) { + self.selected.push(written.to_owned()); + } + if self.key.is_none() { + self.key = Some(SelectionKey::Modules); + } + let mut parts: Vec<&str> = written.split('.').collect(); + let last = parts.pop().unwrap_or(written); + let mut table = &mut self.tables; + for part in parts { + let entry = table + .entry(part.to_owned()) + .or_insert_with(|| Value::Object(Map::new())); + if !entry.is_object() { + *entry = Value::Object(Map::new()); + } + let Value::Object(next) = entry else { + return; + }; + table = next; + } + table.insert(last.to_owned(), Value::Object(settings)); + } + + /// Selects `written` with no settings table of its own. + pub fn select(&mut self, written: &str) { + if !self.selected.iter().any(|name| name == written) { + self.selected.push(written.to_owned()); + } + if self.key.is_none() { + self.key = Some(SelectionKey::Modules); + } + } + + /// Stops selecting `written` and drops its table. + pub fn remove(&mut self, written: &str) { + self.selected.retain(|name| name != written); + let mut parts: Vec<&str> = written.split('.').collect(); + let Some(last) = parts.pop() else { + return; + }; + let mut table = &mut self.tables; + for part in parts { + let Some(Value::Object(next)) = table.get_mut(part) else { + return; + }; + table = next; + } + table.remove(last); + } + + /// Stops selecting everything and drops every table. + pub fn clear(&mut self) { + self.selected.clear(); + self.tables.clear(); + } + + /// Checks the rules every section's selection follows, naming the section. + /// + /// # Errors + /// + /// Returns a message for a name that is not a module name, a name selected + /// twice, and a table that is not the settings of a selected name. + pub fn validate(&self, section: &str) -> Result<(), String> { + let key = self.key.unwrap_or(SelectionKey::Modules).as_str(); + for (position, name) in self.selected.iter().enumerate() { + if !crate::module_name::is_valid(name) { + return Err(format!( + "[{section}] {key} names `{name}`, which is not a module name, being parts \ + joined by `.`, each of lower case letters, digits, `_` or `-`" + )); + } + if self.selected[..position].contains(name) { + return Err(format!("[{section}] {key} names `{name}` more than once")); + } + } + check_tables(section, key, &self.selected, &self.tables, "") + } +} + +/// Checks that every table under `prefix` is a selected name's settings, or +/// lies on the way to one. +fn check_tables( + section: &str, + key: &str, + selected: &[String], + tables: &Map, + prefix: &str, +) -> Result<(), String> { + for (name, value) in tables { + let path = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}.{name}") + }; + if selected.contains(&path) { + continue; + } + let leads_to_one = selected + .iter() + .any(|written| written.starts_with(&format!("{path}."))); + match value { + Value::Object(inner) if leads_to_one => { + check_tables(section, key, selected, inner, &path)?; + } + _ => { + return Err(format!( + "[{section}.{path}] is configured, but [{section}] {key} does not select \ + `{path}`. Add it to {key}, or remove the table" + )); + } + } + } + Ok(()) +} + +impl<'de> Deserialize<'de> for SectionModules { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let entries = Map::::deserialize(deserializer)?; + Self::from_entries(entries) + .map_err(|message| serde::de::Error::custom(format!("the section {message}"))) + } +} + +impl Serialize for SectionModules { + fn serialize(&self, serializer: Z) -> Result + where + Z: Serializer, + { + let selection = + (!self.selected.is_empty()).then(|| match self.key.unwrap_or(SelectionKey::Modules) { + SelectionKey::Module if self.selected.len() == 1 => ( + SelectionKey::Module.as_str(), + Value::String(self.selected[0].clone()), + ), + _ => ( + SelectionKey::Modules.as_str(), + Value::Array(self.selected.iter().cloned().map(Value::String).collect()), + ), + }); + let mut map = + serializer.serialize_map(Some(self.tables.len() + usize::from(selection.is_some())))?; + if let Some((key, value)) = selection { + map.serialize_entry(key, &value)?; + } + for (name, table) in &self.tables { + map.serialize_entry(name, table)?; + } + map.end() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn list(json: Value) -> Result { + serde_json::from_value(json) + } + + #[test] + fn reads_the_selection_and_named_tables() { + let table = list(serde_json::json!({ + "modules": ["main_source", "second_source"], + "main_source": { "implementation": "exchange", "endpoint": "https://exchange.example" } + })) + .expect("should read a module list"); + + assert_eq!(table.selected(), vec!["main_source", "second_source"]); + assert_eq!(table.implementation_of("main_source"), "exchange"); + assert_eq!( + table.implementation_of("second_source"), + "second_source", + "a name with no table should be its own implementation" + ); + assert!( + !table + .settings_of("main_source") + .contains_key(IMPLEMENTATION_KEY), + "settings should not carry the implementation line" + ); + table.validate("demand").expect("should accept the table"); + } + + #[test] + fn rejects_a_table_that_is_not_selected() { + let table = list(serde_json::json!({ + "modules": ["second_source"], + "main_source": { "endpoint": "https://exchange.example" } + })) + .expect("should read the table"); + + let error = table + .validate("demand") + .expect_err("should reject an unselected table"); + assert!( + error.contains("[demand.main_source]") && error.contains("does not select"), + "should name the table and the fix: {error}" + ); + } + + #[test] + fn rejects_a_name_selected_twice() { + let table = list(serde_json::json!({ "modules": ["second_source", "second_source"] })) + .expect("should read the table"); + + let error = table + .validate("demand") + .expect_err("should reject a duplicate"); + assert!(error.contains("more than once"), "should say why: {error}"); + } + + #[test] + fn rejects_names_that_are_not_snake_case() { + for name in ["main-source", "MainSource", "1source", ""] { + let table = list(serde_json::json!({ "modules": [name] })).expect("should read"); + let error = table + .validate("demand") + .expect_err("should reject a name that is not snake_case"); + assert!( + error.contains("snake_case"), + "should say why for `{name}`: {error}" + ); + } + } + + #[test] + fn an_implementation_is_named_by_its_module_path() { + let table = list(serde_json::json!({ + "modules": ["main_source"], + "main_source": { "implementation": "auction.example-exchange" } + })) + .expect("should read the table"); + table + .validate("demand") + .expect("a module path should name an implementation"); + + for implementation in [ + serde_json::json!("Auction.Exchange"), + serde_json::json!("auction/exchange"), + serde_json::json!("auction..exchange"), + serde_json::json!(7), + ] { + let table = list(serde_json::json!({ + "modules": ["main_source"], + "main_source": { "implementation": implementation } + })) + .expect("should read the table"); + let error = table + .validate("demand") + .expect_err("should refuse an implementation that is not a module name"); + assert!( + error.contains("[demand.main_source] implementation must be a module name"), + "should say why for {implementation}: {error}" + ); + } + } + + #[test] + fn rejects_a_value_that_is_not_a_table() { + let error = list(serde_json::json!({ "modules": ["second_source"], "timeout_ms": 500 })) + .expect_err("should reject a stray value"); + assert!( + error.to_string().contains("`timeout_ms` is not a setting"), + "should name the stray key: {error}" + ); + } + + #[test] + fn a_choice_takes_one_name_and_a_list_takes_a_list() { + let choice: ProviderChoice = + serde_json::from_value(serde_json::json!({ "module": "picker" })) + .expect("should read a single name"); + assert_eq!(choice.selected(), vec!["picker"]); + + serde_json::from_value::(serde_json::json!({ "module": ["a"] })) + .expect_err("a single-module type should refuse a list"); + serde_json::from_value::(serde_json::json!({ "modules": "a" })) + .expect_err("a list type should refuse a single name"); + } + + #[test] + fn refuses_the_previous_selection_key() { + let error = list(serde_json::json!({ "provider": ["main_source"] })) + .expect_err("should refuse the key the selection was written under before"); + assert!( + error.to_string().contains("`provider` is `modules` here"), + "the refusal names the key to use: {error}" + ); + + let error = + serde_json::from_value::(serde_json::json!({ "provider": "a" })) + .expect_err("should refuse the previous key for a type that runs one"); + assert!( + error.to_string().contains("`provider` is `module` here"), + "the refusal names the key to use: {error}" + ); + } + + #[test] + fn round_trips_through_serialization() { + let table = list(serde_json::json!({ + "modules": ["main_source"], + "main_source": { "implementation": "exchange", "timeout_ms": 900 } + })) + .expect("should read the table"); + + let value = serde_json::to_value(&table).expect("should serialize"); + let again: ProviderList = serde_json::from_value(value).expect("should read it back"); + assert_eq!(table, again); + } +} + +#[cfg(test)] +mod section_tests { + use super::SectionModules; + use serde_json::{Map, Value, json}; + + fn read(json: Value) -> Result { + serde_json::from_value(json) + } + + #[test] + fn reads_the_selection_and_the_selected_tables() { + let section = read(json!({ + "modules": ["didomi", "sourcepoint"], + "didomi": { "api_key": "k" } + })) + .expect("should read a section"); + assert_eq!(section.selected(), ["didomi", "sourcepoint"]); + assert_eq!( + section.settings_of("didomi").and_then(|t| t.get("api_key")), + Some(&json!("k")) + ); + assert!(section.settings_of("sourcepoint").is_none()); + section.validate("cmp").expect("should pass validation"); + } + + #[test] + fn module_selects_one_and_a_list_there_is_refused() { + let section = read(json!({ "module": "didomi" })).expect("should read one"); + assert_eq!(section.selected(), ["didomi"]); + let error = read(json!({ "module": ["didomi"] })).expect_err("should refuse a list"); + assert!(error.to_string().contains("`module` names one"), "{error}"); + } + + #[test] + fn both_keys_at_once_are_refused() { + let error = + read(json!({ "module": "a", "modules": ["b"] })).expect_err("should refuse both keys"); + assert!( + error + .to_string() + .contains("carries both `module` and `modules`"), + "{error}" + ); + } + + #[test] + fn a_value_that_is_neither_selection_nor_table_is_refused() { + let error = read(json!({ "modules": ["a"], "enabled": true })) + .expect_err("should refuse a stray value"); + assert!( + error + .to_string() + .contains("has `enabled`, which is not a setting it reads"), + "{error}" + ); + } + + #[test] + fn a_table_no_selection_names_is_refused_by_validation() { + let section = read(json!({ "modules": ["a"], "b": {} })).expect("should read"); + let error = section + .validate("cmp") + .expect_err("should refuse the stray table"); + assert_eq!( + error, + "[cmp.b] is configured, but [cmp] modules does not select `b`. Add it to modules, or remove the table" + ); + } + + #[test] + fn a_name_written_with_dots_reads_a_nested_table() { + let section = read(json!({ + "modules": ["google", "google.diagnostics"], + "google": { "timeout_ms": 5, "diagnostics": { "overlay": true } } + })) + .expect("should read"); + section + .validate("ad-tag") + .expect("the nested table is a selected name's"); + assert_eq!( + section + .settings_of("google.diagnostics") + .and_then(|t| t.get("overlay")), + Some(&json!(true)) + ); + } + + #[test] + fn a_name_that_is_not_a_module_name_or_is_repeated_is_refused() { + let section = read(json!({ "modules": ["Didomi"] })).expect("should read"); + let error = section.validate("cmp").expect_err("should refuse the name"); + assert!(error.contains("is not a module name"), "{error}"); + let section = read(json!({ "modules": ["a", "a"] })).expect("should read"); + let error = section + .validate("cmp") + .expect_err("should refuse the repeat"); + assert_eq!(error, "[cmp] modules names `a` more than once"); + } + + #[test] + fn insert_select_remove_and_clear_keep_the_selection_and_tables_together() { + let mut section = SectionModules::default(); + assert!(section.is_unset()); + let mut settings = Map::new(); + settings.insert("api_key".to_owned(), json!("k")); + section.insert("didomi", settings); + section.select("osano"); + assert_eq!(section.selected(), ["didomi", "osano"]); + assert!(section.settings_of("didomi").is_some()); + section.remove("didomi"); + assert_eq!(section.selected(), ["osano"]); + assert!(section.settings_of("didomi").is_none()); + section.clear(); + assert!(section.is_unset()); + } + + #[test] + fn round_trips_through_serialization() { + let section = + read(json!({ "module": "didomi", "didomi": { "api_key": "k" } })).expect("should read"); + let value = serde_json::to_value(§ion).expect("should serialize"); + assert_eq!( + value, + json!({ "module": "didomi", "didomi": { "api_key": "k" } }) + ); + let several = read(json!({ "modules": ["a", "b"] })).expect("should read"); + assert_eq!( + serde_json::to_value(&several).expect("should serialize"), + json!({ "modules": ["a", "b"] }) + ); + } +} diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 6a30645aba..e7160087a3 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -21,7 +21,7 @@ use crate::constants::{ HEADER_USER_AGENT, HEADER_X_FORWARDED_FOR, }; use crate::creative::{CreativeCssProcessor, CreativeHtmlProcessor}; -use crate::edge_cookie::get_ec_id; +use crate::edge_cookie::recognized_ec_id; use crate::error::TrustedServerError; use crate::platform::{ DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformResponse, @@ -107,7 +107,10 @@ const ASSET_PROXY_FORWARD_HEADERS: [header::HeaderName; 11] = [ /// All three take effect against the serving origin, so leaving them in place /// would let an upstream write first-party cookies, pin HSTS, or clear the /// publisher's site data. -pub(crate) const FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS: [&str; 3] = +/// +/// Public so an integration crate that passes an upstream body through strips +/// the same three. +pub const FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS: [&str; 3] = ["set-cookie", "strict-transport-security", "clear-site-data"]; /// Cache-control value used when asset proxy responses must not be stored. @@ -818,7 +821,7 @@ pub async fn proxy_request( })?; if forward_ec_id { - append_ec_id(&req, &mut target_url_parsed); + append_ec_id(settings, services, &req, &mut target_url_parsed); } proxy_with_redirects( @@ -1221,8 +1224,19 @@ fn upsert_ec_query_param(url: &mut url::Url, ec_id: &str) { url.set_query(Some(&serializer.finish())); } -fn append_ec_id(req: &Request, target_url_parsed: &mut url::Url) { - let ec_id_param = match get_ec_id(req) { +/// Forwards the request's Edge Cookie identifier to the outbound target URL. +/// +/// Only an identifier the selected module recognizes is forwarded. A value +/// carrying another deployment's module code, and any value at all in a +/// stateless deployment, is withheld, so nothing this deployment did not issue +/// reaches the origin. +fn append_ec_id( + settings: &Settings, + services: &RuntimeServices, + req: &Request, + target_url_parsed: &mut url::Url, +) { + let ec_id_param = match recognized_ec_id(settings, services, req) { Ok(id) => id, Err(e) => { log::warn!("failed to extract EC ID for forwarding: {:?}", e); @@ -1259,7 +1273,11 @@ fn is_host_permitted>(allowed_domains: &[S], host: &str) -> bool { /// /// Comparison is case-insensitive. The wildcard check requires a dot boundary, /// so `"*.example.com"` does **not** match `"evil-example.com"`. -pub(crate) fn is_host_allowed(host: &str, pattern: &str) -> bool { +/// +/// Public so an integration crate matches a host against an operator's +/// allowed domains the way the proxy does. +#[must_use] +pub fn is_host_allowed(host: &str, pattern: &str) -> bool { let host = host.to_ascii_lowercase(); let pattern = pattern.to_ascii_lowercase(); @@ -1615,7 +1633,7 @@ pub async fn handle_first_party_proxy( /// Returns an error if the signed target cannot be reconstructed or validation fails. pub async fn handle_first_party_click( settings: &Settings, - _services: &RuntimeServices, + services: &RuntimeServices, req: Request, ) -> Result, Report> { let SignedTarget { @@ -1624,7 +1642,10 @@ pub async fn handle_first_party_click( had_params, } = reconstruct_and_validate_signed_target(settings, &req.uri().to_string())?; - let ec_id = match get_ec_id(&req) { + // The redirect target is a third party's URL, so only an identifier the + // selected module recognizes is added to it. A stateless deployment adds + // nothing. + let ec_id = match recognized_ec_id(settings, services, &req) { Ok(id) => id, Err(e) => { log::warn!("failed to extract EC ID for forwarding: {:?}", e); @@ -2238,16 +2259,17 @@ mod tests { use super::{ AssetProxyCachePolicy, IMAGE_FALLBACK_CONTENT_TYPE, ProxyRequestConfig, - SUPPORTED_ENCODINGS, asset_origin_host_header, asset_path_skips_image_optimizer, - build_asset_proxy_target_url, handle_asset_proxy_request, handle_first_party_click, - handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, - is_host_allowed, is_host_permitted, proxy_request, rebuild_response_with_body, - reconstruct_and_validate_signed_target, stream_asset_body, + SUPPORTED_ENCODINGS, append_ec_id, asset_origin_host_header, + asset_path_skips_image_optimizer, build_asset_proxy_target_url, handle_asset_proxy_request, + handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, + handle_first_party_proxy_sign, is_host_allowed, is_host_permitted, proxy_request, + rebuild_response_with_body, reconstruct_and_validate_signed_target, stream_asset_body, }; use crate::cache_policy::{CachePolicy, EdgeCacheHeader}; use crate::constants::{HEADER_ACCEPT, HEADER_X_FORWARDED_FOR}; use crate::creative; use crate::error::{IntoHttpResponse, TrustedServerError}; + use crate::platform::RuntimeServices; use crate::platform::test_support::{ HashMapSecretStore, StubHttpClient, build_services_with_http_client, build_services_with_secret_and_http_client, noop_services, @@ -2267,6 +2289,7 @@ mod tests { use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::response_builder as edge_response_builder; use error_stack::Report; + use http::Request; use http::{HeaderValue, Method, Request as HttpRequest, Response, StatusCode, header}; #[test] @@ -2989,7 +3012,8 @@ mod tests { ); req.headers_mut().insert( crate::constants::HEADER_X_TS_EC, - HeaderValue::from_static("ec-123"), + HeaderValue::from_str(&recognized_hmac_ec_id()) + .expect("should build EC header value"), ); let resp = handle_first_party_click(&settings, &noop_services(), req) @@ -3007,11 +3031,166 @@ mod tests { .map(|(k, v)| (k.into_owned(), v.into_owned())) .collect(); assert_eq!(pairs.remove("foo").as_deref(), Some("1")); - assert_eq!(pairs.remove("ts-ec").as_deref(), Some("ec-123")); + assert_eq!( + pairs.remove("ts-ec").as_deref(), + Some(recognized_hmac_ec_id().as_str()) + ); assert!(pairs.is_empty()); }); } + /// An identifier the built-in HMAC module, the module + /// `create_test_settings` selects, recognizes as its own. + fn recognized_hmac_ec_id() -> String { + format!("hmac~{}", crate::test_support::tests::VALID_SYNTHETIC_ID) + } + + /// A well-formed identifier carrying a module code no deployment here + /// reads, the shape a partner or another deployment would hand back. + const FOREIGN_CODED_EC_ID: &str = "zz00~someone-elses-identifier"; + + fn stateless_settings() -> Settings { + let mut settings = create_test_settings(); + settings.ec.module = None; + settings.ec.module_blocks.clear(); + settings + } + + fn signed_click_request(settings: &Settings, ec_id: &str) -> Request { + let tsurl = "https://cdn.example/a.png"; + let full = format!("{tsurl}?foo=1"); + let sig = crate::http_util::compute_encrypted_sha256_token(settings, &full); + let mut req = build_http_request( + Method::GET, + format!( + "https://edge.example/first-party/click?tsurl={}&foo=1&tstoken={}", + url::form_urlencoded::byte_serialize(tsurl.as_bytes()).collect::(), + sig + ), + ); + req.headers_mut().insert( + crate::constants::HEADER_X_TS_EC, + HeaderValue::from_str(ec_id).expect("should build EC header value"), + ); + req + } + + fn click_ts_ec_param( + settings: &Settings, + services: &RuntimeServices, + req: Request, + ) -> Option { + let resp = futures::executor::block_on(handle_first_party_click(settings, services, req)) + .expect("should redirect"); + let loc = resp + .headers() + .get(header::LOCATION) + .and_then(|h| h.to_str().ok()) + .expect("Location header should be present and valid") + .to_owned(); + url::Url::parse(&loc) + .expect("Location should be a valid URL") + .query_pairs() + .find(|(k, _)| k == "ts-ec") + .map(|(_, v)| v.into_owned()) + } + + #[test] + fn click_withholds_an_ec_id_the_module_does_not_recognize() { + let settings = create_test_settings(); + let param = click_ts_ec_param( + &settings, + &noop_services(), + signed_click_request(&settings, FOREIGN_CODED_EC_ID), + ); + + assert_eq!( + param, None, + "a value carrying another deployment's module code should not reach the click target" + ); + } + + #[test] + fn click_withholds_every_ec_id_in_a_stateless_deployment() { + let settings = stateless_settings(); + // The value is one the built-in module would recognize, so only the + // absence of a selected module can withhold it. + let param = click_ts_ec_param( + &settings, + &noop_services(), + signed_click_request(&settings, &recognized_hmac_ec_id()), + ); + + assert_eq!( + param, None, + "a deployment that creates no identifier should hand none to the click target" + ); + } + + #[test] + fn append_ec_id_forwards_only_what_the_module_recognizes() { + let settings = create_test_settings(); + let services = noop_services(); + let recognized = recognized_hmac_ec_id(); + + let mut url = + url::Url::parse("https://origin.example/page?foo=1").expect("should parse origin URL"); + append_ec_id( + &settings, + &services, + &request_with_ec_cookie(&recognized), + &mut url, + ); + assert_eq!( + ts_ec_param(&url), + Some(recognized.clone()), + "the deployment's own identifier should still reach the origin" + ); + + let mut url = + url::Url::parse("https://origin.example/page?foo=1").expect("should parse origin URL"); + append_ec_id( + &settings, + &services, + &request_with_ec_cookie(FOREIGN_CODED_EC_ID), + &mut url, + ); + assert_eq!( + ts_ec_param(&url), + None, + "a foreign module code should not reach the origin" + ); + + let mut url = + url::Url::parse("https://origin.example/page?foo=1").expect("should parse origin URL"); + append_ec_id( + &stateless_settings(), + &services, + &request_with_ec_cookie(&recognized), + &mut url, + ); + assert_eq!( + ts_ec_param(&url), + None, + "a stateless deployment should forward nothing to the origin" + ); + } + + fn request_with_ec_cookie(ec_id: &str) -> Request { + let mut req = build_http_request(Method::GET, "https://edge.example/page"); + req.headers_mut().insert( + http::header::COOKIE, + HeaderValue::from_str(&format!("ts-ec={ec_id}")).expect("should build cookie header"), + ); + req + } + + fn ts_ec_param(url: &url::Url) -> Option { + url.query_pairs() + .find(|(k, _)| k == "ts-ec") + .map(|(_, v)| v.into_owned()) + } + #[test] fn proxy_rebuild_adds_and_removes_params() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 682ecc0f15..b042715b2d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -51,13 +51,13 @@ use crate::auction::telemetry::{ emit_auction_events_best_effort_lazy, }; use crate::auction::types::{ - AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, + AuctionContext, AuctionRequest, Bid, BidRenderer, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, }; use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_permissions}; +use crate::constants::{COOKIE_EIDS, COOKIE_SHAREDID, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::cookies::template_cache_policy::{TemplateCookieDecision, evaluate_cookie_policy}; use crate::creative_opportunities::{ @@ -69,8 +69,16 @@ use crate::ec::registry::PartnerRegistry; use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; -use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::IntegrationRegistry; +use crate::html_processor::ServePass; +use crate::http_util::{ + RequestInfo, is_navigation_request, remove_deployment_headers, serve_static_with_etag, +}; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry, IntegrationRequestState}; +use crate::middleware::{ + MiddlewarePhase, Prepared, Refusal, ResponseHeader, media_type_of, write_response_headers, +}; +use crate::module_context::{ModuleContext, ModuleRequest}; +use crate::permissions::PermissionState; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, VarySpec, contains_publisher_esi_directive, @@ -526,8 +534,11 @@ fn encode_complete_body( /// Serves two types of bundles: /// - **Unified bundle** (`tsjs-unified.min.js`): core + immediate (non-deferred) /// integration modules. -/// - **Deferred module** (`tsjs-{id}.min.js`): a single self-contained IIFE for -/// modules loaded with `defer` (e.g., prebid). +/// - **Single module** (`tsjs-{id}.min.js`): a single self-contained IIFE for +/// modules loaded with `defer` (e.g., prebid) or registered as standalone. +/// +/// Every module comes from the registry's parts, so a module a registration +/// carries is served and versioned like a compile-time one. /// /// # Errors /// @@ -548,28 +559,28 @@ pub fn handle_tsjs_dynamic( if UNIFIED_FILENAMES.contains(&filename) { // Serve core + immediate modules (excludes deferred like prebid). - let module_ids = integration_registry.js_module_ids_immediate(); - let body = trusted_server_js::concatenate_modules(&module_ids); - let hash = trusted_server_js::concatenated_hash(&module_ids); + let parts = integration_registry.js_parts_immediate(); + let body = crate::tsjs_bundle::compose(&parts); + let hash = crate::tsjs_bundle::compose_hash(&parts); return Ok(serve_tsjs_static(req, &body, &hash, edge_header)); } - if let Some(module_id) = parse_single_module_filename(filename) { - // Deferred modules and the conditionally injected diagnostics module - // are served as content-addressed standalone assets. Delivery remains - // cookie-independent so the static response can stay publicly cached. - let deferred_ids = integration_registry.js_module_ids_deferred(); - let diagnostics_standalone = module_id - == crate::integrations::gpt_diagnostics::GPT_DIAGNOSTICS_INTEGRATION_ID - && integration_registry.integration_enabled(module_id); - if !deferred_ids.contains(&module_id) && !diagnostics_standalone { + if let Some(module_id) = parse_single_module_filename(filename, integration_registry) { + // Deferred modules and standalone modules are served as + // content-addressed single files. Delivery remains cookie-independent + // so the static response can stay publicly cached. + let deferred = integration_registry.js_module_ids_deferred(); + let standalone = integration_registry.js_standalone_ids(); + if !deferred.contains(&module_id) && !standalone.contains(&module_id) { return Ok(not_found_response()); } - if let (Some(content), Some(hash)) = ( - trusted_server_js::module_bundle(module_id), - trusted_server_js::single_module_hash(module_id), - ) { - return Ok(serve_tsjs_static(req, content, hash, edge_header)); + if let Some(part) = integration_registry.js_part(module_id) { + return Ok(serve_tsjs_static( + req, + part.source, + part.sha256, + edge_header, + )); } } @@ -605,21 +616,22 @@ fn request_version_hash(req: &Request) -> Option<&str> { }) } -/// Extract a module ID from a deferred-module filename like `tsjs-sourcepoint.min.js`. +/// Extract a module ID from a single-module filename like `tsjs-sourcepoint.min.js`. /// -/// Returns `Some(&'static str)` if the filename matches a known JS module ID, -/// `None` otherwise. The caller must additionally verify that the module is -/// both deferred and enabled via the [`IntegrationRegistry`]. +/// Returns `Some(&'static str)` if the filename names a module the registry +/// serves somewhere (bundle, deferred or standalone), resolved through the +/// registry so a carried module is found. `None` otherwise. The caller must +/// additionally verify that the module is deferred or standalone. #[must_use] -fn parse_single_module_filename(filename: &str) -> Option<&'static str> { +fn parse_single_module_filename( + filename: &str, + registry: &IntegrationRegistry, +) -> Option<&'static str> { let stem = filename .strip_prefix("tsjs-") .and_then(|s| s.strip_suffix(".min.js").or_else(|| s.strip_suffix(".js")))?; - trusted_server_js::all_module_ids() - .iter() - .copied() - .find(|&id| id == stem) + registry.js_module_id(stem) } /// Parameters for processing response streaming. @@ -629,22 +641,256 @@ struct ProcessResponseParams<'a> { origin_url: &'a str, request_host: &'a str, request_scheme: &'a str, + /// See [`OwnedProcessResponseParams::request_path`]. + request_path: &'a str, settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + /// Head script carrying this request's permission state, or [`None`] under a + /// shared-template mode. See [`template_permissions_script`]. + permissions_script: Option<&'a str>, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, - suppress_datadome_client_side_tag: bool, - gpt_diagnostics: - Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, + /// See [`HtmlStreamProcessorParams::request_state`]. + request_state: &'a IntegrationRequestState, /// See [`HtmlStreamProcessorParams::shared_template_authorized`]. shared_template_authorized: bool, /// See [`HtmlStreamProcessorParams::csp_nonce_observed`]. csp_nonce_observed: Option<&'a Arc>, + prepared: &'a Arc, + /// See [`OwnedProcessResponseParams::public_prefix`]. + public_prefix: &'a crate::prefix::PublicPrefix, } struct PublisherBodyProcessor { inner: Box, + /// What the serve middleware write on the response, taken by the + /// caller before the response leaves. + response_headers: Vec, +} + +/// A response as the middleware the entries covering it are told of it: +/// any document for its preparation, and one that is not HTML for the +/// stream processors run on its body. +struct TextResponse<'a> { + /// The response's `Content-Type`, parameters and all. + content_type: &'a str, + request_path: &'a str, + request_host: &'a str, + request_scheme: &'a str, + origin_host: &'a str, + request_state: &'a IntegrationRequestState, + max_buffered_script_bytes: usize, + /// What the middleware prepared for this document, see + /// [`Middleware::prepare`](crate::middleware::Middleware::prepare). + prepared: &'a Prepared, +} + +/// The stream processors the entries covering a response that is not HTML +/// run on its body, the fetch entry's and then the serve entry's, with the +/// headers the serve entry's middleware write on the response, or none +/// when its media type is not one a middleware runs on, its charset is not +/// UTF-8 or no entry covers it. Both phases run here because the template +/// cache stores HTML alone. +/// +/// # Errors +/// +/// When a middleware decides about HTML on a response that is not HTML, or +/// asks for a selector that does not parse. +fn middleware_stream_processors( + response: &TextResponse<'_>, + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result> { + use crate::middleware::{ + MiddlewareContext, is_dispatched_media_type, is_utf8_or_undeclared_charset, + }; + + let media_type = media_type_of(response.content_type); + if !is_dispatched_media_type(&media_type) + || !is_utf8_or_undeclared_charset(response.content_type) + { + return Ok(TextPass::default()); + } + let mut processors = Vec::new(); + let mut response_headers = Vec::new(); + for phase in &MiddlewarePhase::ALL { + let chain = integration_registry.middleware_chain( + settings.phase_entries(*phase), + *phase, + &media_type, + response.request_path, + ); + if chain.is_empty() { + continue; + } + let document_state = IntegrationDocumentState::default(); + if *phase == MiddlewarePhase::Serve { + response.request_state.seed(&document_state); + } + let plan = chain + .plan(&MiddlewareContext { + phase: *phase, + request_host: response.request_host, + request_scheme: response.request_scheme, + origin_host: response.origin_host, + platform: integration_registry.platform(), + document_state: &document_state, + max_buffered_script_bytes: response.max_buffered_script_bytes, + prepared: response.prepared, + }) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + processors.extend(plan.processors); + response_headers.extend(plan.response_headers); + } + Ok(TextPass { + processors, + response_headers, + }) +} + +/// What the entries covering a text response do to it: the processors over +/// its body and the headers the serve entry's middleware write. +#[derive(Default)] +struct TextPass { + processors: Vec>, + response_headers: Vec, +} + +/// Runs the preparation of the middleware the entries of each phase in +/// `phases` select for `document`, each as its module with the context the +/// phase is given, keeping what they prepare in the document's [`Prepared`]. +/// Each phase's preparations are handed a fresh document state, not the one +/// the plan is handed, so anything a preparation writes there is dropped. +/// +/// # Errors +/// +/// The first refusal, which ends the request. +async fn prepare_document( + document: &TextResponse<'_>, + phases: &[(MiddlewarePhase, &ModuleContext<'_>)], + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result<(), Refusal> { + use crate::middleware::{MiddlewareContext, is_dispatched_media_type}; + + let media_type = media_type_of(document.content_type); + if !is_dispatched_media_type(&media_type) { + return Ok(()); + } + for (phase, module_context) in phases { + let chain = integration_registry.middleware_chain( + settings.phase_entries(*phase), + *phase, + &media_type, + document.request_path, + ); + if chain.is_empty() { + continue; + } + let document_state = IntegrationDocumentState::default(); + if *phase == MiddlewarePhase::Serve { + document.request_state.seed(&document_state); + } + chain + .prepare( + module_context, + &MiddlewareContext { + phase: *phase, + request_host: document.request_host, + request_scheme: document.request_scheme, + origin_host: document.origin_host, + platform: integration_registry.platform(), + document_state: &document_state, + max_buffered_script_bytes: document.max_buffered_script_bytes, + prepared: document.prepared, + }, + ) + .await?; + } + Ok(()) +} + +/// The context a phase's preparations of a document are called with. The +/// fetch phase of a document that may be stored is handed the request's +/// services and nothing of the reader, because what its preparations answer +/// is stored with the document. Every other preparation, the serve phase's +/// on every document included, is handed the reader's state. +fn preparation_context<'r>( + request: ModuleRequest<'r>, + stored: bool, + settings: &'r Settings, + ec_context: &'r EcContext, + services: &'r RuntimeServices, +) -> ModuleContext<'r> { + let context = ModuleContext::new(request).with_settings(settings); + if stored { + context.with_services(services) + } else { + context.with_request_state(ec_context, services) + } +} + +/// The contexts the two phases of a fetched document are prepared with. The +/// fetch phase's carries the services alone when the document may be stored, +/// because what it answers is stored with the document, and the serve +/// phase's carries the reader's state on every document, because what it +/// answers is this reader's, so the reader a document is fetched for is +/// prepared as every later reader of the stored copy is. +fn preparation_contexts<'r>( + request: impl Fn() -> ModuleRequest<'r>, + stored: bool, + settings: &'r Settings, + ec_context: &'r EcContext, + services: &'r RuntimeServices, +) -> (ModuleContext<'r>, ModuleContext<'r>) { + ( + preparation_context(request(), stored, settings, ec_context, services), + preparation_context(request(), false, settings, ec_context, services), + ) +} + +/// The response a refusal answers the request with: its status, +/// `Retry-After` when it gives one, the message as plain text, and no +/// store. +fn refusal_response(refusal: &Refusal) -> Result, Report> { + let mut builder = Response::builder() + .status(refusal.status) + .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") + .header(header::CACHE_CONTROL, "no-store"); + if let Some(after) = refusal.retry_after { + let seconds = after.as_secs() + u64::from(after.subsec_nanos() > 0); + builder = builder.header(header::RETRY_AFTER, seconds.to_string()); + } + builder + .body(EdgeBody::from(refusal.message.clone().into_bytes())) + .change_context(TrustedServerError::Proxy { + message: "failed to build a middleware's refusal".to_string(), + }) +} + +/// A processor followed by the ones the entries add, each handed what the +/// one before produced. +struct WithStreamProcessors { + inner: Box, + processors: Vec>, +} + +impl StreamProcessor for WithStreamProcessors { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + let mut output = self.inner.process_chunk(chunk, is_last)?; + for processor in &mut self.processors { + output = processor.process_chunk(&output, is_last)?; + } + Ok(output) + } + + fn reset(&mut self) { + self.inner.reset(); + for processor in &mut self.processors { + processor.reset(); + } + } } impl PublisherBodyProcessor { @@ -668,23 +914,29 @@ impl PublisherBodyProcessor { params.ad_slots_script.is_some(), is_html && params.dispatched_auction.is_some(), ); + let mut response_headers = Vec::new(); let inner: Box = if is_html { - Box::new(create_html_stream_processor(HtmlStreamProcessorParams { + let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: ¶ms.origin_host, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, + request_path: ¶ms.request_path, settings, integration_registry, + permissions_script: permissions_script_for(params, settings), ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.clone(), + request_state: params.request_state.clone(), shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.clone(), + prepared: Arc::clone(¶ms.prepared), + public_prefix: ¶ms.public_prefix, deferred_inline_marker: inline_seam_token .as_ref() .map(|token| String::from_utf8_lossy(token).into_owned()), - })?) + })?; + response_headers = headers; + Box::new(processor) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( ¶ms.origin_host, @@ -693,15 +945,46 @@ impl PublisherBodyProcessor { ¶ms.request_scheme, )) } else { - Box::new(create_url_replacer( - ¶ms.origin_host, - ¶ms.origin_url, - ¶ms.request_host, - ¶ms.request_scheme, - )) + let pass = middleware_stream_processors( + &TextResponse { + content_type: ¶ms.content_type, + request_path: ¶ms.request_path, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + origin_host: ¶ms.origin_host, + request_state: ¶ms.request_state, + max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, + prepared: ¶ms.prepared, + }, + settings, + integration_registry, + )?; + response_headers = pass.response_headers; + Box::new(WithStreamProcessors { + inner: Box::new(create_url_replacer( + ¶ms.origin_host, + ¶ms.origin_url, + ¶ms.request_host, + ¶ms.request_scheme, + )), + processors: pass.processors, + }) }; - Ok((Self { inner }, inline_seam_token)) + Ok(( + Self { + inner, + response_headers, + }, + inline_seam_token, + )) + } +} + +impl PublisherBodyProcessor { + /// The headers the serve middleware write on the response, once. + fn take_response_headers(&mut self) -> Vec { + std::mem::take(&mut self.response_headers) } } @@ -726,7 +1009,7 @@ fn process_response_streaming( output: &mut W, params: &ProcessResponseParams, output_compression: Compression, -) -> Result<(), Report> { +) -> Result, Report> { let is_html = is_html_content_type(params.content_type); let is_rsc_flight = content_type_contains_ascii_case_insensitive(params.content_type, "text/x-component"); @@ -752,21 +1035,26 @@ fn process_response_streaming( // gzip would reject bodies the identity, deflate and brotli paths accept. let max_pending_decoded_bytes = params.settings.publisher.max_buffered_body_bytes; + let mut response_headers = Vec::new(); if is_html { - let processor = create_html_stream_processor(HtmlStreamProcessorParams { + let (processor, headers) = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: params.origin_host, request_host: params.request_host, request_scheme: params.request_scheme, + request_path: params.request_path, settings: params.settings, integration_registry: params.integration_registry, + permissions_script: params.permissions_script.map(str::to_string), ad_slots_script: params.ad_slots_script.map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.cloned(), + request_state: params.request_state.clone(), shared_template_authorized: params.shared_template_authorized, csp_nonce_observed: params.csp_nonce_observed.cloned(), + prepared: Arc::clone(params.prepared), + public_prefix: params.public_prefix, deferred_inline_marker: None, })?; + response_headers = headers; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; @@ -783,18 +1071,36 @@ fn process_response_streaming( .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } else { - let replacer = create_url_replacer( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - StreamingPipeline::new(config, replacer) + let pass = middleware_stream_processors( + &TextResponse { + content_type: params.content_type, + request_path: params.request_path, + request_host: params.request_host, + request_scheme: params.request_scheme, + origin_host: params.origin_host, + request_state: params.request_state, + max_buffered_script_bytes: params.settings.publisher.max_buffered_body_bytes, + prepared: params.prepared, + }, + params.settings, + params.integration_registry, + )?; + response_headers = pass.response_headers; + let processor = WithStreamProcessors { + inner: Box::new(create_url_replacer( + params.origin_host, + params.origin_url, + params.request_host, + params.request_scheme, + )), + processors: pass.processors, + }; + StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) .process(body_as_reader(body)?, output)?; } - Ok(()) + Ok(response_headers) } async fn process_response_streaming_async( @@ -803,7 +1109,7 @@ async fn process_response_streaming_async( params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, -) -> Result<(), Report> { +) -> Result, Report> { log::debug!( "process_response_streaming_async: content_type={}, content_encoding={}", params.content_type, @@ -822,6 +1128,7 @@ async fn process_response_streaming_async( // pending auction; `deferred_inline_seam_token` returns `None` for it. let (mut processor, inline_seam_token) = PublisherBodyProcessor::new(params, settings, integration_registry)?; + let response_headers = processor.take_response_headers(); if inline_seam_token.is_some() { // A `debug_assert!` would be compiled out of the release wasm builds that // actually ship, which is exactly where an unresolved token would reach a @@ -839,7 +1146,8 @@ async fn process_response_streaming_async( output_compression, settings.publisher.max_buffered_body_bytes, ) - .await + .await?; + Ok(response_headers) } /// Pull, decode, process, and encode the next chunk of a no-hold pipeline. @@ -1314,12 +1622,18 @@ struct HtmlStreamProcessorParams<'a> { origin_host: &'a str, request_host: &'a str, request_scheme: &'a str, + /// See [`OwnedProcessResponseParams::request_path`]. + request_path: &'a str, settings: &'a Settings, integration_registry: &'a IntegrationRegistry, + /// Head script carrying this request's permission state, or [`None`] under a + /// shared-template mode. See [`template_permissions_script`]. + permissions_script: Option, ad_slots_script: Option, ad_bids_state: Arc>>, - suppress_datadome_client_side_tag: bool, - gpt_diagnostics: Option, + /// What modules left on the request, which the `[[serve]]` entry's + /// middleware read where that entry runs on this pass. + request_state: IntegrationRequestState, /// Whether a shared template was authorized for this response. /// /// Carried rather than re-derived so both seams see the same answer. See @@ -1329,29 +1643,9 @@ struct HtmlStreamProcessorParams<'a> { csp_nonce_observed: Option>, /// Request-specific parser marker used by a pending inline auction. deferred_inline_marker: Option, -} - -/// The diagnostics decision the template may carry. -/// -/// Diagnostics is request-scoped — activated by a cookie or query parameter, and -/// documented as an immutable per-request decision — so it must not reach a shared -/// template. -/// -/// It does not leak today even without this gate, but only by coincidence: -/// `requires_private_no_store()` is a strict superset of the conditions under which -/// a script is emitted, and that stamp lands before the template cache gate reads response -/// headers, so the gate refuses. Two independent conditions that happen to align, -/// with nothing enforcing the relationship. This makes the guarantee explicit; -/// `requires_private_no_store_is_a_superset_of_injection` keeps the coincidence as a -/// backstop if this gate is ever removed. -pub(crate) fn template_gpt_diagnostics( - mode: AssemblyMode, - decision: Option, -) -> Option { - match mode { - AssemblyMode::Inline => decision, - AssemblyMode::Esi => None, - } + prepared: Arc, + /// See [`OwnedProcessResponseParams::public_prefix`]. + public_prefix: &'a crate::prefix::PublicPrefix, } /// The marker emitted at the `` seam under [`AssemblyMode::Esi`], reserving the @@ -1472,6 +1766,20 @@ pub(crate) fn body_close_injection( } } +/// The head script this response's permission state belongs in, if any. +/// +/// Derived here rather than carried on [`OwnedProcessResponseParams`] so the state +/// has one representation on the request (the JSON) and the head-or-seam decision is +/// taken from the same effective mode both seams use. Under a shared-template mode +/// the answer is [`None`] and the seam carries the state instead. +fn permissions_script_for( + params: &OwnedProcessResponseParams, + settings: &Settings, +) -> Option { + let mode = effective_assembly_mode(settings, params.template_cache_key.is_some()); + template_permissions_script(mode, ¶ms.permissions_json) +} + fn deferred_inline_seam_token( settings: &Settings, shared_template_authorized: bool, @@ -1495,8 +1803,9 @@ fn deferred_inline_seam_token( fn create_html_stream_processor( params: HtmlStreamProcessorParams<'_>, -) -> Result, Report> { - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; +) -> Result<(impl StreamProcessor + use<>, Vec), Report> { + use crate::html_processor::{HtmlProcessorConfig, create_html_processor_with_middleware}; + use crate::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; let config = HtmlProcessorConfig::from_settings( params.settings, @@ -1514,8 +1823,6 @@ fn create_html_stream_processor( _ => body_close_injection(assembly_mode, params.ad_slots_script.is_some()), }; - let gpt_diagnostics = template_gpt_diagnostics(assembly_mode, params.gpt_diagnostics); - // Only a response that can be stored has a consumer for the observation, so the // handlers are not registered for ordinary inline traffic. let csp_nonce_observed = params @@ -1524,13 +1831,109 @@ fn create_html_stream_processor( .flatten(); let config = config + .with_permissions_script(params.permissions_script) .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(gpt_diagnostics) + .with_request_state(params.request_state) .with_body_close(body_close) .with_csp_nonce_observer(csp_nonce_observed) - .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); + .with_public_prefix(params.public_prefix.clone()) + .with_prepared(params.prepared); + + // The middleware of the first [[fetch]] entry covering this path. + let fetch = params.integration_registry.middleware_chain( + ¶ms.settings.fetch, + MiddlewarePhase::Fetch, + HTML_MEDIA_TYPE, + params.request_path, + ); + // A response that may be stored leaves this processor as the template + // every reader shares, so its [[serve]] entry runs on each reader's copy + // once that copy is assembled. See `run_serve_pass`. + let serve = (!params.shared_template_authorized).then(|| { + params.integration_registry.middleware_chain( + ¶ms.settings.serve, + MiddlewarePhase::Serve, + HTML_MEDIA_TYPE, + params.request_path, + ) + }); + create_html_processor_with_middleware(config, &fetch, serve.as_ref()) + .map_err(|message| Report::new(TrustedServerError::Configuration { message })) +} + +/// What the `[[serve]]` entry covering this request does to a reader's copy +/// that is assembled and not yet encoded, and to the response, which is +/// nothing when no entry covers the request. +/// +/// The origin host is read from the settings. A page served from the store +/// keeps no record of it, and a serve middleware is told the same about a +/// page from the store as about one fetched for this request. +fn reader_serve_processor( + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result> { + use crate::html_processor::{ReaderDocument, create_serve_processor}; + use crate::middleware::{HTML_MEDIA_TYPE, MiddlewarePhase}; + + if !is_html_content_type(¶ms.content_type) { + return Ok(ServePass::default()); + } + let chain = integration_registry.middleware_chain( + &settings.serve, + MiddlewarePhase::Serve, + HTML_MEDIA_TYPE, + ¶ms.request_path, + ); + let origin_host = settings.publisher.origin_host(); + create_serve_processor( + &chain, + &ReaderDocument { + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + origin_host: &origin_host, + platform: integration_registry.platform(), + request_state: ¶ms.request_state, + max_buffered_script_bytes: settings.publisher.max_buffered_body_bytes, + prepared: ¶ms.prepared, + }, + ) + .map_err(|message| Report::new(TrustedServerError::Configuration { message })) +} + +/// Runs the `[[serve]]` entry covering this request over a reader's whole +/// copy, assembled and not yet encoded, and writes its headers on the +/// response. +fn run_serve_pass( + bytes: Vec, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + response: &mut Response, +) -> Result, Report> { + let pass = reader_serve_processor(params, settings, integration_registry)?; + write_response_headers(response, pass.response_headers); + let Some(mut processor) = pass.processor else { + return Ok(bytes); + }; + processor + .process_chunk(&bytes, true) + .change_context(TrustedServerError::Proxy { + message: "a serve middleware could not rewrite the page".to_string(), + }) +} - Ok(create_html_processor(config)) +/// One piece of a streamed reader's copy through the `[[serve]]` entry, when +/// one covers the request. +fn serve_chunk( + serve: &mut Option>, + bytes: Vec, + is_last: bool, +) -> Result, std::io::Error> { + match serve { + Some(processor) => processor.process_chunk(&bytes, is_last), + None => Ok(bytes), + } } /// Result of publisher request handling, indicating whether the response body @@ -1649,6 +2052,43 @@ pub(crate) fn classify_response_route( ResponseRoute::Stream } +/// Whether the origin answered with a page where `entry` places core's +/// robots.txt middleware, as a single-page application's catch-all does. A +/// crawler reads a page there as no rules at all, so the file is made as it is +/// where the origin has none. +fn answers_robots_txt_with_a_page( + response: &Response, + entry: &crate::middleware::PhaseEntry, +) -> bool { + response.status() == StatusCode::OK + && entry + .middleware + .iter() + .any(|name| name == crate::robots_txt::middleware::MODULE) + && response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(is_html_content_type) +} + +/// An empty document of `media_type`, which the middleware a `[[fetch]]` +/// entry names for a path the origin has nothing at is run on. +fn empty_document(media_type: &str) -> Result, Report> { + let content_type = if media_type.starts_with("text/") { + format!("{media_type}; charset=utf-8") + } else { + media_type.to_owned() + }; + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, content_type) + .body(EdgeBody::empty()) + .change_context(TrustedServerError::Proxy { + message: format!("failed to build an empty {media_type} document"), + }) +} + fn response_cache_control_is_private_or_no_store(response: &Response) -> bool { cache_control_headers_are_private_or_no_store(response.headers()) } @@ -1699,6 +2139,14 @@ pub struct OwnedProcessResponseParams { /// /// Request-scoped, so it travels with the request rather than into the template. pub(crate) seam_ad_slots: Option, + /// This request's resolved permission state as page JSON, from + /// [`PermissionState::page_json`]. + /// + /// Carried as the JSON rather than as a rendered script because it is delivered in + /// two different places: the head under an inline response, and the `` seam + /// under a shared-template one. An empty string means no caller filled it in and + /// renders as the empty state. + pub(crate) permissions_json: String, /// Origin policy headers to store with the template and replay on a hit. pub(crate) policy_headers: Vec<(String, String)>, pub(crate) content_encoding: String, @@ -1706,6 +2154,9 @@ pub struct OwnedProcessResponseParams { pub(crate) origin_url: String, pub(crate) request_host: String, pub(crate) request_scheme: String, + /// The path the reader asked for, which decides the `[[fetch]]` and + /// `[[serve]]` entries that cover the document. + pub(crate) request_path: String, pub(crate) content_type: String, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, @@ -1719,17 +2170,21 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, - /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. - pub(crate) suppress_datadome_client_side_tag: bool, - /// Request-scoped conditional diagnostics delivery decision. - pub(crate) gpt_diagnostics: - Option, + /// What modules left on the request, for their serve middleware and + /// their response finalizers. + pub(crate) request_state: IntegrationRequestState, /// Set by the transform when the document carries a response-bound CSP nonce. /// /// `None` wherever no transform runs. Recorded by the HTML parser rather than /// rescanned from the output, which cannot tell a `nonce` attribute from the same /// word inside a script. pub(crate) csp_nonce_observed: Option>, + /// What the middleware prepared for this document, see + /// [`Middleware::prepare`](crate::middleware::Middleware::prepare). + pub(crate) prepared: Arc, + /// The request's public prefix, from its services, which the page's + /// bundle tag tells the client script. + pub(crate) public_prefix: crate::prefix::PublicPrefix, } /// Response-authorized template cache insert inputs. The key is built before origin lookup; the @@ -1824,7 +2279,7 @@ pub async fn buffer_publisher_response_async( return Ok(response); } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); - stream_publisher_body_async( + let response_headers = stream_publisher_body_async( body, &mut output, &mut params, @@ -1834,6 +2289,7 @@ pub async fn buffer_publisher_response_async( services, ) .await?; + write_response_headers(&mut response, response_headers); // Authorized template cache transforms are emitted as identity by // `process_response_streaming_async`; inline transforms retain the origin // coding. This avoids recompressing and immediately decoding a full document. @@ -1938,6 +2394,15 @@ pub async fn buffer_publisher_response_async( set_assembly_response_state(&mut response, state); } let bytes = if was_authorized { + // The [[serve]] entry runs here on a shared template, and + // inside the transform on a page that was never authorized. + let bytes = run_serve_pass( + bytes, + ¶ms, + settings, + integration_registry, + &mut response, + )?; encode_complete_body(bytes, response_compression(&response))? } else { bytes @@ -1987,6 +2452,13 @@ pub async fn buffer_publisher_response_async( assembled.extend_from_slice(head); assembled.extend_from_slice(seam.as_bytes()); assembled.extend_from_slice(tail); + let assembled = run_serve_pass( + assembled, + ¶ms, + settings, + integration_registry, + &mut response, + )?; let assembled = encode_complete_body(assembled, response_compression(&response))?; response.headers_mut().insert( http::header::CONTENT_LENGTH, @@ -2064,7 +2536,7 @@ fn assemble_if_shared( Ok((out, Some(AssemblyResponseState::ByteSeamFallback))) } -/// Fingerprint of settings, browser bundles, and the core implementation. +/// Fingerprint of settings, the browser modules a registry serves, and the core implementation. /// /// This intentionally over-invalidates. Trying to maintain a hand-written list already /// omitted publisher origin identity and creative-opportunity shaping fields. A digest of @@ -2074,20 +2546,44 @@ fn assemble_if_shared( /// files (including embedded head scripts), build logic, manifests, and the workspace /// lockfile when present, so implementation edits automatically invalidate templates. /// +/// The module digest covers the parts this registry can actually serve, carried modules +/// included, so a vendor crate that rebuilds its browser module moves the hash, and +/// templates keyed under the previous hash are no longer read. It also narrows the +/// module set from every module compiled into the binary to the ones this deployment can +/// serve. That is safe because a module this deployment never serves appears in no bundle +/// hash and no injected URL, so it cannot change what a template renders. +/// /// # Panics /// /// Does not panic: serializing the already-deserialized typed settings to a JSON value is /// infallible for this schema. -fn template_fingerprint(settings: &Settings) -> String { - template_fingerprint_with_build_digest(settings, TEMPLATE_BUILD_DIGEST) +fn template_fingerprint( + settings: &Settings, + integration_registry: &IntegrationRegistry, + public_prefix: &crate::prefix::PublicPrefix, +) -> String { + template_fingerprint_with_build_digest( + settings, + integration_registry, + public_prefix, + TEMPLATE_BUILD_DIGEST, + ) } -fn template_fingerprint_with_build_digest(settings: &Settings, build_digest: &str) -> String { +fn template_fingerprint_with_build_digest( + settings: &Settings, + integration_registry: &IntegrationRegistry, + public_prefix: &crate::prefix::PublicPrefix, + build_digest: &str, +) -> String { let mut hasher = sha2::Sha256::new(); hasher.update(build_digest.as_bytes()); - hasher.update( - trusted_server_js::concatenated_hash(trusted_server_js::all_module_ids()).as_bytes(), - ); + hasher + .update(crate::tsjs_bundle::compose_hash(&integration_registry.js_parts_all()).as_bytes()); + // The bundle tag carries the public prefix, and a change to the selected + // module can change it with nothing in the settings changing. + hasher.update(public_prefix.as_str().as_bytes()); + hasher.update([0]); // EdgeZero's canonical form sorts object keys itself, so independently deserialized // HashMaps hash identically even when a dependency enables `serde_json/preserve_order`. // Array order is preserved: set-valued settings must serialize deterministically @@ -2117,21 +2613,28 @@ fn response_carries_a_seam_marker(was_authorized: bool, settings: &Settings) -> /// calls `scheduleInitialAdInit`, which schedules `adInit` for precisely the traffic /// that opted out. Absent is not the same as empty here. /// +/// It is never nothing at all, because the permission state has to reach the +/// page whether or not the ad stack ran, and a shared template's head cannot carry it. +/// The no-ad-stack answer is [`build_permissions_seam_script`], which sets the state and +/// schedules no ad init. +/// /// Shared by the miss path and by **both** hit finalizers. They previously each spelled /// the decision out, and the two hit paths spelled it `unwrap_or("[]")` — so the gate /// held on a cache miss and was ignored on every cache hit. fn seam_script_for(params: &OwnedProcessResponseParams) -> String { - params - .seam_ad_slots - .as_deref() - .map(|slots| params.ad_bids_state.build_seam_script(slots)) - .unwrap_or_default() + match params.seam_ad_slots.as_deref() { + Some(slots) => params + .ad_bids_state + .build_seam_script(slots, ¶ms.permissions_json), + None => build_permissions_seam_script(¶ms.permissions_json), + } } /// Builds the injection state a cached template needs on the way out. /// -/// The template carries no auction state — that is what makes it shareable — so the -/// per-reader parts are attached here, from this request. +/// The template carries no auction state and no permission state, which is what makes +/// it shareable, so the per-reader parts are attached here, from this request. Both +/// leave through the seam, never through the cached head. fn build_template_assembly_params( entry: &crate::platform::TemplateEntry, settings: &Settings, @@ -2139,9 +2642,11 @@ fn build_template_assembly_params( request_scheme: &str, price_granularity: PriceGranularity, ad_bids_state: AdBidsState, + permissions_json: String, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), // Already stored; storing again on a hit would be pointless work. template_cache_key: None, seam_ad_slots: None, @@ -2151,16 +2656,22 @@ fn build_template_assembly_params( origin_url: settings.publisher.origin_url.clone(), request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), + // Set by the caller, with the rest of what belongs to the request. + request_path: String::new(), content_type: entry.metadata.content_type.clone(), // The template already carries the head seam; re-injecting would duplicate it. + permissions_json, ad_slots_script: None, ad_bids_state, auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity, - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + // A request that carries any keeps to the origin path, so none reaches a + // document served from a template. + request_state: IntegrationRequestState::default(), + // Set by the caller, from the request's services. + public_prefix: crate::prefix::PublicPrefix::default(), } } @@ -2457,6 +2968,26 @@ pub async fn publisher_response_into_streaming_response( return Ok(response); } + // Planned before the lazy body is built, so a serve entry that + // cannot run fails the response while it still can be failed. + let serve_pass = match reader_serve_processor(¶ms, &settings, integration_registry) + { + Ok(serve_pass) => serve_pass, + Err(err) => { + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + &services, + params.auction_observation.take(), + dispatched, + "processor_init_error", + ) + .await; + } + return Err(err); + } + }; + write_response_headers(&mut response, serve_pass.response_headers); + let mut serve = serve_pass.processor; let services = services.clone(); let settings = Arc::clone(&settings); let orchestrator = Arc::clone(&orchestrator); @@ -2480,8 +3011,10 @@ pub async fn publisher_response_into_streaming_response( let (head, tail) = split_template_at_seam(&template) .map_err(|e| std::io::Error::other(e.to_string()))?; let mut encoder = BodyStreamEncoder::new(compression); + // Each piece goes through the serve entry as it leaves, so + // the head is still written before the auction is awaited. let encoded_head = encoder - .encode_chunk(head.to_vec()) + .encode_chunk(serve_chunk(&mut serve, head.to_vec(), false)?) .map_err(publisher_stream_error)?; if !encoded_head.is_empty() { yield bytes::Bytes::from(encoded_head); @@ -2512,14 +3045,14 @@ pub async fn publisher_response_into_streaming_response( let seam = seam_script_for(¶ms); if !seam.is_empty() { let encoded_seam = encoder - .encode_chunk(seam.into_bytes()) + .encode_chunk(serve_chunk(&mut serve, seam.into_bytes(), false)?) .map_err(publisher_stream_error)?; if !encoded_seam.is_empty() { yield bytes::Bytes::from(encoded_seam); } } let encoded_tail = encoder - .encode_chunk(tail.to_vec()) + .encode_chunk(serve_chunk(&mut serve, tail.to_vec(), true)?) .map_err(publisher_stream_error)?; if !encoded_tail.is_empty() { yield bytes::Bytes::from(encoded_tail); @@ -2589,6 +3122,7 @@ pub async fn publisher_response_into_streaming_response( return Err(err); } }; + write_response_headers(&mut response, processor.take_response_headers()); // The guard is created before the lazy stream so an auction whose // response body is dropped unpolled still logs the loss. let dispatched_auction = params.dispatched_auction.take().map(|dispatched| { @@ -2761,6 +3295,13 @@ fn is_html_document_request(req: &Request) -> bool { is_navigation_request(req) } +/// Whether an integration marked this request's response as personalized. +fn request_requires_personalized_delivery(req: &Request) -> bool { + req.extensions() + .get::() + .is_some() +} + /// Removes request headers that can produce a bodyless or partial origin response. fn strip_conditional_and_range_headers(req: &mut Request) { req.headers_mut().remove(header::IF_NONE_MATCH); @@ -2784,14 +3325,14 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } -/// Prevent shared caches from replaying tag-suppressed HTML to other clients. -fn apply_datadome_client_tag_cache_privacy( +/// Prevent shared caches from replaying personalized HTML to other clients. +fn apply_personalized_response_cache_privacy( response: &mut Response, method: &Method, - suppress_datadome_client_side_tag: bool, + response_is_personalized: bool, content_type: &str, ) { - if suppress_datadome_client_side_tag + if response_is_personalized && response_carries_body(method, response.status()) && is_html_content_type(content_type) { @@ -2903,22 +3444,26 @@ pub fn stream_publisher_body( params: &OwnedProcessResponseParams, settings: &Settings, integration_registry: &IntegrationRegistry, -) -> Result<(), Report> { +) -> Result, Report> { + let permissions_script = permissions_script_for(params, settings); let borrowed = ProcessResponseParams { content_encoding: ¶ms.content_encoding, origin_host: ¶ms.origin_host, origin_url: ¶ms.origin_url, request_host: ¶ms.request_host, request_scheme: ¶ms.request_scheme, + request_path: ¶ms.request_path, settings, content_type: ¶ms.content_type, integration_registry, + permissions_script: permissions_script.as_deref(), ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: params.ad_bids_state.script_cell(), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.as_ref(), + request_state: ¶ms.request_state, shared_template_authorized: params.template_cache_key.is_some(), csp_nonce_observed: params.csp_nonce_observed.as_ref(), + prepared: ¶ms.prepared, + public_prefix: ¶ms.public_prefix, }; let input_compression = Compression::from_content_encoding(¶ms.content_encoding); let output_compression = if params.template_cache_key.is_some() { @@ -2957,7 +3502,7 @@ pub async fn stream_publisher_body_async( integration_registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, -) -> Result<(), Report> { +) -> Result, Report> { let Some(dispatched) = params.dispatched_auction.take() else { if body.is_stream() { return process_response_streaming_async( @@ -3012,34 +3557,38 @@ pub async fn stream_publisher_body_async( params.ad_slots_script.is_some(), true, ); - let mut processor = match create_html_stream_processor(HtmlStreamProcessorParams { - origin_host: ¶ms.origin_host, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - settings, - integration_registry, - ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), - ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), - suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, - gpt_diagnostics: params.gpt_diagnostics.clone(), - shared_template_authorized: params.template_cache_key.is_some(), - csp_nonce_observed: params.csp_nonce_observed.clone(), - deferred_inline_marker: inline_seam_token - .as_ref() - .map(|token| String::from_utf8_lossy(token).into_owned()), - }) { - Ok(processor) => processor, - Err(err) => { - emit_abandoned_auction( - services, - telemetry.observation, - dispatched, - "processor_init_error", - ) - .await; - return Err(err); - } - }; + let (mut processor, response_headers) = + match create_html_stream_processor(HtmlStreamProcessorParams { + prepared: Arc::clone(¶ms.prepared), + public_prefix: ¶ms.public_prefix, + origin_host: ¶ms.origin_host, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + request_path: ¶ms.request_path, + settings, + integration_registry, + permissions_script: permissions_script_for(params, settings), + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), + request_state: params.request_state.clone(), + shared_template_authorized: params.template_cache_key.is_some(), + csp_nonce_observed: params.csp_nonce_observed.clone(), + deferred_inline_marker: inline_seam_token + .as_ref() + .map(|token| String::from_utf8_lossy(token).into_owned()), + }) { + Ok(built) => built, + Err(err) => { + emit_abandoned_auction( + services, + telemetry.observation, + dispatched, + "processor_init_error", + ) + .await; + return Err(err); + } + }; let input_compression = Compression::from_content_encoding(¶ms.content_encoding); let output_compression = if params.template_cache_key.is_some() { @@ -3067,17 +3616,18 @@ pub async fn stream_publisher_body_async( }, }, ) - .await + .await?; + Ok(response_headers) } -/// Builds the canonical mediator placeholder [`Request`] passed to the collect +/// Builds the canonical ad server placeholder [`Request`] passed to the collect /// phase via [`make_collect_context`]. /// /// The URI is the compile-time constant /// [`MEDIATOR_PLACEHOLDER_URL`](crate::auction::types::MEDIATOR_PLACEHOLDER_URL), /// so the builder is infallible; a default-URI fallback would trip /// [`make_collect_context`]'s `debug_assert_eq!`. -fn mediator_placeholder_request() -> Request { +fn adserver_placeholder_request() -> Request { Request::builder() .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) .body(EdgeBody::empty()) @@ -3150,7 +3700,7 @@ fn rewrite_origin_request( /// See [`AuctionContext::request`]: the orchestrator's collect path runs /// after `send_async` has already consumed the real client request, so this /// context carries a synthetic placeholder. The orchestrator itself -/// instantiates a fresh placeholder when it actually invokes a mediator — +/// instantiates a fresh placeholder when it actually invokes an ad server, /// this argument is plumbing for the (presently unused) case where the /// orchestrator needs the caller's request shape. fn make_collect_context<'a>( @@ -3183,7 +3733,11 @@ pub(crate) const BOT_USER_AGENT_FRAGMENTS: &[&str] = /// Returns true when the request's User-Agent matches any well-known crawler /// fragment in [`BOT_USER_AGENT_FRAGMENTS`]. -pub(crate) fn is_bot_user_agent(req: &Request) -> bool { +/// +/// Public so a module outside this crate treats a crawler the way the page +/// path does. +#[must_use] +pub fn is_bot_user_agent(req: &Request) -> bool { let ua = req .headers() .get("user-agent") @@ -3196,7 +3750,11 @@ pub(crate) fn is_bot_user_agent(req: &Request) -> bool { /// Returns true when the request advertises itself as a prefetch via either /// the standard `Sec-Purpose` or the legacy `Purpose` header. -pub(crate) fn is_prefetch_request(req: &Request) -> bool { +/// +/// Public so a module outside this crate treats a prefetch the way the page +/// path does. +#[must_use] +pub fn is_prefetch_request(req: &Request) -> bool { let header = |name: &str| { req.headers() .get(name) @@ -3295,8 +3853,8 @@ impl AdBidsState { } /// Build the shared-template seam, retaining the same debug prefix as inline. - fn build_seam_script(&self, slots_json: &str) -> String { - let seam = build_seam_script(slots_json, &self.bids()); + fn build_seam_script(&self, slots_json: &str, permissions_json: &str) -> String { + let seam = build_seam_script(slots_json, &self.bids(), permissions_json); let prefix = self .debug_prefix .lock() @@ -3374,11 +3932,10 @@ pub(crate) fn write_bids_to_state( /// enabled cannot bloat every page render without bound. const MAX_AUCTION_DEBUG_DUMP_BYTES: usize = 256 * 1024; -/// Per-bid creative preview length (in bytes) in the `ts-debug` dump. Mirrors -/// the 512-byte upstream-body preview the prebid provider logs on an HTTP error -/// (`integrations/prebid.rs`): enough to identify a creative without copying -/// megabytes of `adm` markup into every page render. The full creative still -/// renders via the injected bids `", + html_escape_for_script(&permissions_json_or_empty(permissions_json)), html_escape_for_script(slots_json), html_escape_for_script(&bids) ) } +/// Build the `` seam script for a request whose ad stack did not run. +/// +/// The head of a shared template carries nothing request-scoped, so the seam is +/// the only place this reader's permission state can be delivered. Before this +/// existed the seam was empty whenever the ad stack was skipped, which left a +/// bot-classified or permission-denied visitor with no state on the page at all. +/// +/// Carries the state and nothing else. It deliberately does not set `adSlots` or +/// `bids` and does not call `scheduleInitialAdInit`, because scheduling `adInit` +/// for traffic that opted out is what the gate in [`seam_script_for`] exists to +/// prevent. +pub(crate) fn build_permissions_seam_script(permissions_json: &str) -> String { + format!( + "", + html_escape_for_script(&permissions_json_or_empty(permissions_json)) + ) +} + /// The slot definitions a shared-mode seam must carry, as JSON. /// /// Mirrors [`template_ad_slots_script`]'s gating: same `should_run_ad_stack` condition, @@ -6064,8 +6815,9 @@ pub(crate) fn build_empty_bids_script() -> String { /// [`handle_page_bids`] (SPA navigation) so the slot wire shape has a single /// definition and the two paths cannot silently diverge. Property names match /// what the client-side TSJS bundle expects: `gam_unit_path`, `div_id`, -/// `formats`, and `targeting`. Returns `None` when the slot's dynamic GAM unit -/// path exceeds its rendering limit. +/// `formats`, and `targeting`, with `viewport_sizes` added when a format names +/// a viewport bound. Returns `None` when the slot's dynamic GAM unit path +/// exceeds its rendering limit. pub(crate) fn build_slot_json( slot: &crate::creative_opportunities::CreativeOpportunitySlot, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, @@ -6083,13 +6835,22 @@ pub(crate) fn build_slot_json( .iter() .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) .collect(); - Some(serde_json::json!({ + let mut json = serde_json::json!({ "id": slot.id, "gam_unit_path": gam_path, "div_id": div_id, "formats": formats, "targeting": targeting, - })) + }); + // Each band of `CreativeOpportunitySlot::viewport_sizes` as the browser + // takes it, with a `[width, 0]` viewport. + if let Some(bands) = slot.viewport_sizes() { + json["viewport_sizes"] = bands + .into_iter() + .map(|(width, sizes)| serde_json::json!([[width, 0], sizes])) + .collect(); + } + Some(json) } /// Match creative-opportunity slots and omit dynamic GAM paths that cannot be @@ -6148,13 +6909,12 @@ pub(crate) enum TemplateCacheBypassReason { /// still-positive origin lifetime rather than inventing one. #[display("origin response has no positive shared freshness")] NoPositiveFreshness, - /// The request was authenticated. #1009 describes a Basic-Auth-gated - /// deployment, so an authorized response entering a shared cache is a live - /// concern rather than a hypothetical one. + /// The request carried credentials, which are bound for the origin, so its + /// response is one reader's and never enters a shared cache. #[display("request carried Authorization")] AuthorizedRequest, - /// Not a 200. This is also what covers a `DataDome` block, which replaces the - /// document with a `403` (`integrations/datadome/protection.rs:778`). + /// Not a 200. This is also what covers a bot protection module's block, which + /// replaces the document with a `403`. #[display("status was not 200 OK")] NonOkStatus, /// Not HTML, so there is no template to transform. @@ -6708,6 +7468,61 @@ pub(crate) fn template_ad_slots_script( } } +/// The permission-state `", + escaped + ) +} + +/// The permission state as page JSON, substituting the empty state for an unset +/// value. +/// +/// [`PermissionState::page_json`] never returns an empty string, so this only +/// covers a params value nothing filled in. `JSON.parse("")` throws, and a +/// thrown head script takes the rest of the snippet with it, so an unset value +/// renders as the empty state rather than as broken JavaScript. +fn permissions_json_or_empty(permissions_json: &str) -> Cow<'_, str> { + if permissions_json.is_empty() { + Cow::Owned(PermissionState::default().page_json()) + } else { + Cow::Borrowed(permissions_json) + } +} + /// Build the `tsjs.adSlots` `") - .expect("should substitute the placeholder the transform emitted"); - - assert_eq!( - replaced, br#"

article

"#, - "should splice the payload exactly where the parser marked the body end" - ); - } - - #[test] - fn a_body_close_written_in_script_data_does_not_attract_the_seam() { - // The reverse byte search this replaced picked the string literal, spliced a - // `` terminated the - // publisher's script — in the served page and in the stored template alike. - let document = format!( - r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"# - ) - .into_bytes(); - - let replaced = replace_seam_placeholder(document, b"") - .expect("should substitute the placeholder the transform emitted"); + assert_eq!( + header_of(&response, header::HeaderName::from_static("x-ts-assembly")), + Some("byte-seam-fallback") + ); + let document = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); - assert_eq!( - replaced, - br#"

article

"#, - "should leave a body-close sequence inside script data untouched" - ); + assert!(document.to_ascii_lowercase().contains(expected_directive)); + assert!(document.contains("scheduleInitialAdInit")); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "publisher-authored ESI must never enter the shared template cache" + ); + assert_eq!( + assembler.calls.load(Ordering::Relaxed), + 0, + "publisher-authored ESI must never reach the platform parser" + ); + } } - #[test] - fn a_publisher_copy_of_the_placeholder_refuses_substitution() { - let document = format!( - "

{TEMPLATE_SEAM_PLACEHOLDER}

article{TEMPLATE_SEAM_PLACEHOLDER}" - ) - .into_bytes(); - - let (returned, error) = - replace_seam_placeholder(document.clone(), b"") - .expect_err("should refuse a document that collides with the placeholder"); - - assert!( - matches!(error, SeamError::Repeated), - "should name the collision rather than guess an occurrence" - ); - assert_eq!( - returned, document, - "should hand back the publisher document byte for byte" + #[tokio::test] + async fn ineligible_requests_and_responses_never_use_the_platform_assembler() { + let inline_stub = Arc::new(StubHttpClient::new()); + let inline_cache = Arc::new(MemoryTemplateCache::default()); + let inline_assembler = Arc::new(RecordingTemplateAssembler::default()); + let inline_settings = Arc::new(settings_with_mode("inline")); + let inline_services = services_with_assembler( + Arc::clone(&inline_stub), + Arc::clone(&inline_cache), + Arc::clone(&inline_assembler), ); - } - - #[test] - fn a_document_without_the_placeholder_refuses_substitution() { - let document = b"

article

".to_vec(); + queue_shareable_html(&inline_stub); - let (returned, error) = - replace_seam_placeholder(document.clone(), b"") - .expect_err("should refuse a document the transform did not mark"); + let _ = + body_of(run(&inline_settings, &inline_services, navigation_request()).await).await; + assert_eq!(inline_assembler.calls.load(Ordering::Relaxed), 0); - assert!( - matches!(error, SeamError::Missing), - "should name the absent placeholder rather than append blindly" - ); - assert_eq!( - returned, document, - "should hand back the document unchanged" + let private_stub = Arc::new(StubHttpClient::new()); + let private_cache = Arc::new(MemoryTemplateCache::default()); + let private_assembler = Arc::new(RecordingTemplateAssembler::default()); + let esi_settings = Arc::new(settings_with_mode("esi")); + let private_services = services_with_assembler( + Arc::clone(&private_stub), + Arc::clone(&private_cache), + Arc::clone(&private_assembler), ); - } - - #[test] - fn parser_validation_does_not_change_the_cached_schema() { - assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 5); - assert_eq!(AD_ASSEMBLY_SEAM, ""); - assert!(!contains_publisher_esi_directive( - AD_ASSEMBLY_SEAM.as_bytes() - )); - } - - /// Shareable HTML that already contains the seam marker. - /// - /// The marker is reserved, but publisher content can still contain it. The - /// transform adds its own terminal placeholder; repeated markers then make the - /// response bypass the template cache rather than requiring normalization. - fn queue_html_that_collides_with_the_marker(stub: &StubHttpClient) { - stub.push_response_with_headers( + private_stub.push_response_with_headers( 200, - format!( - "

origin

" - ) - .into_bytes(), + b"private origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), ("cache-control", "public, max-age=300"), + ("set-cookie", "reader=personalized; Path=/"), ], ); - } - #[tokio::test] - async fn origin_marker_collision_bypasses_template_cache_without_mutating_publisher_bytes() - { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_html_that_collides_with_the_marker(&stub); - queue_html_that_collides_with_the_marker(&stub); + let _ = + body_of(run(&esi_settings, &private_services, navigation_request()).await).await; + assert_eq!(private_assembler.calls.load(Ordering::Relaxed), 0); - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("cold document should be UTF-8"); - let warm = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, + let authenticated_stub = Arc::new(StubHttpClient::new()); + let authenticated_cache = Arc::new(MemoryTemplateCache::default()); + let authenticated_assembler = Arc::new(RecordingTemplateAssembler::default()); + let authenticated_services = services_with_assembler( + Arc::clone(&authenticated_stub), + Arc::clone(&authenticated_cache), + Arc::clone(&authenticated_assembler), + ); + queue_shareable_html(&authenticated_stub); + let mut authenticated_request = navigation_request(); + authenticated_request.headers_mut().insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer example-token"), + ); + + let _ = body_of( + run( + &esi_settings, + &authenticated_services, + authenticated_request, + ) + .await, ) - .expect("warm document should be UTF-8"); + .await; + assert_eq!(authenticated_assembler.calls.load(Ordering::Relaxed), 0); - assert!( - cold.contains("origin") && cold.contains("window.tsjs"), - "a reserved-comment collision must not turn a valid origin 200 into a 500" - ); - for document in [&cold, &warm] { - assert!( - document.contains(&format!("window.publisherMarker=\"{AD_ASSEMBLY_SEAM}\"")), - "should preserve a publisher marker inside script data: {document}" - ); - } - assert_eq!(stub.recorded_request_uris().len(), 2); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "should not store a template with a publisher marker collision" + let failed_cache_stub = Arc::new(StubHttpClient::new()); + let failed_cache = Arc::new(MemoryTemplateCache::default()); + failed_cache.fail_lookup.store(true, Ordering::Relaxed); + let failed_cache_assembler = Arc::new(RecordingTemplateAssembler::default()); + let failed_cache_services = services_with_assembler( + Arc::clone(&failed_cache_stub), + Arc::clone(&failed_cache), + Arc::clone(&failed_cache_assembler), ); + queue_shareable_html(&failed_cache_stub); + + let _ = body_of(run(&esi_settings, &failed_cache_services, navigation_request()).await) + .await; + assert_eq!(failed_cache_assembler.calls.load(Ordering::Relaxed), 0); } #[tokio::test] - async fn html_without_an_explicit_body_gets_a_terminal_seam_instead_of_a_500() { + async fn fastly_surrogate_control_still_allows_a_cold_fill_and_warm_hit() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let settings = Arc::new(settings_with_mode_and_template_cache_max_age("esi", 1_200)); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + + // This is the policy observed on the publisher that exposed the bug. Fastly + // gives the surrogate lifetime edge precedence; the configured safety + // ceiling allows the origin's twenty-minute edge freshness. stub.push_response_with_headers( 200, - b"
origin fragment
".to_vec(), + b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), + ("cache-control", "public, max-age=60"), + ( + "surrogate-control", + "max-age=1200, stale-while-revalidate=21600, \ + stale-if-error=604800", + ), ], ); - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("cold fragment should be UTF-8"); - let warm = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("warm fragment should be UTF-8"); - - for document in [&cold, &warm] { - assert!(document.contains("origin fragment")); - assert!(!document.contains(AD_ASSEMBLY_SEAM)); - } - assert_eq!(stub.recorded_request_uris().len(), 1); + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("miss-stored") + ); + let _ = body_of(cold).await; + let stored_max_age = cache + .stored_max_ages + .lock() + .expect("should lock stored max ages")[0]; assert!( - !cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "the recovered template should remain cacheable" + stored_max_age <= Duration::from_secs(1_200) + && stored_max_age > Duration::from_secs(1_195), + "the real store path should receive the configured twenty-minute ceiling, got {stored_max_age:?}" + ); + + let warm = run(&settings, &services, navigation_request()).await; + assert_eq!( + warm.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("hit") + ); + let _ = body_of(warm).await; + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the warm request must reuse the template authorized by Surrogate-Control" ); } #[tokio::test] - async fn an_unsplittable_cached_template_is_a_miss_before_any_header_commits() { - // The hit path used to check only that a marker existed *somewhere* and - // leave the exactly-one check to the finalizer. By then the 200 and its - // headers were committed and, on the streaming adapter, the document head - // was already on the wire — so the only available failure was a truncated - // response. Falling back to the origin is a slower correct page. + async fn one_identity_template_is_negotiated_per_reader_on_miss_and_hit() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); + stub.push_response_with_headers( + 200, + gzip_encode(b"origin"), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-encoding", "gzip"), + ("cache-control", "public, max-age=300"), + ("vary", "accept-encoding"), + ], + ); - // Fill the cache legitimately, then corrupt the stored body in place so the - // entry is found under exactly the key the next request derives. - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - { - let stored_key = cache - .stored_keys - .lock() - .expect("should lock stored keys") - .first() - .cloned() - .expect("the cold request should have stored a template"); - let mut entries = cache.entries.lock().expect("should lock entries"); - let entry = entries - .get_mut(&stored_key.to_cache_key()) - .expect("the stored template should be readable"); - entry.body = format!( - "origin{AD_ASSEMBLY_SEAM}\ - {AD_ASSEMBLY_SEAM}" - ) - .into_bytes(); - } + let mut gzip_request = navigation_request(); + gzip_request + .headers_mut() + .insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + let cold = run(&settings, &services, gzip_request).await; + assert_eq!( + cold.headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("gzip") + ); + assert!( + cold.headers().get_all(header::VARY).iter().any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|name| name.trim().eq_ignore_ascii_case("accept-encoding")) + }) + }), + "the encoded cold response must declare reader negotiation" + ); + let cold_decoded = gzip_decode(&body_of(cold).await); - // The fall-back must be able to reach the origin. - queue_shareable_html(&stub); - let response = run(&settings, &services, navigation_request()).await; - let status = response.status(); + let warm_identity = run(&settings, &services, navigation_request()).await; + assert!( + warm_identity + .headers() + .get(header::CONTENT_ENCODING) + .is_none(), + "a reader that did not advertise compression must receive identity" + ); + let warm_identity_body = body_of(warm_identity).await; + + let mut warm_gzip_request = navigation_request(); + warm_gzip_request + .headers_mut() + .insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + let warm_gzip = run(&settings, &services, warm_gzip_request).await; assert_eq!( - response + warm_gzip .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) + .get(header::CONTENT_ENCODING) .and_then(|value| value.to_str().ok()), - Some("invalid"), - "an invalid-object recovery must be observable without exposing its key" + Some("gzip") ); - let served = String::from_utf8(body_of(response).await) - .expect("served document should be UTF-8"); - assert_eq!( - status, - StatusCode::OK, - "the reader should get a complete page, not a committed-then-broken one" + warm_gzip + .headers() + .get(header::VARY) + .and_then(|value| value.to_str().ok()), + Some("Accept-Encoding") + ); + let warm_gzip_decoded = gzip_decode(&body_of(warm_gzip).await); + + let mut warm_brotli_request = navigation_request(); + warm_brotli_request.headers_mut().insert( + header::ACCEPT_ENCODING, + HeaderValue::from_static("br, gzip;q=0.5"), ); + let warm_brotli = run(&settings, &services, warm_brotli_request).await; assert_eq!( - stub.recorded_request_uris().len(), - 2, - "an unusable entry must be treated as a miss and refetched" + warm_brotli + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("br") + ); + let warm_brotli_decoded = brotli_decode(&body_of(warm_brotli).await); + + assert_eq!(stub.recorded_request_uris().len(), 1); + assert_eq!(cold_decoded, warm_identity_body); + assert_eq!(warm_gzip_decoded, warm_identity_body); + assert_eq!(warm_brotli_decoded, warm_identity_body); + } + + #[tokio::test] + async fn esi_unsupported_reader_encoding_still_injects_tsjs() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + let mut request = navigation_request(); + request.headers_mut().insert( + header::ACCEPT_ENCODING, + HeaderValue::from_static("zstd, gzip;q=0, deflate;q=0, br;q=0, identity;q=0"), ); + + let body = String::from_utf8(body_of(run(&settings, &services, request).await).await) + .expect("should return UTF-8 HTML"); + + let forwarded = stub.recorded_request_headers(); assert!( - !served.contains(AD_ASSEMBLY_SEAM), - "no marker may survive into the served page: {served}" + forwarded[0].iter().any(|(name, value)| { + name.eq_ignore_ascii_case("accept-encoding") && value == "identity" + }), + "should force an identity origin representation when no reader encoding is \ + available for assembly" ); assert!( - served.contains("window.tsjs"), - "and the fallback must still deliver the seam: {served}" + body.contains("window.tsjs"), + "should preserve TSJS injection while bypassing shared assembly" ); assert!( cache - .entries + .lookups .lock() - .expect("should lock entries") + .expect("should lock lookups") .is_empty(), - "an unusable fresh object must be purged instead of forcing inline fallback \ - until its TTL expires" + "should bypass template cache when the reader accepts no supported representation" ); } - /// The schema version whose templates carried an executable ESI tag at the seam. - /// - /// Pinned as a literal rather than derived from [`TEMPLATE_SCHEMA_VERSION`]: the - /// point is that the current version is *not* this one, and a derived value - /// would move with it and assert nothing. - const ESI_INCLUDE_SCHEMA_VERSION: u32 = 1; - - /// The exact schema-v1 marker retained only as a cache-compatibility fixture. - const LEGACY_ESI_INCLUDE: &str = ""; - #[tokio::test] - async fn a_template_written_under_the_previous_schema_version_is_never_read() { - // v1 put an executable ESI include at the seam. v2 puts an inert comment - // there and hands slots to the scheduler, so a v1 entry has no marker this - // binary can find. A real v1 entry also carries an older build digest, but - // this fixture reuses the current fingerprint, so here `schema_version` is - // the only thing keeping the two apart. Nothing purges template cache on - // deploy. - assert_ne!( - crate::platform::TEMPLATE_SCHEMA_VERSION, - ESI_INCLUDE_SCHEMA_VERSION, - "the seam marker changed shape, so the schema version must have moved \ - off the value under which the old marker was stored" - ); - + async fn an_unsupported_origin_encoding_bypasses_template_cache_without_relabeling_the_body() + { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - // Fill the cache the ordinary way, then plant a predecessor's entry - // alongside it: same request, previous schema version, previous marker. - // Re-keyed from the key the store actually used, so the fixture cannot - // drift from what the request derives. - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - { - let stored_key = cache - .stored_keys - .lock() - .expect("should lock stored keys") - .first() - .cloned() - .expect("the cold request should have stored a template"); - let mut entries = cache.entries.lock().expect("should lock entries"); - let mut predecessor = entries - .get(&stored_key.to_cache_key()) - .cloned() - .expect("the stored template should be readable"); - predecessor.body = - format!("origin{LEGACY_ESI_INCLUDE}") - .into_bytes(); - predecessor.metadata.schema_version = ESI_INCLUDE_SCHEMA_VERSION; - let mut old_key = stored_key; - old_key.schema_version = ESI_INCLUDE_SCHEMA_VERSION; - entries.insert(old_key.to_cache_key(), predecessor); - } - - let served = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("served document should be UTF-8"); + let wire_body = b"opaque-zstd-wire-bytes".to_vec(); + stub.push_response_with_headers( + 200, + wire_body.clone(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-encoding", "zstd"), + ("cache-control", "public, max-age=300"), + ], + ); + let response = run(&settings, &services, navigation_request()).await; + assert_eq!( + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("zstd"), + "the pass-through bytes must keep the origin's representation label" + ); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-response") + ); + assert_eq!(body_of(response).await, wire_body); assert!( cache - .lookups + .entries .lock() - .expect("should lock lookups") - .iter() - .all(|key| key.schema_version != ESI_INCLUDE_SCHEMA_VERSION), - "no lookup may name a schema version whose templates this binary cannot \ - assemble" - ); - assert!( - !served.contains(", name: header::HeaderName) -> Option<&str> { + response.headers().get(name).and_then(|v| v.to_str().ok()) + } - // Bid first, page second: one FIFO queue, and the auction is dispatched - // before the origin fetch. - queue_bid_response(&stub); - queue_shareable_html(&stub); + /// The bid payload the seam actually embedded, decoded from the served page. + /// + /// Deliberately **not** written as the inverse of [`html_escape_for_script`]: the + /// escaper emits `\"`, `\\` and `\uXXXX`, which are all JSON string escapes, so + /// re-quoting the payload and handing it to `serde_json`'s own string parser + /// decodes it without this test agreeing with the encoder about anything. + /// + /// Panics rather than returning an error, because a served page with no decodable + /// bid payload is the failure under test and its diagnostics belong at the + /// assertion. + fn seam_bids(document: &str) -> serde_json::Map { + const OPEN: &str = "var b=JSON.parse(\""; + // The seam's fixed shape. A bare `")` could occur inside the payload; this + // nine-byte terminator cannot. + const CLOSE: &str = "\");var s="; + let start = document + .find(OPEN) + .unwrap_or_else(|| panic!("the seam must carry a bids payload: {document}")); + let rest = &document[start + OPEN.len()..]; + let end = rest + .find(CLOSE) + .unwrap_or_else(|| panic!("the bids payload must terminate: {document}")); + let unescaped: String = serde_json::from_str(&format!("\"{}\"", &rest[..end])) + .expect("the embedded payload should be a valid JSON string literal"); + serde_json::from_str(&unescaped).expect("the embedded payload should be a JSON object") + } + + #[test] + fn the_seam_payload_replaces_the_transform_owned_placeholder() { + let document = + format!(r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"#).into_bytes(); + + let replaced = replace_seam_placeholder(document, b"") + .expect("should substitute the placeholder the transform emitted"); - let cold = String::from_utf8( - body_of(run_bidding(&settings, &services, navigation_request()).await).await, - ) - .expect("cold document should be UTF-8"); - let cold_bids = seam_bids(&cold); - assert_eq!( - cold_bids - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some("3.50"), - "the miss path must deliver the winning bid's bucketed price, not an \ - empty map: {cold}" - ); assert_eq!( - cold_bids - .get("test-slot") - .and_then(|bid| bid.get("hb_bidder")) - .and_then(serde_json::Value::as_str), - Some(STUB_BIDDER), - "the miss path must name the winning bidder: {cold}" + replaced, br#"

article

"#, + "should splice the payload exactly where the parser marked the body end" ); + } - // The warm request needs only the bid response: a hit skips the origin - // entirely, so an unused page fixture here would mask a fetch. - queue_bid_response(&stub); - let warm = String::from_utf8( - body_of(run_bidding(&settings, &services, navigation_request()).await).await, + #[test] + fn a_body_close_written_in_script_data_does_not_attract_the_seam() { + // The reverse byte search this replaced picked the string literal, spliced a + // `` terminated the + // publisher's script — in the served page and in the stored template alike. + let document = format!( + r#"

article

{TEMPLATE_SEAM_PLACEHOLDER}"# ) - .expect("warm document should be UTF-8"); + .into_bytes(); + + let replaced = replace_seam_placeholder(document, b"") + .expect("should substitute the placeholder the transform emitted"); + assert_eq!( - stub.recorded_request_uris() - .iter() - .filter(|uri| uri.contains("/article")) - .count(), - 1, - "the warm request must be served from the template cache" + replaced, + br#"

article

"#, + "should leave a body-close sequence inside script data untouched" ); - let warm_bids = seam_bids(&warm); - assert_eq!( - warm_bids - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some("3.50"), - "the hit path must deliver this reader's winning bid, not an empty \ - map: {warm}" + } + + #[test] + fn a_publisher_copy_of_the_placeholder_refuses_substitution() { + let document = format!( + "

{TEMPLATE_SEAM_PLACEHOLDER}

article{TEMPLATE_SEAM_PLACEHOLDER}" + ) + .into_bytes(); + + let (returned, error) = + replace_seam_placeholder(document.clone(), b"") + .expect_err("should refuse a document that collides with the placeholder"); + + assert!( + matches!(error, SeamError::Repeated), + "should name the collision rather than guess an occurrence" ); assert_eq!( - warm_bids, cold_bids, - "both paths must splice the same bids for the same auction" + returned, document, + "should hand back the publisher document byte for byte" ); } - #[tokio::test] - async fn max_age_zero_reload_reuses_a_warm_template_and_runs_a_fresh_auction() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_bidder("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - - queue_bid_response(&stub); - queue_shareable_html(&stub); - let cold = run_bidding(&settings, &services, navigation_request()).await; - assert_eq!( - cold.headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("miss-stored") - ); - let _ = body_of(cold).await; + #[test] + fn a_document_without_the_placeholder_refuses_substitution() { + let document = b"

article

".to_vec(); - // Keep an origin response available so the old bypass behavior completes - // normally and fails on the observable cache state rather than a fixture - // underflow. A correct warm reload leaves this response untouched. - queue_bid_response(&stub); - queue_shareable_html(&stub); - let mut reload = navigation_request(); - reload - .headers_mut() - .insert(header::CACHE_CONTROL, HeaderValue::from_static("max-age=0")); - let warm = run_bidding(&settings, &services, reload).await; - assert_eq!( - warm.headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("hit"), - "a browser reload must re-run per-reader assembly without refetching the \ - reader-neutral template" - ); - assert_eq!( - warm.headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "reusing template cache must not make the assembled response browser-cacheable" - ); - let warm = String::from_utf8(body_of(warm).await) - .expect("warm reload document should be UTF-8"); + let (returned, error) = + replace_seam_placeholder(document.clone(), b"") + .expect_err("should refuse a document the transform did not mark"); - let requests = stub.recorded_request_uris(); - assert_eq!( - requests - .iter() - .filter(|uri| uri.contains("/article")) - .count(), - 1, - "the reload must reuse the warm article template" + assert!( + matches!(error, SeamError::Missing), + "should name the absent placeholder rather than append blindly" ); assert_eq!( - stub.recorded_backend_names() - .iter() - .filter(|backend| backend.as_str() == "stub-bidder-backend") - .count(), - 2, - "each navigation must dispatch its own auction even when template cache hits" + returned, document, + "should hand back the document unchanged" ); - assert_eq!( - seam_bids(&warm) - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some("3.50"), - "the reload's auction result must reach the assembled seam" + } + + #[test] + fn parser_validation_does_not_change_the_cached_schema() { + assert_eq!(crate::platform::TEMPLATE_SCHEMA_VERSION, 5); + assert_eq!(AD_ASSEMBLY_SEAM, ""); + assert!(!contains_publisher_esi_directive( + AD_ASSEMBLY_SEAM.as_bytes() + )); + } + + /// Shareable HTML that already contains the seam marker. + /// + /// The marker is reserved, but publisher content can still contain it. The + /// transform adds its own terminal placeholder; repeated markers then make the + /// response bypass the template cache rather than requiring normalization. + fn queue_html_that_collides_with_the_marker(stub: &StubHttpClient) { + stub.push_response_with_headers( + 200, + format!( + "

origin

" + ) + .into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], ); } #[tokio::test] - async fn the_marker_never_reaches_the_browser_on_either_path() { - // The user-visible failure this whole chain exists to avoid: a page that - // returns 200, parses fine, renders no ads, and reports no error, because - // the legacy executable ESI tag was served literally. - // - // Both paths are checked. The miss path assembles after the transform; the - // hit path assembles after reading the cache. They are separate call sites - // and only one of them running would still look like success on the other. + async fn origin_marker_collision_bypasses_template_cache_without_mutating_publisher_bytes() + { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); + queue_html_that_collides_with_the_marker(&stub); + queue_html_that_collides_with_the_marker(&stub); let cold = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, ) - .expect("cold response should be utf-8"); + .expect("cold document should be UTF-8"); let warm = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, ) - .expect("warm response should be utf-8"); + .expect("warm document should be UTF-8"); - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "the second request must be a hit, or this only tests one path" + assert!( + cold.contains("origin") && cold.contains("window.tsjs"), + "a reserved-comment collision must not turn a valid origin 200 into a 500" ); - for (label, document) in [("miss", &cold), ("hit", &warm)] { - assert!( - !document.contains(AD_ASSEMBLY_SEAM), - "{label}: an unresolved marker reached the browser: {document}" - ); + for document in [&cold, &warm] { assert!( - document.contains("window.tsjs"), - "{label}: assembly must leave a bids script behind: {document}" + document.contains(&format!("window.publisherMarker=\"{AD_ASSEMBLY_SEAM}\"")), + "should preserve a publisher marker inside script data: {document}" ); } + assert_eq!(stub.recorded_request_uris().len(), 2); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "should not store a template with a publisher marker collision" + ); } #[tokio::test] - async fn the_cached_template_holds_the_marker_and_never_the_bids() { - // Store the reader-neutral template before assembling the final per-user - // response, which must never enter the forbidden assembled-response cache. If - // these were swapped, the cache would hold one visitor's bids and serve them - // to the next — and every test above would still pass, because the served - // page would look correct. + async fn html_without_an_explicit_body_gets_a_terminal_seam_instead_of_a_500() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, navigation_request()).await; + stub.push_response_with_headers( + 200, + b"
origin fragment
".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); - let entries = cache.entries.lock().expect("should lock entries"); - let stored = entries - .values() - .next() - .expect("a template should be stored"); - let template = core::str::from_utf8(&stored.body).expect("template should be utf-8"); + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("cold fragment should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("warm fragment should be UTF-8"); + for document in [&cold, &warm] { + assert!(document.contains("origin fragment")); + assert!(!document.contains(AD_ASSEMBLY_SEAM)); + } + assert_eq!(stub.recorded_request_uris().len(), 1); assert!( - template.contains(AD_ASSEMBLY_SEAM), - "the cached template must still carry the unresolved marker: {template}" - ); - assert!( - !template.contains("window.tsjs"), - "the cached template must not carry a bids script: {template}" + !cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "the recovered template should remain cacheable" ); } #[tokio::test] - async fn nextjs_stored_templates_are_identical_across_request_shapes() { - let mut settings = settings_with_mode("esi"); - settings.publisher.origin_url = "https://origin.example.com".to_owned(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), + async fn an_unsplittable_cached_template_is_a_miss_before_any_header_commits() { + // The hit path used to check only that a marker existed *somewhere* and + // leave the exactly-one check to the finalizer. By then the 200 and its + // headers were committed and, on the streaming adapter, the document head + // was already on the wire — so the only available failure was a truncated + // response. Falling back to the origin is a slower correct page. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + // Fill the cache legitimately, then corrupt the stored body in place so the + // entry is found under exactly the key the next request derives. + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + { + let stored_key = cache + .stored_keys + .lock() + .expect("should lock stored keys") + .first() + .cloned() + .expect("the cold request should have stored a template"); + let mut entries = cache.entries.lock().expect("should lock entries"); + let entry = entries + .get_mut(&stored_key.to_cache_key()) + .expect("the stored template should be readable"); + entry.body = format!( + "origin{AD_ASSEMBLY_SEAM}\ + {AD_ASSEMBLY_SEAM}" ) - .expect("should enable Next.js processing"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); - assert!( - !registry.html_stream_processor_factories().is_empty(), - "should exercise the Next.js stream processor" + .into_bytes(); + } + + // The fall-back must be able to reach the origin. + queue_shareable_html(&stub); + let response = run(&settings, &services, navigation_request()).await; + let status = response.status(); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("invalid"), + "an invalid-object recovery must be observable without exposing its key" ); - let settings = Arc::new(settings); - let html = br#"
"#; - - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let mut templates = Vec::new(); - for request in [navigation_request(), prefetch_navigation_request()] { - // Independent fills force distinct per-request RSC namespaces. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - html.to_vec(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - - let response = run_via(&settings, &services, request, finalizer).await; - let _ = body_of(response).await; - - let entries = cache.entries.lock().expect("should lock stored templates"); - assert_eq!( - entries.len(), - 1, - "should store a Next.js template on every fill" - ); - let template = entries - .values() - .next() - .expect("should find stored template") - .body - .clone(); - let text = core::str::from_utf8(&template).expect("should store UTF-8 HTML"); - assert!( - text.contains("ts.example.com/page"), - "should rewrite the RSC URL" - ); - assert!( - text.contains(AD_ASSEMBLY_SEAM), - "should store before per-reader assembly" - ); - for marker in [ - "__ts_rsc_", - ".adSlots", - ".bids=", - "gpt-diagnostics", - "gpt_history", - ] { - assert!( - !text.contains(marker), - "should exclude request-scoped marker {marker}" - ); - } - templates.push(template); - } - assert_eq!( - templates[0], templates[1], - "should store identical bytes across request shapes and RSC namespaces" - ); - } - } - - #[tokio::test] - async fn a_transform_that_overruns_its_buffer_stores_nothing() { - // The 16 MB cap in production, shrunk here. A partial template in template cache is the - // worst outcome available: it would be served to every subsequent visitor as - // a truncated document, indefinitely, with no error after the first request. - // - // Safe by construction — the cap error propagates before the store runs — but - // "by construction" is exactly the kind of claim that stops being true after - // an unrelated refactor moves one line. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.publisher.max_buffered_body_bytes = 8; - let settings = Arc::new(raw); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test(None, consent); - let publisher_response = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - navigation_request(), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("the request itself should succeed; the cap trips during streaming"); - - let result = publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(&settings), - ®istry, - orchestrator, - services.clone(), - ) - .await; + let served = String::from_utf8(body_of(response).await) + .expect("served document should be UTF-8"); + assert_eq!( + status, + StatusCode::OK, + "the reader should get a complete page, not a committed-then-broken one" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "an unusable entry must be treated as a miss and refetched" + ); assert!( - result.is_err(), - "overrunning the buffer must fail rather than truncate" + !served.contains(AD_ASSEMBLY_SEAM), + "no marker may survive into the served page: {served}" + ); + assert!( + served.contains("window.tsjs"), + "and the fallback must still deliver the seam: {served}" ); assert!( cache @@ -12737,2949 +13916,3009 @@ mod tests { .lock() .expect("should lock entries") .is_empty(), - "a failed transform must leave nothing in the shared cache" + "an unusable fresh object must be purged instead of forcing inline fallback \ + until its TTL expires" ); } - #[tokio::test] - async fn a_cache_hit_is_never_shared_cacheable() { - // Asserted positively, because the obvious negative check does not work. - // Forbidding `public`, `s-maxage` and `Surrogate-Control` passes trivially - // on a response that carries no `Cache-Control` at all — and *that* is the - // real failure mode here, since a template cache hit returns before the point where the - // publisher path stamps the response private. HTML with no `Cache-Control` - // is heuristically cacheable, so "nothing to forbid" is not safety. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); + /// The schema version whose templates carried an executable ESI tag at the seam. + /// + /// Pinned as a literal rather than derived from [`TEMPLATE_SCHEMA_VERSION`]: the + /// point is that the current version is *not* this one, and a derived value + /// would move with it and assert nothing. + const ESI_INCLUDE_SCHEMA_VERSION: u32 = 1; - let _cold = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + /// The exact schema-v1 marker retained only as a cache-compatibility fixture. + const LEGACY_ESI_INCLUDE: &str = ""; - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "the second request must be a hit, or this asserts nothing" - ); - assert_eq!( - header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private"), - "an assembled response is per-user even when its template is not" + #[tokio::test] + async fn a_template_written_under_the_previous_schema_version_is_never_read() { + // v1 put an executable ESI include at the seam. v2 puts an inert comment + // there and hands slots to the scheduler, so a v1 entry has no marker this + // binary can find. A real v1 entry also carries an older build digest, but + // this fixture reuses the current fingerprint, so here `schema_version` is + // the only thing keeping the two apart. Nothing purges template cache on + // deploy. + assert_ne!( + crate::platform::TEMPLATE_SCHEMA_VERSION, + ESI_INCLUDE_SCHEMA_VERSION, + "the seam marker changed shape, so the schema version must have moved \ + off the value under which the old marker was stored" ); - // Validators would let a client revalidate into a shared copy, and the CDN - // directives would instruct an intermediary to store one outright. The - // origin fixture sends a `public, max-age=300` that must not survive. - for stripped in [header::ETAG, header::LAST_MODIFIED, header::EXPIRES] { - assert_eq!( - header_of(&warm, stripped.clone()), - None, - "{stripped} must not survive onto an assembled response" - ); - } - } - - #[tokio::test] - async fn a_returning_visitor_gets_the_same_privacy_headers() { - // The case with no backstop. A first-visit response sets an EC cookie, so - // the adapter's cookie-privacy net force-privatizes it regardless of what - // this path does. A returning visitor sets no cookie, so that net never - // fires and this path is the only thing standing between an assembled - // per-user document and a shared cache. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - let _cold = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + // Fill the cache the ordinary way, then plant a predecessor's entry + // alongside it: same request, previous schema version, previous marker. + // Re-keyed from the key the store actually used, so the fixture cannot + // drift from what the request derives. + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + { + let stored_key = cache + .stored_keys + .lock() + .expect("should lock stored keys") + .first() + .cloned() + .expect("the cold request should have stored a template"); + let mut entries = cache.entries.lock().expect("should lock entries"); + let mut predecessor = entries + .get(&stored_key.to_cache_key()) + .cloned() + .expect("the stored template should be readable"); + predecessor.body = + format!("origin{LEGACY_ESI_INCLUDE}") + .into_bytes(); + predecessor.metadata.schema_version = ESI_INCLUDE_SCHEMA_VERSION; + let mut old_key = stored_key; + old_key.schema_version = ESI_INCLUDE_SCHEMA_VERSION; + entries.insert(old_key.to_cache_key(), predecessor); + } + + let served = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("served document should be UTF-8"); assert!( - header_of(&warm, header::SET_COOKIE).is_none(), - "no cookie here means no privacy net, which is the point of this test" + cache + .lookups + .lock() + .expect("should lock lookups") + .iter() + .all(|key| key.schema_version != ESI_INCLUDE_SCHEMA_VERSION), + "no lookup may name a schema version whose templates this binary cannot \ + assemble" ); - assert_eq!( - header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private") + assert!( + !served.contains(", - ) -> Result, Report> { - Ok(Some(GeoInfo { - city: self.0.to_string(), - country: self.0.to_string(), - continent: self.0.to_string(), - latitude: 1.0, - longitude: 2.0, - metro_code: 3, - region: Some(self.0.to_string()), - asn: Some(4), - })) - } - } - - /// One synthetic user: an identity, a consent posture, and a location. - struct SyntheticUser { - ec_id: &'static str, - jurisdiction: crate::consent::jurisdiction::Jurisdiction, - geo_marker: &'static str, - } - - /// Runs one synthetic user against a fresh cache and returns the stored template. - /// - /// Fresh cache per user deliberately: the point is to compare what each *would* - /// store, so sharing a cache would let the first user's entry answer for the - /// second and the comparison would prove nothing. - async fn stored_template_for(user: &SyntheticUser) -> Vec { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(Arc::clone(&stub) as Arc) - .geo(Arc::new(StubGeo(user.geo_marker))) - .client_info(ClientInfo::default()) - .template_cache( - Arc::clone(&cache) as Arc + // A normal navigation fills the cache; only then is a hit available for + // the opted-out request, which is the whole point — the gate held on the + // miss and was ignored on the hit. + let cold = String::from_utf8( + body_of(run_via(&settings, &services, navigation_request(), finalizer).await) + .await, ) - .build(); - queue_shareable_html(&stub); - - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let consent = crate::consent::ConsentContext { - jurisdiction: user.jurisdiction.clone(), - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test(Some(user.ec_id.to_string()), consent); - let publisher_response = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - navigation_request(), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request"); - let _ = publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(&settings), - ®istry, - orchestrator, - services.clone(), - ) - .await - .expect("should finalize publisher response"); - - let entries = cache.entries.lock().expect("should lock entries"); - entries - .values() - .next() - .expect("a template should have been stored") - .body - .clone() - } - - #[tokio::test] - async fn two_users_differing_in_identity_consent_and_geo_store_the_same_template() { - // The gate the whole design rests on. The template is shared between - // visitors, so anything request-scoped that reaches it is one visitor's data - // served to the next. Byte-identity is the assertion because it does not - // depend on guessing which field might leak. - let alice = SyntheticUser { - ec_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.alice1", - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - geo_marker: "AliceCity", - }; - let bob = SyntheticUser { - ec_id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.bobbb1", - jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, - geo_marker: "BobCity", - }; - - let alice_template = stored_template_for(&alice).await; - let bob_template = stored_template_for(&bob).await; + .expect("cold document should be UTF-8"); + assert!( + cold.contains("scheduleInitialAdInit"), + "the fixture must produce a seam for ordinary traffic, or this test \ + passes for the wrong reason: {cold}" + ); - assert_eq!( - alice_template, bob_template, - "two users differing in identity, consent and geo must produce the same \ - shared template" - ); + let served = String::from_utf8( + body_of( + run_via( + &settings, + &services, + prefetch_navigation_request(), + finalizer, + ) + .await, + ) + .await, + ) + .expect("served document should be UTF-8"); - // Belt and braces: byte-identity would also hold if *both* templates leaked - // the same wrong thing, so name the values that must be absent. - let template = String::from_utf8(alice_template).expect("template should be utf-8"); - for forbidden in [ - alice.ec_id, - bob.ec_id, - alice.geo_marker, - bob.geo_marker, - "adSlots", - "window.tsjs", - ] { + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the opted-out request must be a cache hit, or the hit finalizer is \ + never reached" + ); assert!( - !template.contains(forbidden), - "`{forbidden}` must not appear in a shared template: {template}" + !served.contains("scheduleInitialAdInit"), + "a reader whose ad stack did not run must not have `adInit` \ + scheduled: {served}" + ); + assert!( + !served.contains("t.adSlots="), + "no slot assignment may reach opted-out traffic: {served}" + ); + assert!( + !served.contains(AD_ASSEMBLY_SEAM), + "the marker must still be consumed even when nothing replaces it: \ + {served}" + ); + assert!( + served.contains("origin"), + "the page itself must still be served: {served}" ); } } #[tokio::test] - async fn inline_mode_never_reads_or_writes_the_cache() { - // The shipped path. If this ever cached, per-user ad state would be shared - // between visitors — the exact failure the whole design exists to avoid. + async fn a_winning_bid_reaches_the_page_on_both_the_miss_and_the_hit_path() { + // The fixture every other test here lacks. With no registered provider the + // auction always resolves to `{}`, so a seam that discarded every non-empty + // bid map looked identical to one that worked — which is precisely how the + // shared modes shipped serving zero bids. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("inline")); + let settings = Arc::new(settings_with_bidder("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - let _ = run(&settings, &services, navigation_request()).await; + // Bid first, page second: one FIFO queue, and the auction is dispatched + // before the origin fetch. + queue_bid_response(&stub); + queue_shareable_html(&stub); + let cold = String::from_utf8( + body_of(run_bidding(&settings, &services, navigation_request()).await).await, + ) + .expect("cold document should be UTF-8"); + let cold_bids = seam_bids(&cold); assert_eq!( - stub.recorded_request_uris().len(), - 2, - "inline must fetch the origin every time" + cold_bids + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "the miss path must deliver the winning bid's bucketed price, not an \ + empty map: {cold}" ); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "inline must never write a shared template" + assert_eq!( + cold_bids + .get("test-slot") + .and_then(|bid| bid.get("hb_bidder")) + .and_then(serde_json::Value::as_str), + Some(STUB_BIDDER), + "the miss path must name the winning bidder: {cold}" ); - } - /// [`settings_with_mode`], with one integration configured a stated way. - /// - /// Edits the parsed `[integrations]` map rather than appending TOML, so two - /// fixtures differ in exactly the field under test. - fn settings_with_prebid_timeout(mode: &str, timeout_ms: u32) -> Settings { - let mut settings = settings_with_mode(mode); - settings.integrations.insert( - "prebid".to_string(), - serde_json::json!({ - "enabled": true, - "external_bundle_url": "https://assets.example.com/prebid/bundle.js", - "timeout_ms": timeout_ms, - }), + // The warm request needs only the bid response: a hit skips the origin + // entirely, so an unused page fixture here would mask a fetch. + queue_bid_response(&stub); + let warm = String::from_utf8( + body_of(run_bidding(&settings, &services, navigation_request()).await).await, + ) + .expect("warm document should be UTF-8"); + assert_eq!( + stub.recorded_request_uris() + .iter() + .filter(|uri| uri.contains("/article")) + .count(), + 1, + "the warm request must be served from the template cache" + ); + let warm_bids = seam_bids(&warm); + assert_eq!( + warm_bids + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "the hit path must deliver this reader's winning bid, not an empty \ + map: {warm}" + ); + assert_eq!( + warm_bids, cold_bids, + "both paths must splice the same bids for the same auction" ); - settings - } - - /// Every key the cache was asked to store, rendered. - fn stored_cache_keys(cache: &MemoryTemplateCache) -> Vec { - cache - .stored_keys - .lock() - .expect("should lock stored keys") - .iter() - .map(crate::platform::TemplateCacheKey::to_cache_key) - .collect() - } - - /// Every key the cache was asked to read, rendered. - fn looked_up_cache_keys(cache: &MemoryTemplateCache) -> Vec { - cache - .lookups - .lock() - .expect("should lock lookups") - .iter() - .map(crate::platform::TemplateCacheKey::to_cache_key) - .collect() } #[tokio::test] - async fn two_integration_configurations_never_share_a_template() { - // `template_fingerprint` folds the complete typed config, and its own - // tests call it directly — so reverting the *call site* back to the - // bundle-only hash, a constant for a given binary, left the entire suite - // green while every configuration silently shared one template. - // - // This drives the real request path and asserts on the keys the cache was - // actually handed, which is the only place that mutation is visible. + async fn max_age_zero_reload_reuses_a_warm_template_and_runs_a_fresh_auction() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_bidder("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let first = Arc::new(settings_with_prebid_timeout("esi", 1000)); - let second = Arc::new(settings_with_prebid_timeout("esi", 2500)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - let _ = run(&first, &services, navigation_request()).await; - let _ = run(&second, &services, navigation_request()).await; + queue_bid_response(&stub); + queue_shareable_html(&stub); + let cold = run_bidding(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("miss-stored") + ); + let _ = body_of(cold).await; - let stored = stored_cache_keys(&cache); + // Keep an origin response available so the old bypass behavior completes + // normally and fails on the observable cache state rather than a fixture + // underflow. A correct warm reload leaves this response untouched. + queue_bid_response(&stub); + queue_shareable_html(&stub); + let mut reload = navigation_request(); + reload + .headers_mut() + .insert(header::CACHE_CONTROL, HeaderValue::from_static("max-age=0")); + let warm = run_bidding(&settings, &services, reload).await; assert_eq!( - stored.len(), - 2, - "each configuration must store its own template, got {stored:?}" + warm.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("hit"), + "a browser reload must re-run per-reader assembly without refetching the \ + reader-neutral template" ); - assert_ne!( - stored[0], stored[1], - "two `[integrations]` configurations must key different templates; one key \ - serves the first configuration's injected markup to the second" + assert_eq!( + warm.headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "reusing template cache must not make the assembled response browser-cacheable" + ); + let warm = String::from_utf8(body_of(warm).await) + .expect("warm reload document should be UTF-8"); + + let requests = stub.recorded_request_uris(); + assert_eq!( + requests + .iter() + .filter(|uri| uri.contains("/article")) + .count(), + 1, + "the reload must reuse the warm article template" ); assert_eq!( - stub.recorded_request_uris().len(), + stub.recorded_backend_names() + .iter() + .filter(|backend| backend.as_str() == "stub-bidder-backend") + .count(), 2, - "the second configuration must have missed and fetched the origin rather \ - than reading the first's template" + "each navigation must dispatch its own auction even when template cache hits" + ); + assert_eq!( + seam_bids(&warm) + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "the reload's auction result must reach the assembled seam" ); } #[tokio::test] - async fn one_integration_configuration_keys_one_template() { - // The converse, and the failure mode a fingerprint fix can introduce: - // over-invalidating is as total as under-invalidating. A fingerprint that - // moves between two equal configurations is a cache that never hits, which - // this would read as "no measurable benefit" rather than as a bug. + async fn the_marker_never_reaches_the_browser_on_either_path() { + // The user-visible failure this whole chain exists to avoid: a page that + // returns 200, parses fine, renders no ads, and reports no error, because + // the legacy executable ESI tag was served literally. // - // The two `Settings` are parsed independently, so their `[integrations]` - // maps iterate in different orders — which is what exercises the sort. + // Both paths are checked. The miss path assembles after the transform; the + // hit path assembles after reading the cache. They are separate call sites + // and only one of them running would still look like success on the other. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let first = Arc::new(settings_with_prebid_timeout("esi", 1000)); - let second = Arc::new(settings_with_prebid_timeout("esi", 1000)); queue_shareable_html(&stub); - let _ = run(&first, &services, navigation_request()).await; - let _ = run(&second, &services, navigation_request()).await; + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("cold response should be utf-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("warm response should be utf-8"); - let looked_up = looked_up_cache_keys(&cache); - assert_eq!( - looked_up.len(), - 2, - "both requests must consult the cache, got {looked_up:?}" - ); - assert_eq!( - looked_up[0], looked_up[1], - "equal configurations must name one key, or the cache never hits" - ); assert_eq!( stub.recorded_request_uris().len(), 1, - "the second request must be served from the first's template" + "the second request must be a hit, or this only tests one path" ); + for (label, document) in [("miss", &cold), ("hit", &warm)] { + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "{label}: an unresolved marker reached the browser: {document}" + ); + assert!( + document.contains("window.tsjs"), + "{label}: assembly must leave a bids script behind: {document}" + ); + } } #[tokio::test] - async fn one_context_key_allowlist_keys_one_template() { - let mut source = serde_json::to_value(settings_with_mode("esi")) - .expect("should serialize cache settings"); - source["auction"]["allowed_context_keys"] = - serde_json::json!(["zeta", "alpha", "gamma", "beta", "epsilon", "delta"]); + async fn the_cached_template_holds_the_marker_and_never_the_bids() { + // Store the reader-neutral template before assembling the final per-user + // response, which must never enter the forbidden assembled-response cache. If + // these were swapped, the cache would hold one visitor's bids and serve them + // to the next — and every test above would still pass, because the served + // page would look correct. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); - for _ in 0..16 { - let settings = Arc::new( - serde_json::from_value(source.clone()).expect("should parse cache settings"), - ); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - } + let _ = run(&settings, &services, navigation_request()).await; + + let entries = cache.entries.lock().expect("should lock entries"); + let stored = entries + .values() + .next() + .expect("a template should be stored"); + let template = core::str::from_utf8(&stored.body).expect("template should be utf-8"); - let keys = looked_up_cache_keys(&cache); - assert_eq!(keys.len(), 16, "should consult the cache for every request"); assert!( - keys.iter().all(|key| key == &keys[0]), - "should reuse one key across independently parsed allowlists" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should store one template" + template.contains(AD_ASSEMBLY_SEAM), + "the cached template must still carry the unresolved marker: {template}" ); - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "should fetch the origin only for the first request" + assert!( + !template.contains("window.tsjs"), + "the cached template must not carry a bids script: {template}" ); } - fn cookie_navigation_request() -> Request { - HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .header("sec-fetch-mode", "navigate") - .header(header::COOKIE, "ts-ec=abc123") - .body(EdgeBody::empty()) - .expect("should build cookie-bearing request") - } - #[tokio::test] - async fn readthrough_never_collapses_template_cookie_variants() { - let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .origin_readthrough_enabled = Some(true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for arm in ["absent", "A", "B", "empty"] { - stub.push_response_with_headers( - 200, - format!("arm-{arm}").into_bytes(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - } - for (index, (cookie, arm)) in [ - (None, "absent"), - (Some("ab_bucket=A"), "A"), - (Some("ab_bucket=B"), "B"), - (Some("ab_bucket="), "empty"), - (Some("ab_bucket=A"), "A"), - (Some("ab_bucket=B"), "B"), - ] - .iter() - .enumerate() - { - let fields: Vec<&[u8]> = cookie.iter().map(|value| value.as_bytes()).collect(); - let response = run(&settings, &services, cookie_policy_request(&fields)).await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index < 4 { "miss-stored" } else { "hit" }, - "should still cache separate templates" - ); - let body = String::from_utf8(body_of(response).await).expect("should decode HTML"); - assert!( - body.contains(&format!("arm-{arm}")), - "should preserve each cookie variant" - ); - } - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass; 4], - "should never cache origin bytes without the template cookie dimensions, even when the key cookie is absent" + async fn two_pass_integration_templates_are_identical_across_request_shapes() { + let mut settings = settings_with_mode("esi"); + settings.publisher.origin_url = "https://origin.example.com".to_owned(); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, ); - assert_eq!( - stored_cache_keys(&cache).len(), - 4, - "should store each template variant" + let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + assert!( + !registry + .middleware_chain( + &settings.fetch, + crate::middleware::MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, + "/", + ) + .is_empty(), + "should exercise the stand-in's middleware, which supplies a stream processor" ); - } + let settings = Arc::new(settings); + let html = br#"
"#; - #[tokio::test] - async fn readthrough_respects_named_session_bypass_without_disabling_anonymous_sharing() { - let mut settings = cookie_policy_settings(None, Some(&["session"]), true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .origin_readthrough_enabled = Some(true); + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut templates = Vec::new(); + for request in [navigation_request(), prefetch_navigation_request()] { + // Independent fills force distinct per-document namespaces. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + html.to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + + let response = run_via(&settings, &services, request, finalizer).await; + let _ = body_of(response).await; + + let entries = cache.entries.lock().expect("should lock stored templates"); + assert_eq!(entries.len(), 1, "should store a template on every fill"); + let template = entries + .values() + .next() + .expect("should find stored template") + .body + .clone(); + let text = core::str::from_utf8(&template).expect("should store UTF-8 HTML"); + assert!( + text.contains("ts.example.com/page"), + "should rewrite the URL in the payload" + ); + assert!( + text.contains(AD_ASSEMBLY_SEAM), + "should store before per-reader assembly" + ); + for marker in [ + crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX, + ".adSlots", + ".bids=", + "gpt-diagnostics", + "gpt_history", + ] { + assert!( + !text.contains(marker), + "should exclude request-scoped marker {marker}" + ); + } + templates.push(template); + } + assert_eq!( + templates[0], templates[1], + "should store identical bytes across request shapes and per-document namespaces" + ); + } + } + + #[tokio::test] + async fn a_transform_that_overruns_its_buffer_stores_nothing() { + // The 16 MB cap in production, shrunk here. A partial template in template cache is the + // worst outcome available: it would be served to every subsequent visitor as + // a truncated document, indefinitely, with no error after the first request. + // + // Safe by construction — the cap error propagates before the store runs — but + // "by construction" is exactly the kind of claim that stops being true after + // an unrelated refactor moves one line. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + let mut raw = settings_with_mode("esi"); + raw.publisher.max_buffered_body_bytes = 8; + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); + + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let publisher_response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + navigation_request(), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("the request itself should succeed; the cap trips during streaming"); + + let result = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + services.clone(), + ) + .await; + + assert!( + result.is_err(), + "overrunning the buffer must fail rather than truncate" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "a failed transform must leave nothing in the shared cache" + ); + } + + #[tokio::test] + async fn a_cache_hit_is_never_shared_cacheable() { + // Asserted positively, because the obvious negative check does not work. + // Forbidding `public`, `s-maxage` and `Surrogate-Control` passes trivially + // on a response that carries no `Cache-Control` at all — and *that* is the + // real failure mode here, since a template cache hit returns before the point where the + // publisher path stamps the response private. HTML with no `Cache-Control` + // is heuristically cacheable, so "nothing to forbid" is not safety. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - let _ = run(&settings, &services, cookie_policy_request(&[])).await; - let response = run(&settings, &services, cookie_policy_request(&[b"session="])).await; + + let _cold = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should not read an anonymous template for a session" + stub.recorded_request_uris().len(), + 1, + "the second request must be a hit, or this asserts nothing" ); assert_eq!( - stub.recorded_cache_intents(), - vec![ - PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - reader_url_surrogate_key("http://ts.example.com/article") - ) - }, - PlatformCacheIntent::Bypass, - ], - "should share anonymous bytes while refusing even an empty named session" + header_of(&warm, header::CACHE_CONTROL), + Some("no-store, private"), + "an assembled response is per-user even when its template is not" ); + + // Validators would let a client revalidate into a shared copy, and the CDN + // directives would instruct an intermediary to store one outright. The + // origin fixture sends a `public, max-age=300` that must not survive. + for stripped in [header::ETAG, header::LAST_MODIFIED, header::EXPIRES] { + assert_eq!( + header_of(&warm, stripped.clone()), + None, + "{stripped} must not survive onto an assembled response" + ); + } } #[tokio::test] - async fn readthrough_preload_bypasses_configured_cookie_dimensions() { - let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - let config = Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities"); - config.origin_readthrough_enabled = Some(true); - config.assembly_mode = Some(AssemblyMode::Inline); + async fn a_returning_visitor_gets_the_same_privacy_headers() { + // The case with no backstop. A first-visit response sets an EC cookie, so + // the adapter's cookie-privacy net force-privatizes it regardless of what + // this path does. A returning visitor sets no cookie, so that net never + // fires and this path is the only thing standing between an assembled + // per-user document and a shared cache. let stub = Arc::new(StubHttpClient::new()); - stub.set_pending_streaming_responses_supported(true); - let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - run_through_ec_preload( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await; + + let _cold = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + + assert!( + header_of(&warm, header::SET_COOKIE).is_none(), + "no cookie here means no privacy net, which is the point of this test" + ); assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], - "should apply the cookie-dimension exclusion on the pending origin path too" + header_of(&warm, header::CACHE_CONTROL), + Some("no-store, private") ); } - fn cookie_policy_settings( - key_names: Option<&[&str]>, - bypass_names: Option<&[&str]>, - independent: bool, - ) -> Arc { - let mut settings = settings_with_mode("esi"); - let config = settings - .creative_opportunities - .as_mut() - .expect("should configure opportunities"); - config.template_cache_key_cookies = - key_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); - config.template_cache_bypass_cookies = - bypass_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); - config.origin_is_cookie_independent = Some(independent); - config - .validate_runtime() - .expect("should validate cookie policy fixture"); - Arc::new(settings) - } + /// Geo that reports a fixed, recognizable location. + /// + /// A distinct value per synthetic user, so a geo leak into the template shows up + /// as a substring rather than requiring inference. + struct StubGeo(&'static str); - fn cookie_policy_request(fields: &[&[u8]]) -> Request { - let mut request = navigation_request(); - for field in fields { - request.headers_mut().append( - header::COOKIE, - HeaderValue::from_bytes(field).expect("should build cookie field"), - ); + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformGeo for StubGeo { + async fn lookup( + &self, + _client_ip: Option, + _services: &crate::platform::RuntimeServices, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: self.0.to_string(), + country: self.0.to_string(), + continent: self.0.to_string(), + latitude: 1.0, + longitude: 2.0, + metro_code: 3, + region: Some(self.0.to_string()), + asn: Some(4), + })) } - request } - // Exercise raw fields at the prepared-request boundary. Ordinary diagnostics - // preparation removes invalid fields/empty pairs before generic cookie handling. - fn prepared_cookie_policy_request(fields: &[&[u8]]) -> Request { - let mut request = cookie_policy_request(fields); - request.extensions_mut().insert( - crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision::default(), - ); - request + /// One synthetic user: an identity, a consent posture, and a location. + struct SyntheticUser { + ec_id: &'static str, + jurisdiction: crate::consent::jurisdiction::Jurisdiction, + geo_marker: &'static str, } - #[tokio::test] - async fn template_cookie_publisher_key_only_admits_listed_cookies() { - for unused in [None, Some([].as_slice())] { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), unused, false); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - for _ in 0..2 { - let _ = body_of( - run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await, - ) - .await; - } - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "should share a listed-only variant even with independence false" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should store one variant" - ); - let lookups = looked_up_cache_keys(&cache).len(); - let _ = body_of( - run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A; unknown=1"]), - ) - .await, + /// Runs one synthetic user against a fresh cache and returns the stored template. + /// + /// Fresh cache per user deliberately: the point is to compare what each *would* + /// store, so sharing a cache would let the first user's entry answer for the + /// second and the comparison would prove nothing. + async fn stored_template_for(user: &SyntheticUser) -> Vec { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(Arc::clone(&stub) as Arc) + .geo(Arc::new(StubGeo(user.geo_marker))) + .client_info(ClientInfo::default()) + .template_cache( + Arc::clone(&cache) as Arc ) - .await; - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should bypass for any unlisted cookie" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should not store an unlisted-cookie response" - ); - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "should fetch unlisted-cookie origin HTML" - ); - } + .build(); + queue_shareable_html(&stub); + + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: user.jurisdiction.clone(), + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(Some(user.ec_id.to_string()), consent); + let publisher_response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + navigation_request(), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + let _ = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + services.clone(), + ) + .await + .expect("should finalize publisher response"); + + let entries = cache.entries.lock().expect("should lock entries"); + entries + .values() + .next() + .expect("a template should have been stored") + .body + .clone() } #[tokio::test] - async fn template_cookie_publisher_session_bypasses_warm_and_cold_cache() { - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - for unused in [None, Some([].as_slice()), Some(["ab_bucket"].as_slice())] { - for warm in [false, true] { - let settings = cookie_policy_settings(unused, Some(&["session"]), true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - if warm { - queue_shareable_html(&stub); - let _ = body_of( - run_via( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader"]), - finalizer, - ) - .await, - ) - .await; - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should warm an anonymous template first" - ); - } - let lookups = looked_up_cache_keys(&cache).len(); - let stores = stored_cache_keys(&cache).len(); - for session in [b"session=".as_slice(), b"session=token".as_slice()] { - stub.push_response_with_headers( - 200, - b"personal-account".to_vec(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - let response = run_via( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader", session]), - finalizer, - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should report request bypass" - ); - assert!( - !response.headers().contains_key(HEADER_X_TS_ASSEMBLY), - "should omit shared-assembly diagnostics on inline responses" - ); - assert!( - String::from_utf8(body_of(response).await) - .expect("should decode HTML") - .contains("personal-account"), - "should render this request's origin HTML" - ); - } - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should never look up or reserve for session requests" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - stores, - "should never store session HTML" - ); - assert_eq!( - stub.recorded_request_uris().len(), - usize::from(warm) + 2, - "should fetch every session request" - ); - assert!( - cache - .entries - .lock() - .expect("should lock templates") - .values() - .all(|entry| !String::from_utf8_lossy(&entry.body) - .contains("personal-account")), - "should keep personal bytes out of templates" - ); - } - } + async fn two_users_differing_in_identity_consent_and_geo_store_the_same_template() { + // The gate the whole design rests on. The template is shared between + // visitors, so anything request-scoped that reaches it is one visitor's data + // served to the next. Byte-identity is the assertion because it does not + // depend on guessing which field might leak. + let alice = SyntheticUser { + ec_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.alice1", + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + geo_marker: "AliceCity", + }; + let bob = SyntheticUser { + ec_id: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.bobbb1", + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + geo_marker: "BobCity", + }; + + let alice_template = stored_template_for(&alice).await; + let bob_template = stored_template_for(&bob).await; + + assert_eq!( + alice_template, bob_template, + "two users differing in identity, consent and geo must produce the same \ + shared template" + ); + + // Belt and braces: byte-identity would also hold if *both* templates leaked + // the same wrong thing, so name the values that must be absent. + let template = String::from_utf8(alice_template).expect("template should be utf-8"); + for forbidden in [ + alice.ec_id, + bob.ec_id, + alice.geo_marker, + bob.geo_marker, + "adSlots", + "window.tsjs", + ] { + assert!( + !template.contains(forbidden), + "`{forbidden}` must not appear in a shared template: {template}" + ); } } #[tokio::test] - async fn template_cookie_publisher_downstream_variants_remain_separate() { - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let mut settings = - cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for arm in ["A", "B"] { - stub.push_response_with_headers( - 200, - format!("arm-{arm}").into_bytes(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ("vary", "X-Exp-Variant"), - ], - ); - } - for (index, arm) in ["A", "B", "A", "B"].iter().enumerate() { - let cookies = format!( - "ab_bucket={arm}; ts-ec=reader{index}; ignored=scope{index}; ignored=other{index}" - ); - let request = cookie_policy_request(&[cookies.as_bytes()]); - assert!( - !request.headers().contains_key("x-exp-variant"), - "should reproduce downstream-only header topology" - ); - let response = run_via(&settings, &services, request, finalizer).await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index < 2 { "miss-stored" } else { "hit" }, - "should share within each arm" - ); - assert!( - response.headers()[header::CACHE_CONTROL] - .to_str() - .expect("should read cache control") - .contains("private"), - "should keep assembled output private" - ); - let html = - String::from_utf8(body_of(response).await).expect("should decode HTML"); - assert!( - html.contains(&format!("arm-{arm}")), - "should render the correct experiment arm" - ); - assert!( - !html.contains(if *arm == "A" { "arm-B" } else { "arm-A" }), - "should never cross-serve arms" - ); - } - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "should skip origin on both warm arms" - ); - let stored = stored_cache_keys(&cache); - assert_eq!(stored.len(), 2, "should store two variants"); - assert_ne!(stored[0], stored[1], "should distinguish arms in the key"); - } + async fn inline_mode_never_reads_or_writes_the_cache() { + // The shipped path. If this ever cached, per-user ad state would be shared + // between visitors — the exact failure the whole design exists to avoid. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("inline")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + let _ = run(&settings, &services, navigation_request()).await; + + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "inline must fetch the origin every time" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "inline must never write a shared template" + ); + } + + /// [`settings_with_mode`], with one integration, core's stand-in, + /// configured a stated way. + /// + /// Edits the parsed settings rather than appending TOML, so two + /// fixtures differ in exactly the field under test. + fn settings_with_integration_timeout(mode: &str, timeout_ms: u32) -> Settings { + let mut settings = settings_with_mode(mode); + settings + .insert_module_config( + "testing", + crate::integrations::registry_test_support::deferred_fixture::MODULE, + &serde_json::json!({ + "label": "example", + "timeout_ms": timeout_ms, + }), + ) + .expect("should insert the stand-in's table"); + crate::integrations::registry_test_support::deferred_fixture::place(&mut settings); + settings + } + + /// Every key the cache was asked to store, rendered. + fn stored_cache_keys(cache: &MemoryTemplateCache) -> Vec { + cache + .stored_keys + .lock() + .expect("should lock stored keys") + .iter() + .map(crate::platform::TemplateCacheKey::to_cache_key) + .collect() + } + + /// Every key the cache was asked to read, rendered. + fn looked_up_cache_keys(cache: &MemoryTemplateCache) -> Vec { + cache + .lookups + .lock() + .expect("should lock lookups") + .iter() + .map(crate::platform::TemplateCacheKey::to_cache_key) + .collect() } #[tokio::test] - async fn template_cookie_publisher_absent_and_empty_are_separate() { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + async fn two_integration_configurations_never_share_a_template() { + // `template_fingerprint` folds the complete typed config, and its own + // tests call it directly — so reverting the *call site* back to the + // bundle-only hash, a constant for a given binary, left the entire suite + // green while every configuration silently shared one template. + // + // This drives the real request path and asserts on the keys the cache was + // actually handed, which is the only place that mutation is visible. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for variant in ["absent", "empty"] { - stub.push_response_with_headers( - 200, - format!("variant-{variant}") - .into_bytes(), - vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ], - ); - } - for (field, expected) in [ - (b"ts-ec=reader1".as_slice(), "absent"), - (b"ab_bucket=".as_slice(), "empty"), - (b"ts-ec=reader2".as_slice(), "absent"), - (b"ab_bucket=; ts-ec=reader3".as_slice(), "empty"), - ] { - let html = String::from_utf8( - body_of(run(&settings, &services, cookie_policy_request(&[field])).await).await, - ) - .expect("should decode HTML"); - assert!( - html.contains(&format!("variant-{expected}")), - "should preserve absent versus empty variants" - ); - } + let first = Arc::new(settings_with_integration_timeout("esi", 1000)); + let second = Arc::new(settings_with_integration_timeout("esi", 2500)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + + let first_page = String::from_utf8( + body_of(run(&first, &services, navigation_request()).await).await, + ) + .expect("should serve the first configuration's page as UTF-8"); + let second_page = String::from_utf8( + body_of(run(&second, &services, navigation_request()).await).await, + ) + .expect("should serve the second configuration's page as UTF-8"); + + assert!( + first_page.contains("\"timeoutMs\":1000") + && second_page.contains("\"timeoutMs\":2500"), + "each page should carry the markup its own configuration writes" + ); + let stored = stored_cache_keys(&cache); assert_eq!( - stored_cache_keys(&cache).len(), + stored.len(), 2, - "should store both presence variants" + "each configuration must store its own template, got {stored:?}" + ); + assert_ne!( + stored[0], stored[1], + "two configurations of one integration must key different templates, because \ + one key serves the first configuration's injected markup to the second" ); assert_eq!( stub.recorded_request_uris().len(), 2, - "should hit both variants without origin" + "the second configuration must have missed and fetched the origin rather \ + than reading the first's template" ); } #[tokio::test] - async fn template_cookie_publisher_response_guards_remain_effective() { - for guard in [ - vec![("vary", "Cookie")], - vec![("vary", "X-Exp-Variant"), ("vary", "cOoKiE")], - vec![("vary", "X-Exp-Variant, Cookie")], - vec![("vary", "*")], - vec![("vary", "uncovered-header")], - vec![("set-cookie", "origin=value")], - ] { - let mut settings = - cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - Arc::make_mut(&mut settings) - .creative_opportunities - .as_mut() - .expect("should configure opportunities") - .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let mut response_headers = vec![ - ("content-type", "text/html"), - ("cache-control", "public, max-age=300"), - ]; - response_headers.extend(guard); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - response_headers.clone(), - ); - let response = run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-response", - "should preserve origin response restrictions" - ); - let _ = body_of(response).await; - } - assert!( - stored_cache_keys(&cache).is_empty(), - "should never store a disqualified response" - ); - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "should fetch each disqualified response" - ); - } - } + async fn one_integration_configuration_keys_one_template() { + // The converse, and the failure mode a fingerprint fix can introduce: + // over-invalidating is as total as under-invalidating. A fingerprint that + // moves between two equal configurations is a cache that never hits, which + // this would read as "no measurable benefit" rather than as a bug. + // + // The two `Settings` are parsed independently, so the integration's + // table iterates in different orders, which is what exercises the sort. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let first = Arc::new(settings_with_integration_timeout("esi", 1000)); + let second = Arc::new(settings_with_integration_timeout("esi", 1000)); + queue_shareable_html(&stub); - #[tokio::test] - async fn template_cookie_publisher_malformed_later_fields_and_duplicates_bypass() { - for fields in [ - vec![b"ab_bucket=A".as_slice(), b"unknown=\xff".as_slice()], - vec![b"ab_bucket=A".as_slice(), b"ab_bucket=B".as_slice()], - vec![b"ab_bucket=A; ab_bucket=A".as_slice()], - vec![b"ab_bucket=A; broken".as_slice()], - vec![b"ab_bucket=A;".as_slice()], - vec![b"ab_bucket=A".as_slice(), b"".as_slice()], - ] { - for warm in [false, true] { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - if warm { - queue_shareable_html(&stub); - let _ = body_of( - run( - &settings, - &services, - cookie_policy_request(&[b"ab_bucket=A"]), - ) - .await, - ) - .await; - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should warm cache before malformed input" - ); - } - let lookups = looked_up_cache_keys(&cache).len(); - let stores = stored_cache_keys(&cache).len(); - queue_shareable_html(&stub); - let response = run( - &settings, - &services, - prepared_cookie_policy_request(&fields), - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should bypass malformed fields reaching the evaluator" - ); - let _ = body_of(response).await; - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should never look up ambiguous input" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - stores, - "should never store ambiguous input" - ); - assert_eq!( - stub.recorded_request_uris().len(), - usize::from(warm) + 1, - "should forward requests accepted by earlier validation" - ); - } - } + let _ = run(&first, &services, navigation_request()).await; + let _ = run(&second, &services, navigation_request()).await; + + let looked_up = looked_up_cache_keys(&cache); + assert_eq!( + looked_up.len(), + 2, + "both requests must consult the cache, got {looked_up:?}" + ); + assert_eq!( + looked_up[0], looked_up[1], + "equal configurations must name one key, or the cache never hits" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the second request must be served from the first's template" + ); } #[tokio::test] - async fn template_cookie_publisher_selected_invalid_header_keeps_existing_error() { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + async fn one_context_key_allowlist_keys_one_template() { + let mut source = serde_json::to_value(settings_with_mode("esi")) + .expect("should serialize cache settings"); + source["auction"]["allowed_context_keys"] = + serde_json::json!(["zeta", "alpha", "gamma", "beta", "epsilon", "delta"]); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); - let error = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - prepared_cookie_policy_request(&[b"ab_bucket=\xff"]), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .err() - .expect("should retain the existing selected-header error"); - assert!( - matches!( - error.current_context(), - TrustedServerError::InvalidHeaderValue { .. } - ), - "should preserve the existing error type" - ); + + for _ in 0..16 { + let settings = Arc::new( + serde_json::from_value(source.clone()).expect("should parse cache settings"), + ); + queue_shareable_html(&stub); + let _ = run(&settings, &services, navigation_request()).await; + } + + let keys = looked_up_cache_keys(&cache); + assert_eq!(keys.len(), 16, "should consult the cache for every request"); assert!( - looked_up_cache_keys(&cache).is_empty(), - "should not reach shared lookup" + keys.iter().all(|key| key == &keys[0]), + "should reuse one key across independently parsed allowlists" ); - assert!( - stored_cache_keys(&cache).is_empty(), - "should not store invalid requests" + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should store one template" ); - assert!( - stub.recorded_request_uris().is_empty(), - "should retain earlier rejection before origin" + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "should fetch the origin only for the first request" ); } + fn cookie_navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") + .header(header::COOKIE, "ts-ec=abc123") + .body(EdgeBody::empty()) + .expect("should build cookie-bearing request") + } + #[tokio::test] - async fn template_cookie_publisher_ignores_json_without_changing_origin_cookies() { - for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let settings = - cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let cold_cookie = r#"ab_bucket=A; g_state={"enabled":true,"count":1}"#; - queue_shareable_html(&stub); - for (cookie, state) in [ - (cold_cookie, "miss-stored"), - (r#"ab_bucket=A; g_state={"enabled":false,"count":2}"#, "hit"), - ] { - let response = run_via( - &settings, - &services, - cookie_policy_request(&[cookie.as_bytes()]), - finalizer, - ) - .await; - assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - state, - "should share the template despite unrelated JSON cookie changes" - ); - let _ = body_of(response).await; - } - let forwarded = stub.recorded_request_headers(); - assert_eq!( - forwarded.len(), - 1, - "should fetch origin only for the cold request" - ); - assert_eq!( - forwarded[0] - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(), - [cold_cookie], - "should preserve the ignored cookie when forwarding to origin" + async fn readthrough_never_collapses_template_cookie_variants() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .origin_readthrough_enabled = Some(true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for arm in ["absent", "A", "B", "empty"] { + stub.push_response_with_headers( + 200, + format!("arm-{arm}").into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], ); - let lookups = looked_up_cache_keys(&cache).len(); - queue_shareable_html(&stub); - let response = run_via( - &settings, - &services, - cookie_policy_request(&[br#"ab_bucket=A; g_state={"enabled":true}; session="#]), - finalizer, - ) - .await; + } + for (index, (cookie, arm)) in [ + (None, "absent"), + (Some("ab_bucket=A"), "A"), + (Some("ab_bucket=B"), "B"), + (Some("ab_bucket="), "empty"), + (Some("ab_bucket=A"), "A"), + (Some("ab_bucket=B"), "B"), + ] + .iter() + .enumerate() + { + let fields: Vec<&[u8]> = cookie.iter().map(|value| value.as_bytes()).collect(); + let response = run(&settings, &services, cookie_policy_request(&fields)).await; assert_eq!( response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "bypass-request", - "should honor bypass cookie presence alongside ignored JSON" - ); - let _ = body_of(response).await; - assert_eq!( - looked_up_cache_keys(&cache).len(), - lookups, - "should skip shared lookup for session requests" + if index < 4 { "miss-stored" } else { "hit" }, + "should still cache separate templates" ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should not store session responses" + let body = String::from_utf8(body_of(response).await).expect("should decode HTML"); + assert!( + body.contains(&format!("arm-{arm}")), + "should preserve each cookie variant" ); } + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass; 4], + "should never cache origin bytes without the template cookie dimensions, even when the key cookie is absent" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 4, + "should store each template variant" + ); } #[tokio::test] - async fn template_cookie_publisher_uses_cookies_after_existing_preparation() { - let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, false); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - // Normal preparation strips invalid fields and empty pairs before origin - // forwarding. Preserve it; the cache policy sees the prepared request. - let cold = run( + async fn readthrough_respects_named_session_bypass_without_disabling_anonymous_sharing() { + let mut settings = cookie_policy_settings(None, Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .origin_readthrough_enabled = Some(true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); + let _ = run(&settings, &services, cookie_policy_request(&[])).await; + let response = run(&settings, &services, cookie_policy_request(&[b"session="])).await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should not read an anonymous template for a session" + ); + assert_eq!( + stub.recorded_cache_intents(), + vec![ + PlatformCacheIntent::Shared { + surrogate_key: format!( + "ts-template {}", + reader_url_surrogate_key("http://ts.example.com/article") + ) + }, + PlatformCacheIntent::Bypass, + ], + "should share anonymous bytes while refusing even an empty named session" + ); + } + + #[tokio::test] + async fn readthrough_preload_bypasses_configured_cookie_dimensions() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let config = Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.origin_readthrough_enabled = Some(true); + config.assembly_mode = Some(AssemblyMode::Inline); + let stub = Arc::new(StubHttpClient::new()); + stub.set_pending_streaming_responses_supported(true); + let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + queue_shareable_html(&stub); + run_through_ec_preload( &settings, &services, - cookie_policy_request(&[b"ab_bucket=A;", b"unknown=\xff"]), + cookie_policy_request(&[b"ab_bucket=A"]), ) .await; assert_eq!( - cold.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "miss-stored", - "should classify the cookies actually forwarded" + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "should apply the cookie-dimension exclusion on the pending origin path too" ); - let _ = body_of(cold).await; - let warm = run( + } + + fn cookie_policy_settings( + key_names: Option<&[&str]>, + bypass_names: Option<&[&str]>, + independent: bool, + ) -> Arc { + let mut settings = settings_with_mode("esi"); + let config = settings + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.template_cache_key_cookies = + key_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); + config.template_cache_bypass_cookies = + bypass_names.map(|names| names.iter().map(|name| (*name).to_string()).collect()); + config.origin_is_cookie_independent = Some(independent); + config + .validate_runtime() + .expect("should validate cookie policy fixture"); + Arc::new(settings) + } + + /// The settings check refuses a key cookie named as core's own identity + /// cookies are. A deployment's own prefix is known only on a request, + /// so a key cookie naming this request's Edge Cookie shares no + /// template. + #[tokio::test] + async fn a_key_cookie_naming_the_requests_edge_cookie_shares_no_template() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = cookie_policy_settings(Some(&["_ex-ec"]), None, true); + let services = services(Arc::clone(&stub), Arc::clone(&cache)).with_prefixes( + crate::prefix::Prefixes::new( + crate::prefix::PublicPrefix::new(["_ex"]).expect("should accept the prefix"), + crate::platform::ServerKeyPrefix::new("ex", "test-publisher.com"), + ), + ); + queue_shareable_html(&stub); + + let response = run( &settings, &services, - cookie_policy_request(&[b"ab_bucket=A"]), + cookie_policy_request(&[b"_ex-ec=reader"]), ) .await; + assert_eq!( - warm.headers()[HEADER_X_TS_TEMPLATE_CACHE], - "hit", - "should share identical prepared origin inputs" - ); - let _ = body_of(warm).await; - assert_eq!( - stub.recorded_request_uris().len(), - 1, - "should preserve existing request preparation" + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request" ); - let forwarded = stub.recorded_request_headers(); - assert_eq!(forwarded.len(), 1, "should record one origin request"); - let forwarded_cookies = forwarded[0] - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) - .map(|(_, value)| value.as_str()) - .collect::>(); - assert_eq!( - forwarded_cookies, - ["ab_bucket=A"], - "should forward exactly the prepared cookie represented by the cache key" + let _ = body_of(response).await; + assert!( + stored_cache_keys(&cache).is_empty(), + "a reader's identifier must never key a stored template" ); } + fn cookie_policy_request(fields: &[&[u8]]) -> Request { + let mut request = navigation_request(); + for field in fields { + request.headers_mut().append( + header::COOKIE, + HeaderValue::from_bytes(field).expect("should build cookie field"), + ); + } + request + } + + // Exercise raw fields at the prepared-request boundary. A module's preparer + // may remove invalid fields and empty pairs before generic cookie handling. + fn prepared_cookie_policy_request(fields: &[&[u8]]) -> Request { + let mut request = cookie_policy_request(fields); + IntegrationRegistry::mark_prepared_for_tests(&mut request); + request + } + #[tokio::test] - async fn template_cookie_publisher_empty_lists_keep_legacy_behavior() { - for independent in [false, true] { - let settings = cookie_policy_settings(Some(&[]), Some(&[]), independent); + async fn template_cookie_publisher_key_only_admits_listed_cookies() { + for unused in [None, Some([].as_slice())] { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), unused, false); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); for _ in 0..2 { - queue_shareable_html(&stub); - let _ = - body_of(run(&settings, &services, cookie_navigation_request()).await).await; + let _ = body_of( + run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await, + ) + .await; } assert_eq!( stub.recorded_request_uris().len(), - if independent { 1 } else { 2 }, - "should retain legacy boolean eligibility" + 1, + "should share a listed-only variant even with independence false" ); assert_eq!( stored_cache_keys(&cache).len(), - usize::from(independent), - "should preserve legacy storage" + 1, + "should store one variant" ); - } - } - - #[tokio::test] - async fn template_cookie_publisher_policy_changes_invalidate_templates() { - let first = cookie_policy_settings(None, None, true); - let second = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - let third = cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); - assert_ne!( - template_fingerprint(&first), - template_fingerprint(&second), - "should fingerprint the key policy" - ); - assert_ne!( - template_fingerprint(&second), - template_fingerprint(&third), - "should fingerprint the bypass policy" - ); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for settings in [&first, &second, &third] { - queue_shareable_html(&stub); + let lookups = looked_up_cache_keys(&cache).len(); let _ = body_of( run( - settings, + &settings, &services, - cookie_policy_request(&[b"ab_bucket=A"]), + cookie_policy_request(&[b"ab_bucket=A; unknown=1"]), ) .await, ) .await; - } - let keys = stored_cache_keys(&cache); - assert_eq!( - keys.len(), - 3, - "should store a fresh entry under each policy" - ); - assert_ne!(keys[0], keys[1], "should not reuse pre-policy templates"); - assert_ne!( - keys[1], keys[2], - "should not reuse entries admitted under another bypass policy" - ); - assert_eq!( - stub.recorded_request_uris().len(), - 3, - "should fetch after each policy change" - ); - } - - #[tokio::test] - async fn template_cookie_publisher_warm_variant_runs_fresh_reader_assembly() { - let mut raw = settings_with_bidder("esi"); - let config = raw - .creative_opportunities - .as_mut() - .expect("should configure opportunities"); - config.template_cache_key_cookies = Some(vec!["ab_bucket".to_string()]); - config.origin_is_cookie_independent = Some(true); - let settings = Arc::new(raw); - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for (index, field) in [ - b"ab_bucket=A; ts-ec=reader1".as_slice(), - b"ab_bucket=A; ts-ec=reader2".as_slice(), - ] - .iter() - .enumerate() - { - queue_bid_response(&stub); - if index == 0 { - queue_shareable_html(&stub); - } - let response = run_bidding_at_price( - &settings, - &services, - cookie_policy_request(&[field]), - if index == 0 { 3.5 } else { 7.5 }, - ) - .await; assert_eq!( - response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index == 0 { "miss-stored" } else { "hit" }, - "should serve the second reader from the shared template" - ); - assert!( - response.headers()[header::CACHE_CONTROL] - .to_str() - .expect("should read cache policy") - .contains("private"), - "should keep reader output private" + looked_up_cache_keys(&cache).len(), + lookups, + "should bypass for any unlisted cookie" ); - let document = - String::from_utf8(body_of(response).await).expect("should decode document"); assert_eq!( - seam_bids(&document) - .get("test-slot") - .and_then(|bid| bid.get("hb_pb")) - .and_then(serde_json::Value::as_str), - Some(if index == 0 { "3.50" } else { "7.50" }), - "should assemble this request's winning bid" + stored_cache_keys(&cache).len(), + 1, + "should not store an unlisted-cookie response" ); - } - let requests = stub.recorded_request_uris(); - assert_eq!( - requests - .iter() - .filter(|uri| uri.contains("/article")) - .count(), - 1, - "should fetch one shared origin template" - ); - assert_eq!( - requests.len(), - 3, - "should run a fresh auction on each reader request" - ); - let entries = cache.entries.lock().expect("should lock templates"); - assert_eq!(entries.len(), 1, "should share within a variant"); - let stored = String::from_utf8_lossy( - &entries - .values() - .next() - .expect("should store a template") - .body, - ); - assert!( - stored.contains(AD_ASSEMBLY_SEAM), - "should keep the unresolved reader assembly marker" - ); - for reader_bytes in ["reader1", "reader2", "window.tsjs", "hb_pb"] { - assert!( - !stored.contains(reader_bytes), - "should exclude per-reader state from stored bytes" + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "should fetch unlisted-cookie origin HTML" ); } } #[tokio::test] - async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { + async fn template_cookie_publisher_session_bypasses_warm_and_cold_cache() { for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { - let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); - Arc::make_mut(&mut settings).auction.providers = - crate::auction_config_types::AuctionConfig::legacy_provider_map(&[ - SCHEDULING_PROVIDER, - ]); + for unused in [None, Some([].as_slice()), Some(["ab_bucket"].as_slice())] { + for warm in [false, true] { + let settings = cookie_policy_settings(unused, Some(&["session"]), true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + if warm { + queue_shareable_html(&stub); + let _ = body_of( + run_via( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader"]), + finalizer, + ) + .await, + ) + .await; + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should warm an anonymous template first" + ); + } + let lookups = looked_up_cache_keys(&cache).len(); + let stores = stored_cache_keys(&cache).len(); + for session in [b"session=".as_slice(), b"session=token".as_slice()] { + stub.push_response_with_headers( + 200, + b"personal-account".to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + let response = run_via( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A; ts-ec=reader", session]), + finalizer, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should report request bypass" + ); + assert!( + !response.headers().contains_key(HEADER_X_TS_ASSEMBLY), + "should omit shared-assembly diagnostics on inline responses" + ); + assert!( + String::from_utf8(body_of(response).await) + .expect("should decode HTML") + .contains("personal-account"), + "should render this request's origin HTML" + ); + } + assert_eq!( + looked_up_cache_keys(&cache).len(), + lookups, + "should never look up or reserve for session requests" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + stores, + "should never store session HTML" + ); + assert_eq!( + stub.recorded_request_uris().len(), + usize::from(warm) + 2, + "should fetch every session request" + ); + assert!( + cache + .entries + .lock() + .expect("should lock templates") + .values() + .all(|entry| !String::from_utf8_lossy(&entry.body) + .contains("personal-account")), + "should keep personal bytes out of templates" + ); + } + } + } + } + + #[tokio::test] + async fn template_cookie_publisher_downstream_variants_remain_separate() { + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut settings = + cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - let graph = KvIdentityGraph::in_memory("cookie-withdrawal-store"); - let identities = [ - format!("{}.Read01", "a".repeat(64)), - format!("{}.Read02", "b".repeat(64)), - ]; - for (index, identity) in identities.iter().enumerate() { - assert!( - crate::ec::generation::is_valid_ec_id(identity), - "should use valid EC identities in the fixture" + for arm in ["A", "B"] { + stub.push_response_with_headers( + 200, + format!("arm-{arm}").into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ("vary", "X-Exp-Variant"), + ], ); - graph - .create( - identity, - &crate::ec::kv_types::KvEntry::minimal( - "example.com", - &format!("partner-reader-{index}"), - crate::ec::current_timestamp(), - ), - ) - .expect("should seed a live reader identity"); } - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry"); - let partner = serde_json::from_value(serde_json::json!({ - "name": "Example partner", - "source_domain": "example.com", - "bidstream_enabled": true - })) - .expect("should deserialize fixture partner"); - let partners = PartnerRegistry::from_config(&[partner]) - .expect("should create partner registry"); - // Only the cold request has an origin response available. - queue_shareable_html(&stub); - - for (index, (identity, withdrawn)) in [ - (&identities[0], false), - (&identities[1], false), - (&identities[1], true), - ] - .into_iter() - .enumerate() - { - let consent = if withdrawn { - ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::UsState( - "CA".to_owned(), - ), - gpc: true, - ..Default::default() - } - } else { - scheduling_consent() - }; - let mut ec_context = EcContext::new_for_test(Some(identity.clone()), consent); - assert_eq!( - ec_context.ec_allowed(), - !withdrawn, - "should apply reader consent" + for (index, arm) in ["A", "B", "A", "B"].iter().enumerate() { + let cookies = format!( + "ab_bucket={arm}; ts-ec=reader{index}; ignored=scope{index}; ignored=other{index}" ); - let captured = Arc::new(Mutex::new(None)); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { - captured: Arc::clone(&captured), - http: Arc::clone(&stub), - lookups: Arc::new(AtomicUsize::new(0)), - })); - let orchestrator = Arc::new(orchestrator); - let cookies = format!("ab_bucket=A; ts-ec={identity}"); - let response = handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: Some(&partners), - }, - cookie_policy_request(&[cookies.as_bytes()]), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should serve a reader with an existing EC"); + let request = cookie_policy_request(&[cookies.as_bytes()]); assert!( - ec_context.kv_snapshot().entry_for(identity).is_some(), - "should preload this reader's identity even during withdrawal on a hit" - ); - let mut response = finalize_test_publisher_response( - response, - &settings, - &services, - ®istry, - orchestrator, - finalizer, - ) - .await; - crate::ec::finalize::ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - &partners, - None, - None, - &mut response, + !request.headers().contains_key("x-exp-variant"), + "should reproduce downstream-only header topology" ); + let response = run_via(&settings, &services, request, finalizer).await; assert_eq!( response.headers()[HEADER_X_TS_TEMPLATE_CACHE], - if index == 0 { "miss-stored" } else { "hit" }, - "should share one variant across identities and withdrawal" + if index < 2 { "miss-stored" } else { "hit" }, + "should share within each arm" ); - let cache_control = response.headers()[header::CACHE_CONTROL] - .to_str() - .expect("should decode cache policy"); - for directive in ["private", "no-store"] { - assert!( - cache_control - .split(',') - .any(|value| value.trim() == directive), - "should keep finalized reader responses private and uncacheable" - ); - } - assert_eq!( - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .any(|value| { - let value = value.to_str().expect("should decode response cookie"); - value.starts_with("ts-ec=") && value.contains("Max-Age=0") - }), - withdrawn, - "should expire the EC cookie only for the withdrawing reader" + assert!( + response.headers()[header::CACHE_CONTROL] + .to_str() + .expect("should read cache control") + .contains("private"), + "should keep assembled output private" ); - let body = body_of(response).await; - assert!(!body.is_empty(), "should render a complete reader response"); - let captured = captured.lock().expect("should lock captured auction"); - let auction = captured.as_ref().expect("should dispatch a reader auction"); - assert_eq!( - auction.request.user.id.as_deref(), - if withdrawn { - None - } else { - Some(identity.as_str()) - }, - "should use this reader's identity and suppress it after withdrawal" + let html = + String::from_utf8(body_of(response).await).expect("should decode HTML"); + assert!( + html.contains(&format!("arm-{arm}")), + "should render the correct experiment arm" + ); + assert!( + !html.contains(if *arm == "A" { "arm-B" } else { "arm-A" }), + "should never cross-serve arms" ); - if withdrawn { - assert!( - auction.request.user.eids.is_none(), - "should suppress withdrawn EIDs" - ); - } else { - let eids = auction - .request - .user - .eids - .as_ref() - .expect("should include consenting reader EIDs"); - assert_eq!(eids.len(), 1, "should expose only the configured partner"); - assert_eq!( - eids[0].source, "example.com", - "should use the registered source" - ); - assert_eq!( - eids[0].uids[0].id, - format!("partner-reader-{index}"), - "should use this reader's partner identity on cold and warm requests" - ); - } } assert_eq!( stub.recorded_request_uris().len(), - 1, - "should fetch origin only once" - ); - assert_eq!( - looked_up_cache_keys(&cache).len(), - 3, - "should look up every reader" - ); - assert_eq!( - stored_cache_keys(&cache).len(), - 1, - "should store only the cold template" + 2, + "should skip origin on both warm arms" ); - for (index, identity) in identities.iter().enumerate() { - let (entry, _) = graph - .get(identity) - .expect("should read reader identity") - .expect("should retain the live row or its tombstone"); - assert_eq!( - entry.consent.ok, - index == 0, - "should revoke only the second reader" - ); - assert_eq!( - entry.ids.is_empty(), - index == 1, - "should clear only revoked partner IDs" - ); - } - let entries = cache.entries.lock().expect("should lock cached templates"); - let stored = &entries - .values() - .next() - .expect("should retain shared template") - .body; - let stored = String::from_utf8_lossy(stored); - for identity in &identities { - assert!( - !stored.contains(identity), - "should keep reader identities out of cached bytes" - ); - } + let stored = stored_cache_keys(&cache); + assert_eq!(stored.len(), 2, "should store two variants"); + assert_ne!(stored[0], stored[1], "should distinguish arms in the key"); } } #[tokio::test] - async fn by_default_a_cookie_bearing_request_uses_no_shared_cache() { - // The shipped default, and the reason the cache is nearly inert on real - // traffic: TS sets its own identity cookie, so essentially every repeat - // visitor arrives carrying one and is excluded in both directions. + async fn template_cookie_publisher_absent_and_empty_are_separate() { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, cookie_navigation_request()).await; - let _ = run(&settings, &services, cookie_navigation_request()).await; - + for variant in ["absent", "empty"] { + stub.push_response_with_headers( + 200, + format!("variant-{variant}") + .into_bytes(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ], + ); + } + for (field, expected) in [ + (b"ts-ec=reader1".as_slice(), "absent"), + (b"ab_bucket=".as_slice(), "empty"), + (b"ts-ec=reader2".as_slice(), "absent"), + (b"ab_bucket=; ts-ec=reader3".as_slice(), "empty"), + ] { + let html = String::from_utf8( + body_of(run(&settings, &services, cookie_policy_request(&[field])).await).await, + ) + .expect("should decode HTML"); + assert!( + html.contains(&format!("variant-{expected}")), + "should preserve absent versus empty variants" + ); + } assert_eq!( - stub.recorded_request_uris().len(), + stored_cache_keys(&cache).len(), 2, - "both requests must reach the origin" - ); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "and neither may store a template" + "should store both presence variants" ); - } - - #[tokio::test] - async fn a_declared_cookie_independent_origin_lets_repeat_visitors_share() { - // The opt-in. Without it the cache can only ever serve first-ever page - // views, which is not the population the issue cares about. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.creative_opportunities - .as_mut() - .expect("fixture configures creative opportunities") - .origin_is_cookie_independent = Some(true); - let settings = Arc::new(raw); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, cookie_navigation_request()).await; - let _ = run(&settings, &services, cookie_navigation_request()).await; - assert_eq!( stub.recorded_request_uris().len(), - 1, - "the second cookie-bearing request should be served from the cache" + 2, + "should hit both variants without origin" ); } #[tokio::test] - async fn an_active_diagnostics_request_never_stores_a_template() { - // An independent review reintroduced a diagnostics leak scoped to - // A request-private diagnostics mutation could otherwise leak through a - // shared template. `requires_private_no_store()` is a - // strict superset of the condition under which diagnostics markup is - // emitted, and that stamp lands *before* the template cache gate reads response headers, - // so such a request never stores a template at all. - // - // That is a coincidence between two independent conditions, and the whole - // protection rests on it. This pins the consequence directly, so the - // relationship is checked rather than merely reasoned about. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, diagnostics_navigation_request()).await; - - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "a reader running diagnostics must not contribute to a shared cache" - ); + async fn template_cookie_publisher_response_guards_remain_effective() { + for guard in [ + vec![("vary", "Cookie")], + vec![("vary", "X-Exp-Variant"), ("vary", "cOoKiE")], + vec![("vary", "X-Exp-Variant, Cookie")], + vec![("vary", "*")], + vec![("vary", "uncovered-header")], + vec![("set-cookie", "origin=value")], + ] { + let mut settings = + cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + Arc::make_mut(&mut settings) + .creative_opportunities + .as_mut() + .expect("should configure opportunities") + .template_cache_vary = Some(vec!["x-exp-variant".to_string()]); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let mut response_headers = vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=300"), + ]; + response_headers.extend(guard); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + response_headers.clone(), + ); + let response = run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-response", + "should preserve origin response restrictions" + ); + let _ = body_of(response).await; + } + assert!( + stored_cache_keys(&cache).is_empty(), + "should never store a disqualified response" + ); + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "should fetch each disqualified response" + ); + } } #[tokio::test] - async fn active_diagnostics_bypass_an_already_warm_template() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let response = run(&settings, &services, diagnostics_navigation_request()).await; - let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); + async fn template_cookie_publisher_malformed_later_fields_and_duplicates_bypass() { + for fields in [ + vec![b"ab_bucket=A".as_slice(), b"unknown=\xff".as_slice()], + vec![b"ab_bucket=A".as_slice(), b"ab_bucket=B".as_slice()], + vec![b"ab_bucket=A; ab_bucket=A".as_slice()], + vec![b"ab_bucket=A; broken".as_slice()], + vec![b"ab_bucket=A;".as_slice()], + vec![b"ab_bucket=A".as_slice(), b"".as_slice()], + ] { + for warm in [false, true] { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + if warm { + queue_shareable_html(&stub); + let _ = body_of( + run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await, + ) + .await; + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should warm cache before malformed input" + ); + } + let lookups = looked_up_cache_keys(&cache).len(); + let stores = stored_cache_keys(&cache).len(); + queue_shareable_html(&stub); + let response = run( + &settings, + &services, + prepared_cookie_policy_request(&fields), + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should bypass malformed fields reaching the evaluator" + ); + let _ = body_of(response).await; + assert_eq!( + looked_up_cache_keys(&cache).len(), + lookups, + "should never look up ambiguous input" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + stores, + "should never store ambiguous input" + ); + assert_eq!( + stub.recorded_request_uris().len(), + usize::from(warm) + 1, + "should forward requests accepted by earlier validation" + ); + } + } + } - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "request-private diagnostics must reach the origin even when an ordinary \ - shared template is warm" + #[tokio::test] + async fn template_cookie_publisher_selected_invalid_header_keeps_existing_error() { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + let registry = test_registry(&settings); + let error = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + prepared_cookie_policy_request(&[b"ab_bucket=\xff"]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .err() + .expect("should retain the existing selected-header error"); + assert!( + matches!( + error.current_context(), + TrustedServerError::InvalidHeaderValue { .. } + ), + "should preserve the existing error type" ); - assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), - 1, - "the diagnostics request must not consult template cache at all" + assert!( + looked_up_cache_keys(&cache).is_empty(), + "should not reach shared lookup" ); assert!( - document.contains("__tsjs_gpt_diagnostics_active"), - "origin fallback must retain the request-private diagnostics bootstrap" + stored_cache_keys(&cache).is_empty(), + "should not store invalid requests" + ); + assert!( + stub.recorded_request_uris().is_empty(), + "should retain earlier rejection before origin" ); } #[tokio::test] - async fn datadome_suppressed_request_bypasses_a_warm_shared_template() { + async fn template_cookie_publisher_ignores_json_without_changing_origin_cookies() { + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let settings = + cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let cold_cookie = r#"ab_bucket=A; g_state={"enabled":true,"count":1}"#; + queue_shareable_html(&stub); + for (cookie, state) in [ + (cold_cookie, "miss-stored"), + (r#"ab_bucket=A; g_state={"enabled":false,"count":2}"#, "hit"), + ] { + let response = run_via( + &settings, + &services, + cookie_policy_request(&[cookie.as_bytes()]), + finalizer, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + state, + "should share the template despite unrelated JSON cookie changes" + ); + let _ = body_of(response).await; + } + let forwarded = stub.recorded_request_headers(); + assert_eq!( + forwarded.len(), + 1, + "should fetch origin only for the cold request" + ); + assert_eq!( + forwarded[0] + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(), + [cold_cookie], + "should preserve the ignored cookie when forwarding to origin" + ); + let lookups = looked_up_cache_keys(&cache).len(); + queue_shareable_html(&stub); + let response = run_via( + &settings, + &services, + cookie_policy_request(&[br#"ab_bucket=A; g_state={"enabled":true}; session="#]), + finalizer, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "bypass-request", + "should honor bypass cookie presence alongside ignored JSON" + ); + let _ = body_of(response).await; + assert_eq!( + looked_up_cache_keys(&cache).len(), + lookups, + "should skip shared lookup for session requests" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should not store session responses" + ); + } + } + + #[tokio::test] + async fn template_cookie_publisher_uses_cookies_after_existing_preparation() { + let settings = cookie_policy_settings(Some(&["ab_bucket"]), None, false); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.integrations - .insert_config( - "datadome", - &serde_json::json!({ - "enabled": true, - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let ordinary = run(&settings, &services, navigation_request()).await; - let ordinary_document = String::from_utf8(body_of(ordinary).await) - .expect("ordinary document should be UTF-8"); - assert!( - ordinary_document.contains("/integrations/datadome/tags.js"), - "the warm template fixture must contain the ordinary DataDome tag" + // A module's preparer may rewrite the Cookie header before origin + // forwarding, as the stand-in does when it strips the cookie it + // reserves, dropping a field it cannot read and an empty pair as it + // goes. The cache policy sees the prepared request. + let reserved = format!( + "{}=1", + crate::integrations::registry_test_support::request_fixture::COOKIE_NAME ); - - let mut suppressed_request = navigation_request(); - suppressed_request - .headers_mut() - .insert("sec-fetch-dest", HeaderValue::from_static("script")); - suppressed_request - .extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - let suppressed = run(&settings, &services, suppressed_request).await; + let cold = run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A;", b"unknown=\xff", reserved.as_bytes()]), + ) + .await; assert_eq!( - suppressed - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request"), - "request-scoped tag suppression must not read a shared template" + cold.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "miss-stored", + "should classify the cookies actually forwarded" ); - let suppressed_document = String::from_utf8(body_of(suppressed).await) - .expect("suppressed document should be UTF-8"); - + let _ = body_of(cold).await; + let warm = run( + &settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await; assert_eq!( - stub.recorded_request_uris().len(), - 2, - "the suppressed navigation must reach the origin even when template cache is warm" + warm.headers()[HEADER_X_TS_TEMPLATE_CACHE], + "hit", + "should share identical prepared origin inputs" ); + let _ = body_of(warm).await; assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), + stub.recorded_request_uris().len(), 1, - "the suppressed request must bypass template cache before lookup" + "should preserve existing request preparation" ); - assert!( - !suppressed_document.contains("/integrations/datadome/tags.js"), - "the request-scoped suppression decision must survive origin processing" + let forwarded = stub.recorded_request_headers(); + assert_eq!(forwarded.len(), 1, "should record one origin request"); + let forwarded_cookies = forwarded[0] + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())) + .map(|(_, value)| value.as_str()) + .collect::>(); + assert_eq!( + forwarded_cookies, + ["ab_bucket=A"], + "should forward exactly the prepared cookie represented by the cache key" ); } #[tokio::test] - async fn real_diagnostics_query_bypasses_template_cache_and_keeps_its_private_bootstrap() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should configure diagnostics"); - let settings = Arc::new(raw); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let mut request = navigation_request(); - *request.uri_mut() = "https://ts.example.com/article?ts_console=1" - .parse() - .expect("should parse diagnostics URI"); - let response = run(&settings, &services, request).await; - assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request") - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private") - ); - assert!(response.headers().contains_key(header::SET_COOKIE)); - let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); - assert!(document.contains("__tsjs_gpt_diagnostics_active")); - assert!(document.contains("tsjs-gpt_diagnostics.min.js")); - assert!( - cache - .lookups - .lock() - .expect("should lock lookups") - .is_empty(), - "the real query activation must bypass lookup before origin work" - ); + async fn template_cookie_publisher_empty_lists_keep_legacy_behavior() { + for independent in [false, true] { + let settings = cookie_policy_settings(Some(&[]), Some(&[]), independent); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + queue_shareable_html(&stub); + let _ = + body_of(run(&settings, &services, cookie_navigation_request()).await).await; + } + assert_eq!( + stub.recorded_request_uris().len(), + if independent { 1 } else { 2 }, + "should retain legacy boolean eligibility" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + usize::from(independent), + "should preserve legacy storage" + ); + } } #[tokio::test] - async fn real_diagnostics_cookie_bypasses_a_warm_cookie_independent_template() { + async fn template_cookie_publisher_policy_changes_invalidate_templates() { + let first = cookie_policy_settings(None, None, true); + let second = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + let third = cookie_policy_settings(Some(&["ab_bucket"]), Some(&["session"]), true); + // The registry is held constant, so the only thing the + // fingerprints differ on is the cookie policy. + let registry = test_registry(&first); + assert_ne!( + template_fingerprint(&first, ®istry, &crate::prefix::PublicPrefix::default()), + template_fingerprint(&second, ®istry, &crate::prefix::PublicPrefix::default()), + "should fingerprint the key policy" + ); + assert_ne!( + template_fingerprint(&second, ®istry, &crate::prefix::PublicPrefix::default()), + template_fingerprint(&third, ®istry, &crate::prefix::PublicPrefix::default()), + "should fingerprint the bypass policy" + ); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let mut raw = settings_with_mode("esi"); - raw.creative_opportunities - .as_mut() - .expect("fixture configures creative opportunities") - .origin_is_cookie_independent = Some(true); - raw.integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should configure diagnostics"); - let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - queue_shareable_html(&stub); - - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let mut diagnostics = navigation_request(); - diagnostics.headers_mut().insert( - header::COOKIE, - HeaderValue::from_static("__Host-ts-console=1"), - ); - let response = run(&settings, &services, diagnostics).await; + for settings in [&first, &second, &third] { + queue_shareable_html(&stub); + let _ = body_of( + run( + settings, + &services, + cookie_policy_request(&[b"ab_bucket=A"]), + ) + .await, + ) + .await; + } + let keys = stored_cache_keys(&cache); assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("bypass-request") + keys.len(), + 3, + "should store a fresh entry under each policy" ); - let document = String::from_utf8(body_of(response).await) - .expect("diagnostics document should be UTF-8"); - - assert_eq!(stub.recorded_request_uris().len(), 2); - assert_eq!( - cache.lookups.lock().expect("should lock lookups").len(), - 1, - "the diagnostics cookie must bypass the otherwise-eligible warm lookup" + assert_ne!(keys[0], keys[1], "should not reuse pre-policy templates"); + assert_ne!( + keys[1], keys[2], + "should not reuse entries admitted under another bypass policy" ); - assert!(document.contains("__tsjs_gpt_diagnostics_active")); - } - - #[tokio::test] - async fn a_cache_backend_failure_falls_back_to_origin_and_is_observable() { - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - cache.fail_lookup.store(true, Ordering::Relaxed); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - let response = run(&settings, &services, navigation_request()).await; - assert_eq!(response.status(), StatusCode::OK); assert_eq!( - response - .headers() - .get(HEADER_X_TS_TEMPLATE_CACHE) - .and_then(|value| value.to_str().ok()), - Some("backend-error") + stub.recorded_request_uris().len(), + 3, + "should fetch after each policy change" ); - let document = String::from_utf8(body_of(response).await) - .expect("fallback document should be UTF-8"); - assert!(document.contains("origin")); - assert_eq!(stub.recorded_request_uris().len(), 1); } #[tokio::test] - async fn a_cache_hit_streams_rather_than_buffering() { - // The property that makes the cache worth having, and the one that regressed - // silently. Buffered assembly held the first byte until the auction resolved - // — measured at ~100x worse TTFB than shipping nothing at all, because the - // inline path already streams and waits only at ``. - // - // Asserted on the body *shape* rather than on timing: a timing test would be - // flaky, and `EdgeBody::Stream` is the structural fact that produces the - // timing. + async fn template_cookie_publisher_warm_variant_runs_fresh_reader_assembly() { + let mut raw = settings_with_bidder("esi"); + let config = raw + .creative_opportunities + .as_mut() + .expect("should configure opportunities"); + config.template_cache_key_cookies = Some(vec!["ab_bucket".to_string()]); + config.origin_is_cookie_independent = Some(true); + let settings = Arc::new(raw); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - queue_shareable_html(&stub); - - // Warm the cache, then take the hit. - let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; - + for (index, field) in [ + b"ab_bucket=A; ts-ec=reader1".as_slice(), + b"ab_bucket=A; ts-ec=reader2".as_slice(), + ] + .iter() + .enumerate() + { + queue_bid_response(&stub); + if index == 0 { + queue_shareable_html(&stub); + } + let response = run_bidding_at_price( + &settings, + &services, + cookie_policy_request(&[field]), + if index == 0 { 3.5 } else { 7.5 }, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + if index == 0 { "miss-stored" } else { "hit" }, + "should serve the second reader from the shared template" + ); + assert!( + response.headers()[header::CACHE_CONTROL] + .to_str() + .expect("should read cache policy") + .contains("private"), + "should keep reader output private" + ); + let document = + String::from_utf8(body_of(response).await).expect("should decode document"); + assert_eq!( + seam_bids(&document) + .get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some(if index == 0 { "3.50" } else { "7.50" }), + "should assemble this request's winning bid" + ); + } + let requests = stub.recorded_request_uris(); assert_eq!( - stub.recorded_request_uris().len(), + requests + .iter() + .filter(|uri| uri.contains("/article")) + .count(), 1, - "the second request must be a hit, or this asserts nothing" + "should fetch one shared origin template" ); - assert!( - warm.headers().get(header::CONTENT_LENGTH).is_none(), - "a streamed assembly has no length until bids resolve, and headers \ - commit before the first byte" + assert_eq!( + requests.len(), + 3, + "should run a fresh auction on each reader request" ); - - // The invariant that actually matters, and the one a body-shape assertion - // misses: a stream that awaited the auction before its first yield would - // still be an `EdgeBody::Stream` and would still hold the first byte. - // - // So pull exactly one chunk and prove the auction has not been collected - // yet — `ad_bids_state` is only written by the collector. No timing - // involved, so nothing to be flaky about. - let EdgeBody::Stream(mut stream) = warm.into_body() else { - panic!("a cache hit must stream, not buffer"); - }; - let first = stream - .next() - .await - .expect("the stream should yield a first chunk") - .expect("the first chunk should read"); - - assert!( - first.starts_with(b"") || first.starts_with(b"` seam emitted - // the marker because the mode was `Esi`, while assembly was skipped because - // the gate had refused a key — a fallback at one seam and not the other. - // - // Bypassing is the *normal* case against a real origin, so this path runs - // far more often than the shared one. It has to produce a working page. - let stub = Arc::new(StubHttpClient::new()); - let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - - // A `Set-Cookie` from the origin disqualifies the response, exactly as a - // real origin does. - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ("set-cookie", "sess=1"), - ], - ); - - let document = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("response should be utf-8"); + async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { + for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings).auction.provider_names = + vec![SCHEDULING_PROVIDER.to_owned()]; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let graph = KvIdentityGraph::in_memory("cookie-withdrawal-store"); + let identities = [ + format!("{}.Read01", "a".repeat(64)), + format!("{}.Read02", "b".repeat(64)), + ]; + for (index, identity) in identities.iter().enumerate() { + assert!( + crate::ec::generation::is_valid_ec_id(identity), + "should use valid EC identities in the fixture" + ); + graph + .create( + identity, + &crate::ec::kv_types::KvEntry::minimal( + "example.com", + &format!("partner-reader-{index}"), + crate::ec::current_timestamp(), + ), + ) + .expect("should seed a live reader identity"); + } + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry"); + let partner = serde_json::from_value(serde_json::json!({ + "name": "Example partner", + "source_domain": "example.com", + "bidstream_enabled": true + })) + .expect("should deserialize fixture partner"); + let partners = PartnerRegistry::from_config(&[partner]) + .expect("should create partner registry"); + // Only the cold request has an origin response available. + queue_shareable_html(&stub); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty(), - "the gate refused, so nothing may be stored" - ); - assert!( - !document.contains(AD_ASSEMBLY_SEAM), - "a marker must never be emitted when nothing will resolve it: {document}" - ); - assert!( - document.contains("window.tsjs"), - "and the reader must still get their bids: {document}" - ); + for (index, (identity, withdrawn)) in [ + (&identities[0], false), + (&identities[1], false), + (&identities[1], true), + ] + .into_iter() + .enumerate() + { + // A US-style opt-out suppresses use without expiring the + // cookie, so the destructive path this test is about needs + // a request that withdrew device storage outright. + let mut ec_context = if withdrawn { + EcContext::new_for_test_withdrawn( + Some(identity.clone()), + ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::UsState( + "CA".to_owned(), + ), + gpc: true, + ..Default::default() + }, + ) + } else { + EcContext::new_for_test(Some(identity.clone()), scheduling_consent()) + }; + assert_eq!( + ec_context.ec_allowed(), + !withdrawn, + "should apply reader consent" + ); + let captured = Arc::new(Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { + captured: Arc::clone(&captured), + http: Arc::clone(&stub), + lookups: Arc::new(AtomicUsize::new(0)), + })); + let orchestrator = Arc::new(orchestrator); + let cookies = format!("ab_bucket=A; ts-ec={identity}"); + let response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: Some(&partners), + }, + cookie_policy_request(&[cookies.as_bytes()]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should serve a reader with an existing EC"); + assert!( + ec_context.kv_snapshot().entry_for(identity).is_some(), + "should preload this reader's identity even during withdrawal on a hit" + ); + let mut response = finalize_test_publisher_response( + response, + &settings, + &services, + ®istry, + orchestrator, + finalizer, + ) + .await; + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &partners, + None, + None, + &mut response, + &services, + ) + .await; + assert_eq!( + response.headers()[HEADER_X_TS_TEMPLATE_CACHE], + if index == 0 { "miss-stored" } else { "hit" }, + "should share one variant across identities and withdrawal" + ); + let cache_control = response.headers()[header::CACHE_CONTROL] + .to_str() + .expect("should decode cache policy"); + for directive in ["private", "no-store"] { + assert!( + cache_control + .split(',') + .any(|value| value.trim() == directive), + "should keep finalized reader responses private and uncacheable" + ); + } + assert_eq!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .any(|value| { + let value = value.to_str().expect("should decode response cookie"); + value.starts_with("ts-ec=") && value.contains("Max-Age=0") + }), + withdrawn, + "should expire the EC cookie only for the withdrawing reader" + ); + let body = body_of(response).await; + assert!(!body.is_empty(), "should render a complete reader response"); + let captured = captured.lock().expect("should lock captured auction"); + let auction = captured.as_ref().expect("should dispatch a reader auction"); + assert_eq!( + auction.request.user.id.as_deref(), + if withdrawn { + None + } else { + Some(identity.as_str()) + }, + "should use this reader's identity and suppress it after withdrawal" + ); + if withdrawn { + assert!( + auction.request.user.eids.is_none(), + "should suppress withdrawn EIDs" + ); + } else { + let eids = auction + .request + .user + .eids + .as_ref() + .expect("should include consenting reader EIDs"); + assert_eq!(eids.len(), 1, "should expose only the configured partner"); + assert_eq!( + eids[0].source, "example.com", + "should use the registered source" + ); + assert_eq!( + eids[0].uids[0].id, + format!("partner-reader-{index}"), + "should use this reader's partner identity on cold and warm requests" + ); + } + } + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "should fetch origin only once" + ); + assert_eq!( + looked_up_cache_keys(&cache).len(), + 3, + "should look up every reader" + ); + assert_eq!( + stored_cache_keys(&cache).len(), + 1, + "should store only the cold template" + ); + for (index, identity) in identities.iter().enumerate() { + let (entry, _) = graph + .get(identity) + .expect("should read reader identity") + .expect("should retain the live row or its tombstone"); + assert_eq!( + entry.consent.ok, + index == 0, + "should revoke only the second reader" + ); + assert_eq!( + entry.ids.is_empty(), + index == 1, + "should clear only revoked partner IDs" + ); + } + let entries = cache.entries.lock().expect("should lock cached templates"); + let stored = &entries + .values() + .next() + .expect("should retain shared template") + .body; + let stored = String::from_utf8_lossy(stored); + for identity in &identities { + assert!( + !stored.contains(identity), + "should keep reader identities out of cached bytes" + ); + } + } } #[tokio::test] - async fn the_seam_carries_slot_definitions_or_the_page_serves_no_ads() { - // Shared modes suppress the head slot script, so the seam is the only place - // `tsjs.adSlots` can come from. Sending only bids left it at its `[]` default, - // `adInit` defined nothing, and the page rendered perfectly with zero TS ads — - // green tests, healthy-looking page, no revenue. + async fn by_default_a_cookie_bearing_request_uses_no_shared_cache() { + // The shipped default, and the reason the cache is nearly inert on real + // traffic: TS sets its own identity cookie, so essentially every repeat + // visitor arrives carrying one and is excluded in both directions. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); + queue_shareable_html(&stub); - let miss = body_of(run(&settings, &services, navigation_request()).await).await; - let hit = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; - for (label, body) in [("miss", miss), ("hit", hit)] { - let text = String::from_utf8(body).expect("utf-8"); - assert!( - text.contains("var a=JSON.parse(") && text.contains("s(b,a)"), - "{label}: the seam must carry slot definitions and hand them to the \ - scheduler: {text}" - ); - assert!( - text.contains("test-slot"), - "{label}: the slot definitions must be populated, not `[]`" - ); - } + assert_eq!( + stub.recorded_request_uris().len(), + 2, + "both requests must reach the origin" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "and neither may store a template" + ); } #[tokio::test] - async fn an_assembled_response_is_private_even_when_the_ad_stack_is_off() { - // The private stamp was gated on `should_run_ad_stack`. A bot, prefetch, - // kill-switched or consent-denied request can still assemble an empty-bids - // document, and would have kept the origin's public caching directives — so a - // downstream cache could serve that to a later eligible reader. + async fn a_declared_cookie_independent_origin_lets_repeat_visitors_share() { + // The opt-in. Without it the cache can only ever serve first-ever page + // views, which is not the population the issue cares about. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + raw.creative_opportunities + .as_mut() + .expect("fixture configures creative opportunities") + .origin_is_cookie_independent = Some(true); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); + queue_shareable_html(&stub); - let bot = HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .header("sec-fetch-mode", "navigate") - .header( - header::USER_AGENT, - "Googlebot/2.1 (+http://www.google.com/bot.html)", - ) - .body(EdgeBody::empty()) - .expect("should build bot request"); - let response = run(&settings, &services, bot).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; + let _ = run(&settings, &services, cookie_navigation_request()).await; assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()), - Some("no-store, private"), - "an assembled response must be private whatever the ad stack decided" + stub.recorded_request_uris().len(), + 1, + "the second cookie-bearing request should be served from the cache" ); } #[tokio::test] - async fn a_cache_hit_keeps_the_origin_security_headers() { - // Reconstructing headers keeps origin `Set-Cookie` and caching directives out - // of a shared cache, and also silently dropped Content-Security-Policy — - // a weaker page, served faster. Policy headers are per-URL, so they belong - // with the template. + async fn a_request_a_module_left_state_on_never_stores_a_template() { + // What a module leaves on a request is written into that request's + // document, so a template stored from it would hand one reader's + // document to every later one. The request is kept off the shared + // path before the origin is asked, and this pins the consequence + // directly rather than leaving it to be reasoned about. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ("content-security-policy", "default-src 'self'"), - ("x-frame-options", "SAMEORIGIN"), - ], - ); + queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + let _ = run(&settings, &services, marked_navigation_request()).await; - assert_eq!( - warm.headers() - .get(header::CONTENT_SECURITY_POLICY) - .and_then(|v| v.to_str().ok()), - Some("default-src 'self'"), - "a hit must not drop the origin's CSP" - ); - assert_eq!( - warm.headers() - .get("x-frame-options") - .and_then(|v| v.to_str().ok()), - Some("SAMEORIGIN") + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "a request a module left state on must not contribute to a shared cache" ); } #[tokio::test] - async fn a_cache_hit_preserves_every_repeated_policy_header_in_order() { + async fn a_request_a_module_left_state_on_bypasses_an_already_warm_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + crate::integrations::registry_test_support::request_fixture::select_and_place(&mut raw); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ("content-security-policy", "default-src 'self'"), - ("content-security-policy", "script-src 'self'"), - ("link", "; rel=preload; as=script"), - ("link", "; rel=preload; as=style"), - ("cross-origin-opener-policy", "same-origin"), - ("cross-origin-embedder-policy", "require-corp"), - ], - ); + queue_shareable_html(&stub); + queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - let warm = run(&settings, &services, navigation_request()).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let response = run(&settings, &services, marked_navigation_request()).await; + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a document a module's request state is written into is one reader's" + ); + assert!( + response.headers().contains_key(header::SET_COOKIE), + "the module's response finalizer should run on the page path" + ); + let document = String::from_utf8(body_of(response).await) + .expect("the marked request's document should be UTF-8"); - let values = |name: &'static str| { - warm.headers() - .get_all(name) - .iter() - .map(|value| value.to_str().expect("policy header should be text")) - .collect::>() - }; assert_eq!( - values("content-security-policy"), - ["default-src 'self'", "script-src 'self'"] + stub.recorded_request_uris().len(), + 2, + "a request a module left state on must reach the origin even when an \ + ordinary shared template is warm" ); assert_eq!( - values("link"), - [ - "; rel=preload; as=script", - "; rel=preload; as=style" - ] + cache.lookups.lock().expect("should lock lookups").len(), + 1, + "the marked request must not consult template cache at all" + ); + assert!( + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ), + "origin fallback must keep what the module writes for the request" ); - assert_eq!(values("cross-origin-opener-policy"), ["same-origin"]); - assert_eq!(values("cross-origin-embedder-policy"), ["require-corp"]); } #[tokio::test] - async fn nonce_bearing_csp_bypasses_the_shared_template_cache() { + async fn a_personalized_request_bypasses_a_warm_shared_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - b"origin" - .to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ( - "content-security-policy", - "default-src 'self'; script-src 'nonce-reader-nonce'", - ), - ], - ); - } + queue_shareable_html(&stub); + queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let _ = body_of(run(&settings, &services, navigation_request()).await).await; + + let mut personalized_request = navigation_request(); + personalized_request + .headers_mut() + .insert("sec-fetch-dest", HeaderValue::from_static("script")); + personalized_request + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + let personalized = run(&settings, &services, personalized_request).await; + assert_eq!( + personalized + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-request"), + "a personalized request must not read a shared template" + ); + let _ = body_of(personalized).await; assert_eq!( stub.recorded_request_uris().len(), 2, - "a response-bound CSP nonce and its HTML must never be reused from template cache" + "the personalized request must reach the origin even when template cache is warm" ); - assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty() + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + 1, + "the personalized request must bypass template cache before lookup" ); } #[tokio::test] - async fn a_meta_delivered_nonce_policy_bypasses_the_shared_template_cache() { - // The response-header gate cannot see this policy at all. Storing the document - // would replay one response's nonce to every later reader — the exact thing the - // header check exists to prevent. + async fn a_query_a_module_acts_on_bypasses_template_cache_and_keeps_what_the_module_writes() + { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + crate::integrations::registry_test_support::request_fixture::select_and_place(&mut raw); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - br#"origin"# - .to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], - ); - } + queue_shareable_html(&stub); - let cold = run(&settings, &services, navigation_request()).await; + // The handler prepares a request that reaches it unprepared, so the + // module's preparer reads its query here as it does behind an adapter. + let mut request = navigation_request(); + *request.uri_mut() = format!( + "https://ts.example.com/article?{}", + crate::integrations::registry_test_support::request_fixture::QUERY + ) + .parse() + .expect("should parse the URI"); + let response = run(&settings, &services, request).await; assert_eq!( - cold.headers() + response + .headers() .get(HEADER_X_TS_TEMPLATE_CACHE) .and_then(|value| value.to_str().ok()), - Some("bypass-response"), - "the cold response must refuse to store a nonce-bearing document" + Some("bypass-request") ); - let _ = body_of(cold).await; - let _ = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "the warm request must reach the origin, not a replayed nonce" + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private") ); + assert!( + response.headers().contains_key(header::SET_COOKIE), + "the module's response finalizer should run on the page path" + ); + assert!( + stub.recorded_request_uris().iter().all(|uri| !uri + .contains(crate::integrations::registry_test_support::request_fixture::QUERY)), + "the origin should be asked without the query the module reserved" + ); + let document = + String::from_utf8(body_of(response).await).expect("the document should be UTF-8"); + assert!( + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ) + ); + assert!(document.contains( + crate::integrations::registry_test_support::request_fixture::MODULE_FILE + )); assert!( cache - .entries + .lookups .lock() - .expect("should lock entries") + .expect("should lock lookups") .is_empty(), - "nothing nonce-bearing may reach the shared cache" + "a query a module acts on must bypass lookup before origin work" ); } #[tokio::test] - async fn a_nonce_attribute_without_a_policy_bypasses_the_shared_template_cache() { - // Fail closed: the attributes say the document was written for a per-response - // policy, whether or not the policy itself survived to this scan. + async fn a_request_a_module_left_state_on_bypasses_a_warm_cookie_independent_template() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_mode("esi")); + let mut raw = settings_with_mode("esi"); + raw.creative_opportunities + .as_mut() + .expect("fixture configures creative opportunities") + .origin_is_cookie_independent = Some(true); + crate::integrations::registry_test_support::request_fixture::select_and_place(&mut raw); + let settings = Arc::new(raw); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - for _ in 0..2 { - stub.push_response_with_headers( - 200, - br#"origin"# - .to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], - ); - } + queue_shareable_html(&stub); + queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; - let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let response = run(&settings, &services, marked_navigation_request()).await; + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-request") + ); + let document = + String::from_utf8(body_of(response).await).expect("the document should be UTF-8"); assert_eq!(stub.recorded_request_uris().len(), 2); + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + 1, + "a request a module left state on must bypass the otherwise-eligible warm lookup" + ); assert!( - cache - .entries - .lock() - .expect("should lock entries") - .is_empty() + document.contains( + crate::integrations::registry_test_support::request_fixture::HEAD_FLAG + ) ); } #[tokio::test] - async fn a_body_close_in_script_data_corrupts_neither_the_cold_nor_the_warm_document() { - // No structural `` anywhere: a reverse byte search takes the string - // literal, and the payload's `` then terminates the publisher's script - // — in the response served cold and in the template every warm reader gets. - const PUBLISHER_SCRIPT: &str = r#""#; + async fn a_cache_backend_failure_falls_back_to_origin_and_is_observable() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); + cache.fail_lookup.store(true, Ordering::Relaxed); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( - 200, - format!("{PUBLISHER_SCRIPT}

article

").into_bytes(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", "public, max-age=300"), - ], + queue_shareable_html(&stub); + + let response = run(&settings, &services, navigation_request()).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("backend-error") ); + let document = String::from_utf8(body_of(response).await) + .expect("fallback document should be UTF-8"); + assert!(document.contains("origin")); + assert_eq!(stub.recorded_request_uris().len(), 1); + } - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("the cold document should be UTF-8"); - let warm = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("the warm document should be UTF-8"); + #[tokio::test] + async fn a_cache_hit_streams_rather_than_buffering() { + // The property that makes the cache worth having, and the one that regressed + // silently. Buffered assembly held the first byte until the auction resolved + // — measured at ~100x worse TTFB than shipping nothing at all, because the + // inline path already streams and waits only at ``. + // + // Asserted on the body *shape* rather than on timing: a timing test would be + // flaky, and `EdgeBody::Stream` is the structural fact that produces the + // timing. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + + // Warm the cache, then take the hit. + let _ = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; assert_eq!( stub.recorded_request_uris().len(), 1, - "the document is otherwise shareable and must still be stored and reused" + "the second request must be a hit, or this asserts nothing" + ); + assert!( + warm.headers().get(header::CONTENT_LENGTH).is_none(), + "a streamed assembly has no length until bids resolve, and headers \ + commit before the first byte" + ); + + // The invariant that actually matters, and the one a body-shape assertion + // misses: a stream that awaited the auction before its first yield would + // still be an `EdgeBody::Stream` and would still hold the first byte. + // + // So pull exactly one chunk and prove the auction has not been collected + // yet — `ad_bids_state` is only written by the collector. No timing + // involved, so nothing to be flaky about. + let EdgeBody::Stream(mut stream) = warm.into_body() else { + panic!("a cache hit must stream, not buffer"); + }; + let first = stream + .next() + .await + .expect("the stream should yield a first chunk") + .expect("the first chunk should read"); + + assert!( + first.starts_with(b"") || first.starts_with(b"` sequence, so the seam landed - // inside this comment and the assembled bids never executed. - const TRAILING_COMMENT: &str = ""; + async fn a_bypassed_response_falls_back_to_inline_at_every_seam() { + // Observed against a real origin: a page arrived with a raw + // executable ESI markup in it and no bids at all. The `` seam emitted + // the marker because the mode was `Esi`, while assembly was skipped because + // the gate had refused a key — a fallback at one seam and not the other. + // + // Bypassing is the *normal* case against a real origin, so this path runs + // far more often than the shared one. It has to produce a working page. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); + + // A `Set-Cookie` from the origin disqualifies the response, exactly as a + // real origin does. stub.push_response_with_headers( 200, - format!("

article

{TRAILING_COMMENT}") - .into_bytes(), + b"origin".to_vec(), vec![ ("content-type", "text/html; charset=utf-8"), ("cache-control", "public, max-age=300"), + ("set-cookie", "sess=1"), ], ); - let cold = String::from_utf8( - body_of(run(&settings, &services, navigation_request()).await).await, - ) - .expect("the cold document should be UTF-8"); - let warm = String::from_utf8( + let document = String::from_utf8( body_of(run(&settings, &services, navigation_request()).await).await, ) - .expect("the warm document should be UTF-8"); + .expect("response should be utf-8"); - assert_eq!(stub.recorded_request_uris().len(), 1); - for (label, document) in [("cold", &cold), ("warm", &warm)] { - assert!( - document.contains(TRAILING_COMMENT), - "the {label} document must leave the publisher comment intact: {document}" - ); - assert!( - !document.contains(AD_ASSEMBLY_SEAM), - "the {label} document must not ship an unresolved seam: {document}" - ); - assert!( - !document.contains(TEMPLATE_SEAM_PLACEHOLDER), - "the transform-owned placeholder must never reach a reader: {document}" - ); - } + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "the gate refused, so nothing may be stored" + ); + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "a marker must never be emitted when nothing will resolve it: {document}" + ); + assert!( + document.contains("window.tsjs"), + "and the reader must still get their bids: {document}" + ); } #[tokio::test] - async fn a_post_is_never_answered_from_a_cached_get() { - // `handle_publisher_request` is the `*`-method fallback route, so a publisher - // path that renders a page on GET and accepts a form or webhook on POST reaches - // here for both. Serving the cached GET to the POST swallows the mutating - // request entirely: the origin never sees it, the caller gets 200 and a page, - // and nothing anywhere reports a problem. + async fn the_seam_carries_slot_definitions_or_the_page_serves_no_ads() { + // Shared modes suppress the head slot script, so the seam is the only place + // `tsjs.adSlots` can come from. Sending only bids left it at its `[]` default, + // `adInit` defined nothing, and the page rendered perfectly with zero TS ads — + // green tests, healthy-looking page, no revenue. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - queue_shareable_html(&stub); - - // Warm the cache with a GET. - let _ = run(&settings, &services, navigation_request()).await; - assert_eq!(stub.recorded_request_uris().len(), 1); - let post = HttpRequest::builder() - .method(Method::POST) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .body(EdgeBody::from("field=value")) - .expect("should build post request"); - let _ = run(&settings, &services, post).await; + let miss = body_of(run(&settings, &services, navigation_request()).await).await; + let hit = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!( - stub.recorded_request_uris().len(), - 2, - "the POST must reach the origin rather than being answered from the \ - cached GET" - ); - assert_eq!( - cache.entries.lock().expect("should lock entries").len(), - 1, - "and it must not store a template of its own" - ); + for (label, body) in [("miss", miss), ("hit", hit)] { + let text = String::from_utf8(body).expect("utf-8"); + assert!( + text.contains("var a=JSON.parse(") && text.contains("s(b,a)"), + "{label}: the seam must carry slot definitions and hand them to the \ + scheduler: {text}" + ); + assert!( + text.contains("test-slot"), + "{label}: the slot definitions must be populated, not `[]`" + ); + } } #[tokio::test] - async fn an_authenticated_request_is_not_served_a_shared_template() { - // The stored template is perfectly cacheable; this request is not entitled - // to it. The store gate cannot express that, because it is a property of - // the reader rather than of the bytes — which is why the lookup re-checks. + async fn an_assembled_response_is_private_even_when_the_ad_stack_is_off() { + // The private stamp was gated on `should_run_ad_stack`. A bot, prefetch, + // kill-switched or consent-denied request can still assemble an empty-bids + // document, and would have kept the origin's public caching directives — so a + // downstream cache could serve that to a later eligible reader. let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; - assert_eq!( - cache.entries.lock().expect("should lock entries").len(), - 1, - "the cold request should have populated the cache" - ); - - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test(None, consent); - let authenticated = HttpRequest::builder() + let bot = HttpRequest::builder() .method(Method::GET) .uri("https://ts.example.com/article") .header(header::HOST, "ts.example.com") .header("sec-fetch-dest", "document") - .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .header("sec-fetch-mode", "navigate") + .header( + header::USER_AGENT, + "Googlebot/2.1 (+http://www.google.com/bot.html)", + ) .body(EdgeBody::empty()) - .expect("should build authenticated request"); - let _ = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[article_slot()], - registry: None, - }, - authenticated, - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request"); + .expect("should build bot request"); + let response = run(&settings, &services, bot).await; assert_eq!( - stub.recorded_request_uris().len(), - 2, - "an authenticated request must reach the origin rather than read a \ - shared template" + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store, private"), + "an assembled response must be private whatever the ad stack decided" ); } - } - - mod template_cache_gate_tests { - //! `cache::core` stores whatever it is handed and rejects nothing, so every - //! one of these conditions is the caller's to enforce. Each is a leak vector - //! or an eligibility rule, not a preference. - use super::*; - use crate::creative_opportunities::AssemblyMode; - use edgezero_core::http::HeaderName; + #[tokio::test] + async fn a_cache_hit_keeps_the_origin_security_headers() { + // Reconstructing headers keeps origin `Set-Cookie` and caching directives out + // of a shared cache, and also silently dropped Content-Security-Policy — + // a weaker page, served faster. Policy headers are per-URL, so they belong + // with the template. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("content-security-policy", "default-src 'self'"), + ("x-frame-options", "SAMEORIGIN"), + ], + ); - fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { - let mut map = edgezero_core::http::HeaderMap::new(); - for (name, value) in pairs { - map.insert( - name.clone(), - HeaderValue::from_str(value).expect("should build header value"), - ); - } - map - } + let _ = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; - fn shareable() -> edgezero_core::http::HeaderMap { - headers(&[(header::CACHE_CONTROL, "max-age=60")]) + assert_eq!( + warm.headers() + .get(header::CONTENT_SECURITY_POLICY) + .and_then(|v| v.to_str().ok()), + Some("default-src 'self'"), + "a hit must not drop the origin's CSP" + ); + assert_eq!( + warm.headers() + .get("x-frame-options") + .and_then(|v| v.to_str().ok()), + Some("SAMEORIGIN") + ); } - /// The shipped default: no operator has stated what the origin varies on, so the - /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at - /// all disqualifies. - fn nothing_covered() -> VarySpec { - VarySpec::new([]) - } + #[tokio::test] + async fn a_cache_hit_preserves_every_repeated_policy_header_in_order() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("content-security-policy", "default-src 'self'"), + ("content-security-policy", "script-src 'self'"), + ("link", "; rel=preload; as=script"), + ("link", "; rel=preload; as=style"), + ("cross-origin-opener-policy", "same-origin"), + ("cross-origin-embedder-policy", "require-corp"), + ], + ); - #[test] - fn an_unconfigured_deployment_never_caches_a_varying_response() { - // The fail-closed default. An operator who has not stated the origin's Vary - // must not acquire a shared cache by omission — and a real origin varies on - // something, so this is the common path, not an edge case. - // Deliberately not `Accept-Encoding`: the shared path normalizes supported - // content codings to one identity template, so that header is covered - // whatever the operator configured. Using it here would test the - // structural-coverage carve-out rather than the drift guard. - let mut varying = shareable(); - varying.insert(header::VARY, HeaderValue::from_static("rsc")); + let _ = run(&settings, &services, navigation_request()).await; + let warm = run(&settings, &services, navigation_request()).await; + let values = |name: &'static str| { + warm.headers() + .get_all(name) + .iter() + .map(|value| value.to_str().expect("policy header should be text")) + .collect::>() + }; assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &varying, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ - "rsc".to_string() - ]))), - "an unstated Vary must disqualify rather than silently under-key" + values("content-security-policy"), + ["default-src 'self'", "script-src 'self'"] ); - } - - #[test] - fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { - // The backstop that makes `origin_is_cookie_independent` safe to offer. The - // operator asserts their origin ignores cookies; if the origin then says - // otherwise, the assertion loses. Without this, a wrong assertion would - // silently cross-serve personalized HTML. - let mut varying = shareable(); - varying.insert(header::VARY, HeaderValue::from_static("Cookie")); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - // The operator's assertion has already been applied here: this is - // `false` precisely because they declared independence. - false, - StatusCode::OK, - "text/html", - &varying, - &VarySpec::new(["cookie".to_string()]), - ), - Some(TemplateCacheBypassReason::VaryCookie), - "the origin's declaration must override both cookie independence and an \ - accidentally configured per-cookie key" + values("link"), + [ + "; rel=preload; as=script", + "; rel=preload; as=style" + ] ); + assert_eq!(values("cross-origin-opener-policy"), ["same-origin"]); + assert_eq!(values("cross-origin-embedder-policy"), ["require-corp"]); } - #[test] - fn a_private_directive_on_a_second_cache_control_line_is_refused() { - // `HeaderMap::get` returns the first value only. An origin that sends - // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a - // separate line means exactly what one comma-joined line would mean, but the - // second line was invisible — so a response the origin marked private was - // written to a cache shared between readers. The `Vary` reads a few lines up - // already use `get_all` for the same reason. - let mut split = edgezero_core::http::HeaderMap::new(); - split.append( - header::CACHE_CONTROL, - HeaderValue::from_static("public, max-age=300"), - ); - split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); + #[tokio::test] + async fn nonce_bearing_csp_bypasses_the_shared_template_cache() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + b"origin" + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ( + "content-security-policy", + "default-src 'self'; script-src 'nonce-reader-nonce'", + ), + ], + ); + } + + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "a directive on any Cache-Control line must disqualify the response" + stub.recorded_request_uris().len(), + 2, + "a response-bound CSP nonce and its HTML must never be reused from template cache" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() ); } - #[test] - fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { - // Same defect, the other directive that matters — `no-store` is the one an - // origin uses for a response that must not be written down anywhere. - let mut split = edgezero_core::http::HeaderMap::new(); - split.append( - header::CACHE_CONTROL, - HeaderValue::from_static("max-age=60"), + #[tokio::test] + async fn a_meta_delivered_nonce_policy_bypasses_the_shared_template_cache() { + // The response-header gate cannot see this policy at all. Storing the document + // would replay one response's nonce to every later reader — the exact thing the + // header check exists to prevent. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + br#"origin"# + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); + } + + let cold = run(&settings, &services, navigation_request()).await; + assert_eq!( + cold.headers() + .get(HEADER_X_TS_TEMPLATE_CACHE) + .and_then(|value| value.to_str().ok()), + Some("bypass-response"), + "the cold response must refuse to store a nonce-bearing document" ); - split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + let _ = body_of(cold).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable) + stub.recorded_request_uris().len(), + 2, + "the warm request must reach the origin, not a replayed nonce" + ); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty(), + "nothing nonce-bearing may reach the shared cache" ); } - #[test] - fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { - let mut split = shareable(); - split.insert( - header::HeaderName::from_static(name), - HeaderValue::from_static("no-store"), - ); - - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "template cache must fail closed on the CDN-specific policy header {name}" + #[tokio::test] + async fn a_nonce_attribute_without_a_policy_bypasses_the_shared_template_cache() { + // Fail closed: the attributes say the document was written for a per-response + // policy, whether or not the policy itself survived to this scan. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + for _ in 0..2 { + stub.push_response_with_headers( + 200, + br#"origin"# + .to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], ); } - } - #[test] - fn unsupported_vendor_freshness_does_not_authorize_template_cache() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES - .iter() - .filter(|name| **name != "surrogate-control") - { - let mut split = shareable(); - split.insert( - header::HeaderName::from_static(name), - HeaderValue::from_static("max-age=60"), - ); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let _ = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "the Fastly exception must not authorize the vendor policy {name}" - ); - } + assert_eq!(stub.recorded_request_uris().len(), 2); + assert!( + cache + .entries + .lock() + .expect("should lock entries") + .is_empty() + ); } - #[test] - fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { - let publisher_headers = headers(&[ - (header::CACHE_CONTROL, "public, max-age=60"), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", - ), - ]); - - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), - ), - Ok(Duration::from_secs(300)), - "Fastly's edge freshness should take precedence over the shorter browser \ - lifetime, while the configured safety ceiling remains authoritative" + #[tokio::test] + async fn a_body_close_in_script_data_corrupts_neither_the_cold_nor_the_warm_document() { + // No structural `` anywhere: a reverse byte search takes the string + // literal, and the payload's `` then terminates the publisher's script + // — in the response served cold and in the template every warm reader gets. + const PUBLISHER_SCRIPT: &str = r#""#; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + format!("{PUBLISHER_SCRIPT}

article

").into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], ); - } - #[test] - fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { - for (cache_control, surrogate_control, expected) in [ - ("public, max-age=300", "max-age=30", 30), - ("public, max-age=30", "max-age=300", 300), - ] { - let publisher_headers = headers(&[ - (header::CACHE_CONTROL, cache_control), - ( - header::HeaderName::from_static("surrogate-control"), - surrogate_control, - ), - ]); + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the cold document should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the warm document should be UTF-8"); - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test( - ¬hing_covered(), - Duration::from_secs(600), - ), - ), - Ok(Duration::from_secs(expected)) + assert_eq!( + stub.recorded_request_uris().len(), + 1, + "the document is otherwise shareable and must still be stored and reused" + ); + for (label, document) in [("cold", &cold), ("warm", &warm)] { + assert!( + document.contains(PUBLISHER_SCRIPT), + "the {label} document must carry the publisher's script byte for byte: {document}" + ); + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "the {label} document must not ship an unresolved seam: {document}" ); } } - #[test] - fn surrogate_stale_windows_do_not_extend_fresh_reuse() { - let publisher_headers = headers(&[ - (header::CACHE_CONTROL, "public, max-age=300"), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=20, stale-while-revalidate=600, stale-if-error=1200", - ), - (header::AGE, "10"), - ]); + #[tokio::test] + async fn a_body_close_in_trailing_comment_data_does_not_attract_the_seam() { + // The reverse search took the *last* `` sequence, so the seam landed + // inside this comment and the assembled bids never executed. + const TRAILING_COMMENT: &str = ""; + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + stub.push_response_with_headers( + 200, + format!("

article

{TRAILING_COMMENT}") + .into_bytes(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ], + ); - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), - ), - Ok(Duration::from_secs(10)), - "stale windows are validated metadata, not fresh template cache lifetime" - ); - } - - #[test] - fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { - for (policy, expected) in [ - ("max-age", TemplateCacheBypassReason::MalformedCachePolicy), - ( - "max-age=30, max-age=60", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - "max-age=tomorrow", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - "max-age=30, public", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - "stale-if-error=60", - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ("max-age=0", TemplateCacheBypassReason::NoPositiveFreshness), - ( - "max-age=30,", - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ] { - let mut publisher_headers = shareable(); - publisher_headers.insert( - header::HeaderName::from_static("surrogate-control"), - HeaderValue::from_str(policy).expect("should build Surrogate-Control"), - ); + let cold = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the cold document should be UTF-8"); + let warm = String::from_utf8( + body_of(run(&settings, &services, navigation_request()).await).await, + ) + .expect("the warm document should be UTF-8"); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - ¬hing_covered(), - ), - Some(expected), - "`{policy}` must fail closed" + assert_eq!(stub.recorded_request_uris().len(), 1); + for (label, document) in [("cold", &cold), ("warm", &warm)] { + assert!( + document.contains(TRAILING_COMMENT), + "the {label} document must leave the publisher comment intact: {document}" ); - } - } - - #[test] - fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { - for directive in ["private", "no-store", "no-cache"] { - let mut publisher_headers = shareable(); - publisher_headers.insert( - header::HeaderName::from_static("surrogate-control"), - HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + assert!( + !document.contains(AD_ASSEMBLY_SEAM), + "the {label} document must not ship an unresolved seam: {document}" ); - - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "`{directive}` must remain authoritative" + assert!( + !document.contains(TEMPLATE_SEAM_PLACEHOLDER), + "the transform-owned placeholder must never reach a reader: {document}" ); } } - #[test] - fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { - for directive in ["private", "no-store", "no-cache"] { - let publisher_headers = headers(&[ - ( - header::CACHE_CONTROL, - &format!("public, max-age=60, {directive}"), - ), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=1200", - ), - ]); + #[tokio::test] + async fn a_post_is_never_answered_from_a_cached_get() { + // `handle_publisher_request` is the `*`-method fallback route, so a publisher + // path that renders a page on GET and accepts a form or webhook on POST reaches + // here for both. Serving the cached GET to the POST swallows the mutating + // request entirely: the origin never sees it, the caller gets 200 and a page, + // and nothing anywhere reports a problem. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "standard `{directive}` must refuse template cache even with positive edge freshness" - ); - } - } + // Warm the cache with a GET. + let _ = run(&settings, &services, navigation_request()).await; + assert_eq!(stub.recorded_request_uris().len(), 1); - #[test] - fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { - let publisher_headers = headers(&[( - header::HeaderName::from_static("surrogate-control"), - "max-age=1200", - )]); + let post = HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .body(EdgeBody::from("field=value")) + .expect("should build post request"); + let _ = run(&settings, &services, post).await; assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &publisher_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), - ), - Ok(Duration::from_secs(300)), - "Fastly edge freshness should not require browser freshness" - ); - } - - #[test] - fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { - // The other direction: reading every value must not turn an ordinary - // multi-line `Cache-Control` into a bypass, or the fix would disable the - // cache instead of tightening it. - let mut split = edgezero_core::http::HeaderMap::new(); - split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); - split.append( - header::CACHE_CONTROL, - HeaderValue::from_static("max-age=60"), + stub.recorded_request_uris().len(), + 2, + "the POST must reach the origin rather than being answered from the \ + cached GET" ); - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &split, - ¬hing_covered(), - ), - None + cache.entries.lock().expect("should lock entries").len(), + 1, + "and it must not store a template of its own" ); } - #[test] - fn origin_freshness_is_positive_age_adjusted_and_capped() { - let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); - assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &fresh_headers, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), - ), - Ok(Duration::from_secs(60)) - ); + #[tokio::test] + async fn an_authenticated_request_is_not_served_a_shared_template() { + // The stored template is perfectly cacheable; this request is not entitled + // to it. The store gate cannot express that, because it is a property of + // the reader rather than of the bytes — which is why the lookup re-checks. + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + queue_shareable_html(&stub); - let aged = headers(&[ - (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), - (header::AGE, "35"), - ]); + let _ = run(&settings, &services, navigation_request()).await; assert_eq!( - template_cache_ttl( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &aged, - &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), - ), - Ok(Duration::from_secs(15)) + cache.entries.lock().expect("should lock entries").len(), + 1, + "the cold request should have populated the cache" ); - let old_date_without_age = headers(&[ - (header::CACHE_CONTROL, "public, max-age=60"), - (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), - ]); - let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") - .expect("should parse fixture time"); - assert_eq!( - origin_shared_ttl_at( - &old_date_without_age, - one_minute_later, - Duration::from_secs(60), - ), - Err(TemplateCacheBypassReason::NoPositiveFreshness), - "an old Date is apparent age even when an upstream omitted Age" - ); - } - - #[test] - fn zero_exhausted_missing_and_malformed_freshness_are_refused() { - for (map, expected) in [ - ( - headers(&[(header::CACHE_CONTROL, "max-age=0")]), - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ( - headers(&[(header::CACHE_CONTROL, "public")]), - TemplateCacheBypassReason::NoPositiveFreshness, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ( - headers(&[(header::CACHE_CONTROL, "max-age=+60")]), - TemplateCacheBypassReason::MalformedCachePolicy, - ), - ] { - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &map, - ¬hing_covered(), - ), - Some(expected) - ); - } - } - - #[test] - fn expires_can_authorize_but_never_extend_an_expired_response() { - let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") - .expect("should parse fixture time"); - let fresh = headers(&[ - (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), - (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), - ]); - assert_eq!( - origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), - Ok(Duration::from_secs(30)) + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let authenticated = HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build authenticated request"); + let registry = test_registry(&settings); + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + authenticated, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); - let expired = headers(&[ - (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), - (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), - ]); assert_eq!( - origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), - Err(TemplateCacheBypassReason::NoPositiveFreshness) + stub.recorded_request_uris().len(), + 2, + "an authenticated request must reach the origin rather than read a \ + shared template" ); } + } - #[test] - fn request_semantics_bypass_template_cache_except_for_a_max_age_zero_reload() { - for (name, value) in [ - (header::CACHE_CONTROL, "no-cache"), - (header::CACHE_CONTROL, "max-age=30"), - (header::CACHE_CONTROL, "max-age=\"0"), - (header::CACHE_CONTROL, "min-fresh=10"), - (header::CACHE_CONTROL, "no-store"), - (header::PRAGMA, "no-cache"), - (header::PRAGMA, "legacy-extension, no-cache"), - (header::RANGE, "bytes=0-99"), - (header::IF_NONE_MATCH, "\"etag\""), - (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), - ] { - let map = headers(&[(name.clone(), value)]); - assert!( - request_bypasses_template_cache(&map), - "{name}: {value} must bypass" + mod template_cache_gate_tests { + //! `cache::core` stores whatever it is handed and rejects nothing, so every + //! one of these conditions is the caller's to enforce. Each is a leak vector + //! or an eligibility rule, not a preference. + + use super::*; + use crate::creative_opportunities::AssemblyMode; + use edgezero_core::http::HeaderName; + + fn headers(pairs: &[(HeaderName, &str)]) -> edgezero_core::http::HeaderMap { + let mut map = edgezero_core::http::HeaderMap::new(); + for (name, value) in pairs { + map.insert( + name.clone(), + HeaderValue::from_str(value).expect("should build header value"), ); } - assert!( - !request_bypasses_template_cache(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), - "a browser reload may reuse template cache because the assembled response and auction \ - are still rebuilt for this reader" - ); - assert!(!request_bypasses_template_cache(&headers(&[( - header::CACHE_CONTROL, - "public" - )]))); + map + } + + fn shareable() -> edgezero_core::http::HeaderMap { + headers(&[(header::CACHE_CONTROL, "max-age=60")]) + } + + /// The shipped default: no operator has stated what the origin varies on, so the + /// key covers nothing. Responses without a `Vary` are unaffected; any `Vary` at + /// all disqualifies. + fn nothing_covered() -> VarySpec { + VarySpec::new([]) } #[test] - fn a_wildcard_vary_is_refused() { - // `VarySpec::uncovered_by` filters `*` out, with a comment saying the - // eligibility gate handles it. It did not — nothing rejected the wildcard, so - // a response the origin said no key can select was shareable. + fn an_unconfigured_deployment_never_caches_a_varying_response() { + // The fail-closed default. An operator who has not stated the origin's Vary + // must not acquire a shared cache by omission — and a real origin varies on + // something, so this is the common path, not an edge case. + // Deliberately not `Accept-Encoding`: the shared path normalizes supported + // content codings to one identity template, so that header is covered + // whatever the operator configured. Using it here would test the + // structural-coverage carve-out rather than the drift guard. let mut varying = shareable(); - varying.insert(header::VARY, HeaderValue::from_static("*")); + varying.insert(header::VARY, HeaderValue::from_static("rsc")); assert_eq!( template_cache_bypass_reason( @@ -15691,43 +16930,54 @@ mod tests { &varying, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::VaryWildcard) + Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ + "rsc".to_string() + ]))), + "an unstated Vary must disqualify rather than silently under-key" ); } #[test] - fn a_fully_covered_vary_is_cacheable() { + fn an_origin_that_varies_on_cookie_is_refused_even_when_declared_independent() { + // The backstop that makes `origin_is_cookie_independent` safe to offer. The + // operator asserts their origin ignores cookies; if the origin then says + // otherwise, the assertion loses. Without this, a wrong assertion would + // silently cross-serve personalized HTML. let mut varying = shareable(); - varying.insert( - header::VARY, - HeaderValue::from_static("rsc, Accept-Encoding"), - ); + varying.insert(header::VARY, HeaderValue::from_static("Cookie")); assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, false, + // The operator's assertion has already been applied here: this is + // `false` precisely because they declared independence. false, StatusCode::OK, "text/html", &varying, - &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + &VarySpec::new(["cookie".to_string()]), ), - None, - "a key covering everything the origin varies on is safe to store" + Some(TemplateCacheBypassReason::VaryCookie), + "the origin's declaration must override both cookie independence and an \ + accidentally configured per-cookie key" ); } #[test] - fn config_drift_names_the_missing_header() { - // The failure this guards: the origin adds a header to its Vary, nobody - // updates config, and requests differing only in that header start sharing a - // template. The reason must name it, or diagnosing means a bisect. - let mut varying = shareable(); - varying.insert( - header::VARY, - HeaderValue::from_static("rsc, next-router-prefetch"), + fn a_private_directive_on_a_second_cache_control_line_is_refused() { + // `HeaderMap::get` returns the first value only. An origin that sends + // `Cache-Control: public, max-age=300` and then `Cache-Control: private` on a + // separate line means exactly what one comma-joined line would mean, but the + // second line was invisible — so a response the origin marked private was + // written to a cache shared between readers. The `Vary` reads a few lines up + // already use `get_all` for the same reason. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=300"), ); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("private")); assert_eq!( template_cache_bypass_reason( @@ -15736,41 +16986,25 @@ mod tests { false, StatusCode::OK, "text/html", - &varying, - &VarySpec::new(["rsc".to_string()]), + &split, + ¬hing_covered(), ), - Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ - "next-router-prefetch".to_string() - ]))), - "the uncovered header must be named" + Some(TemplateCacheBypassReason::OriginNotShareable), + "a directive on any Cache-Control line must disqualify the response" ); } #[test] - fn a_vary_split_across_repeated_headers_is_still_checked() { - // Vary is a list header, so an origin may send it once or many times. Reading - // only the first would let the rest through unkeyed. - let mut varying = shareable(); - varying.append(header::VARY, HeaderValue::from_static("rsc")); - varying.append(header::VARY, HeaderValue::from_static("cookie")); - - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::OK, - "text/html", - &varying, - &VarySpec::new(["rsc".to_string()]), - ), - Some(TemplateCacheBypassReason::VaryCookie), - "a repeated Vary header must not hide names behind the first value" + fn a_no_store_directive_on_a_second_cache_control_line_is_refused() { + // Same defect, the other directive that matters — `no-store` is the one an + // origin uses for a response that must not be written down anywhere. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), ); - } + split.append(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); - #[test] - fn a_plain_shareable_html_200_is_cacheable() { assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -15778,102 +17012,182 @@ mod tests { false, StatusCode::OK, "text/html", - &shareable(), + &split, ¬hing_covered(), ), - None, - "ESI shareable HTML 200 should be eligible" + Some(TemplateCacheBypassReason::OriginNotShareable) ); } #[test] - fn inline_mode_never_writes_a_template() { - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Inline, - false, - false, - StatusCode::OK, - "text/html", - &shareable(), - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::InlineMode), - "inline has no shared template to write" - ); + fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { + for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("no-store"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "template cache must fail closed on the CDN-specific policy header {name}" + ); + } } #[test] - fn an_authorized_request_is_never_cached() { - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - true, - false, - StatusCode::OK, - "text/html", - &shareable(), - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::AuthorizedRequest), - "an authenticated response must not enter a shared cache" - ); + fn unsupported_vendor_freshness_does_not_authorize_template_cache() { + for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES + .iter() + .filter(|name| **name != "surrogate-control") + { + let mut split = shareable(); + split.insert( + header::HeaderName::from_static(name), + HeaderValue::from_static("max-age=60"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &split, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "the Fastly exception must not authorize the vendor policy {name}" + ); + } } #[test] - fn cookie_policy_disqualifies_even_without_set_cookie() { - // The dangerous case: session established on an earlier request, so this - // response carries no Set-Cookie, has no Cache-Control at all, is a 200, - // and is HTML — yet is personalized because TS forwarded the Cookie to - // origin unchanged. Every other condition reports it cacheable. - let no_cache_control = edgezero_core::http::HeaderMap::new(); + fn observed_fastly_surrogate_policy_uses_edge_freshness_capped_by_configuration() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200, stale-while-revalidate=21600, stale-if-error=604800", + ), + ]); + assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, false, - true, + false, StatusCode::OK, "text/html", - &no_cache_control, - ¬hing_covered(), + &publisher_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), ), - Some(TemplateCacheBypassReason::CookiePolicy), - "cookie-personalized HTML must not become a shared template" + Ok(Duration::from_secs(300)), + "Fastly's edge freshness should take precedence over the shorter browser \ + lifetime, while the configured safety ceiling remains authoritative" ); } #[test] - fn an_origin_set_cookie_is_never_cached() { - let with_cookie = headers(&[ - (header::CACHE_CONTROL, "max-age=60"), - (header::SET_COOKIE, "sid=abc; Path=/"), + fn fastly_surrogate_freshness_takes_precedence_over_standard_freshness() { + for (cache_control, surrogate_control, expected) in [ + ("public, max-age=300", "max-age=30", 30), + ("public, max-age=30", "max-age=300", 300), + ] { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, cache_control), + ( + header::HeaderName::from_static("surrogate-control"), + surrogate_control, + ), + ]); + + assert_eq!( + template_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + &TemplateCachePolicy::for_test( + ¬hing_covered(), + Duration::from_secs(600), + ), + ), + Ok(Duration::from_secs(expected)) + ); + } + } + + #[test] + fn surrogate_stale_windows_do_not_extend_fresh_reuse() { + let publisher_headers = headers(&[ + (header::CACHE_CONTROL, "public, max-age=300"), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=20, stale-while-revalidate=600, stale-if-error=1200", + ), + (header::AGE, "10"), ]); + assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, false, false, StatusCode::OK, "text/html", - &with_cookie, - ¬hing_covered(), + &publisher_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(600),), ), - Some(TemplateCacheBypassReason::OriginSetCookie), - "caching this would replay one visitor's cookie to the next" + Ok(Duration::from_secs(10)), + "stale windows are validated metadata, not fresh template cache lifetime" ); } #[test] - fn non_shareable_cache_control_is_refused_case_insensitively() { - for directive in [ - "private", - "no-store", - "no-cache", - "Private, max-age=60", - "NO-STORE", - "public, No-Cache", + fn ambiguous_or_unsupported_surrogate_policy_fails_closed() { + for (policy, expected) in [ + ("max-age", TemplateCacheBypassReason::MalformedCachePolicy), + ( + "max-age=30, max-age=60", + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + "max-age=tomorrow", + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + "max-age=30, public", + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + "stale-if-error=60", + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ("max-age=0", TemplateCacheBypassReason::NoPositiveFreshness), + ( + "max-age=30,", + TemplateCacheBypassReason::MalformedCachePolicy, + ), ] { - let map = headers(&[(header::CACHE_CONTROL, directive)]); + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(policy).expect("should build Surrogate-Control"), + ); + assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -15881,79 +17195,104 @@ mod tests { false, StatusCode::OK, "text/html", - &map, + &publisher_headers, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::OriginNotShareable), - "`{directive}` should disqualify the response" + Some(expected), + "`{policy}` must fail closed" ); } } #[test] - fn a_datadome_block_is_refused_by_the_status_check() { - // DataDome replaces the document with a 403 - // (`integrations/datadome/protection.rs:778`). There is no separate - // marker to detect, and none is needed. - assert_eq!( - template_cache_bypass_reason( - AssemblyMode::Esi, - false, - false, - StatusCode::FORBIDDEN, - "text/html", - &shareable(), - ¬hing_covered(), - ), - Some(TemplateCacheBypassReason::NonOkStatus), - "a blocked document must not become the shared template" - ); + fn restrictive_surrogate_policy_is_never_overridden_by_standard_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let mut publisher_headers = shareable(); + publisher_headers.insert( + header::HeaderName::from_static("surrogate-control"), + HeaderValue::from_str(directive).expect("should build Surrogate-Control"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &publisher_headers, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "`{directive}` must remain authoritative" + ); + } } #[test] - fn non_html_is_refused() { - for content_type in ["text/x-component", "application/json", ""] { + fn restrictive_standard_policy_is_never_overridden_by_surrogate_freshness() { + for directive in ["private", "no-store", "no-cache"] { + let publisher_headers = headers(&[ + ( + header::CACHE_CONTROL, + &format!("public, max-age=60, {directive}"), + ), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + ), + ]); + assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, false, false, StatusCode::OK, - content_type, - &shareable(), + "text/html", + &publisher_headers, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::NotHtml), - "`{content_type}` has no HTML template to transform" + Some(TemplateCacheBypassReason::OriginNotShareable), + "standard `{directive}` must refuse template cache even with positive edge freshness" ); } } #[test] - fn unsupported_content_encoding_is_refused_before_representation_headers_change() { - let map = headers(&[ - (header::CACHE_CONTROL, "public, max-age=60"), - (header::CONTENT_ENCODING, "zstd"), - ]); + fn surrogate_control_can_authorize_fastly_edge_freshness_without_browser_freshness() { + let publisher_headers = headers(&[( + header::HeaderName::from_static("surrogate-control"), + "max-age=1200", + )]); + assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, false, false, StatusCode::OK, "text/html", - &map, - ¬hing_covered(), + &publisher_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(300),), ), - Some(TemplateCacheBypassReason::UnsupportedContentEncoding) + Ok(Duration::from_secs(300)), + "Fastly edge freshness should not require browser freshness" ); + } - let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); - repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); - repeated.append( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), + #[test] + fn repeated_cache_control_lines_without_a_disqualifier_still_cache() { + // The other direction: reading every value must not turn an ordinary + // multi-line `Cache-Control` into a bypass, or the fix would disable the + // cache instead of tightening it. + let mut split = edgezero_core::http::HeaderMap::new(); + split.append(header::CACHE_CONTROL, HeaderValue::from_static("public")); + split.append( + header::CACHE_CONTROL, + HeaderValue::from_static("max-age=60"), ); + assert_eq!( template_cache_bypass_reason( AssemblyMode::Esi, @@ -15961,182 +17300,782 @@ mod tests { false, StatusCode::OK, "text/html", - &repeated, + &split, ¬hing_covered(), ), - Some(TemplateCacheBypassReason::MalformedRepresentationHeaders) + None ); } #[test] - fn leak_vectors_are_reported_before_mere_ineligibility() { - // A response that fails several conditions should name the most serious - // one, so an operator reading the log sees the security reason rather - // than a content-type quibble. - let map = headers(&[ - (header::CACHE_CONTROL, "private"), - (header::SET_COOKIE, "sid=abc"), - ]); + fn origin_freshness_is_positive_age_adjusted_and_capped() { + let fresh_headers = headers(&[(header::CACHE_CONTROL, "public, max-age=300")]); assert_eq!( - template_cache_bypass_reason( + template_cache_ttl( AssemblyMode::Esi, - true, false, - StatusCode::FORBIDDEN, - "application/json", - &map, - ¬hing_covered(), + false, + StatusCode::OK, + "text/html", + &fresh_headers, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), ), - Some(TemplateCacheBypassReason::AuthorizedRequest), - "authorization is the most serious disqualifier and should win" + Ok(Duration::from_secs(60)) ); - } - } - - mod template_neutrality_tests { - //! The gate for #1009's shared-template design. - //! - //! An "absence of per-user values" scan is not sufficient here: the bug - //! that nearly shipped was a *conditionally present* element whose own - //! content was per-URL. These tests assert byte-identity across requests - //! that differ only in the gating decision. - - use super::*; - use crate::creative_opportunities::{ - AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, - }; - - pub(super) fn slot() -> CreativeOpportunitySlot { - CreativeOpportunitySlot { - id: "atf".to_string(), - gam_unit_path: Some("/99999/example/home".to_string()), - div_id: Some("ad-atf".to_string()), - page_patterns: vec!["/**".to_string()], - formats: vec![CreativeOpportunityFormat { - width: 300, - height: 250, - media_type: MediaType::Banner, - }], - floor_price: None, - targeting: Default::default(), - providers: Default::default(), - compiled_patterns: Vec::new(), - compiled_unit: None, - } - } - - pub(super) fn settings_with_slots() -> Settings { - let mut settings = crate::test_support::tests::create_test_settings(); - // Construct the section rather than mutating it if present: the shared - // fixture does not carry `[creative_opportunities]`, and an `if let - // Some(..)` here would silently no-op and make the inline assertion - // below vacuous. - settings.creative_opportunities = Some(CreativeOpportunitiesConfig { - enabled: true, - gam_network_id: "99999".to_string(), - auction_timeout_ms: Some(500), - price_granularity: Default::default(), - section_root: None, - assembly_mode: None, - template_cache_vary: None, - template_cache_max_age_seconds: None, - template_cache_key_cookies: None, - template_cache_bypass_cookies: None, - origin_is_cookie_independent: None, - origin_readthrough_enabled: None, - section_segment: None, - slot: vec![slot()], - }); - settings - } - - #[test] - fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { - let settings = settings_with_slots(); - let slots = [slot()]; - let mode = AssemblyMode::Esi; - let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); - let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); + let aged = headers(&[ + (header::CACHE_CONTROL, "s-maxage=50, max-age=300"), + (header::AGE, "35"), + ]); assert_eq!( - ran, did_not_run, - "{mode:?}: the template must be byte-identical whether or not the ad \ - stack ran; a cached object cannot carry one request's consent, bot, \ - prefetch or kill-switch decision" + template_cache_ttl( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &aged, + &TemplateCachePolicy::for_test(¬hing_covered(), Duration::from_secs(60),), + ), + Ok(Duration::from_secs(15)) ); + + let old_date_without_age = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ]); + let one_minute_later = httpdate::parse_http_date("Wed, 12 Aug 2026 08:01:00 GMT") + .expect("should parse fixture time"); assert_eq!( - ran, None, - "{mode:?}: adSlots belongs in the per-request seam, not the template" + origin_shared_ttl_at( + &old_date_without_age, + one_minute_later, + Duration::from_secs(60), + ), + Err(TemplateCacheBypassReason::NoPositiveFreshness), + "an old Date is apparent age even when an upstream omitted Age" ); } #[test] - fn inline_mode_keeps_its_request_dependent_behaviour() { - // Inline responses are per-navigation and never shared, so gating is - // correct there. This guards against "fixing" the shared-mode bug by - // breaking the shipped path. - let settings = settings_with_slots(); - let slots = [slot()]; - - assert!( - template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") - .is_some(), - "inline should emit adSlots when the ad stack runs" - ); - assert_eq!( - template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), - None, - "inline should emit nothing when the ad stack does not run" - ); + fn zero_exhausted_missing_and_malformed_freshness_are_refused() { + for (map, expected) in [ + ( + headers(&[(header::CACHE_CONTROL, "max-age=0")]), + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=60"), (header::AGE, "60")]), + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "public")]), + TemplateCacheBypassReason::NoPositiveFreshness, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=tomorrow")]), + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=\"60")]), + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ( + headers(&[(header::CACHE_CONTROL, "max-age=+60")]), + TemplateCacheBypassReason::MalformedCachePolicy, + ), + ] { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(expected) + ); + } } #[test] - fn shared_modes_are_neutral_across_differing_slot_matches() { - // Slot matching folds in the request path. Under a shared mode even - // that must not reach the template. - let settings = settings_with_slots(); - - let matched = template_ad_slots_script( - AssemblyMode::Esi, - true, - &settings, - &[slot()], - "/news/article", + fn expires_can_authorize_but_never_extend_an_expired_response() { + let now = httpdate::parse_http_date("Wed, 12 Aug 2026 08:00:00 GMT") + .expect("should parse fixture time"); + let fresh = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:00:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); + assert_eq!( + origin_shared_ttl_at(&fresh, now, Duration::from_secs(60)), + Ok(Duration::from_secs(30)) ); - let unmatched = - template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); + let expired = headers(&[ + (header::DATE, "Wed, 12 Aug 2026 08:01:00 GMT"), + (header::EXPIRES, "Wed, 12 Aug 2026 08:00:30 GMT"), + ]); assert_eq!( - matched, unmatched, - "the template must not vary with slot matching under a shared mode" + origin_shared_ttl_at(&expired, now, Duration::from_secs(60)), + Err(TemplateCacheBypassReason::NoPositiveFreshness) ); } - } + + #[test] + fn request_semantics_bypass_template_cache_except_for_a_max_age_zero_reload() { + for (name, value) in [ + (header::CACHE_CONTROL, "no-cache"), + (header::CACHE_CONTROL, "max-age=30"), + (header::CACHE_CONTROL, "max-age=\"0"), + (header::CACHE_CONTROL, "min-fresh=10"), + (header::CACHE_CONTROL, "no-store"), + (header::PRAGMA, "no-cache"), + (header::PRAGMA, "legacy-extension, no-cache"), + (header::RANGE, "bytes=0-99"), + (header::IF_NONE_MATCH, "\"etag\""), + (header::IF_MODIFIED_SINCE, "Wed, 12 Aug 2026 08:00:00 GMT"), + ] { + let map = headers(&[(name.clone(), value)]); + assert!( + request_bypasses_template_cache(&map), + "{name}: {value} must bypass" + ); + } + assert!( + !request_bypasses_template_cache(&headers(&[(header::CACHE_CONTROL, "max-age=0")])), + "a browser reload may reuse template cache because the assembled response and auction \ + are still rebuilt for this reader" + ); + assert!(!request_bypasses_template_cache(&headers(&[( + header::CACHE_CONTROL, + "public" + )]))); + } + + #[test] + fn a_wildcard_vary_is_refused() { + // `VarySpec::uncovered_by` filters `*` out, with a comment saying the + // eligibility gate handles it. It did not — nothing rejected the wildcard, so + // a response the origin said no key can select was shareable. + let mut varying = shareable(); + varying.insert(header::VARY, HeaderValue::from_static("*")); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::VaryWildcard) + ); + } + + #[test] + fn a_fully_covered_vary_is_cacheable() { + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, Accept-Encoding"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string(), "accept-encoding".to_string()]), + ), + None, + "a key covering everything the origin varies on is safe to store" + ); + } + + #[test] + fn config_drift_names_the_missing_header() { + // The failure this guards: the origin adds a header to its Vary, nobody + // updates config, and requests differing only in that header start sharing a + // template. The reason must name it, or diagnosing means a bisect. + let mut varying = shareable(); + varying.insert( + header::VARY, + HeaderValue::from_static("rsc, next-router-prefetch"), + ); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(TemplateCacheBypassReason::VaryNotCovered(VaryGap(vec![ + "next-router-prefetch".to_string() + ]))), + "the uncovered header must be named" + ); + } + + #[test] + fn a_vary_split_across_repeated_headers_is_still_checked() { + // Vary is a list header, so an origin may send it once or many times. Reading + // only the first would let the rest through unkeyed. + let mut varying = shareable(); + varying.append(header::VARY, HeaderValue::from_static("rsc")); + varying.append(header::VARY, HeaderValue::from_static("cookie")); + + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &varying, + &VarySpec::new(["rsc".to_string()]), + ), + Some(TemplateCacheBypassReason::VaryCookie), + "a repeated Vary header must not hide names behind the first value" + ); + } + + #[test] + fn a_plain_shareable_html_200_is_cacheable() { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + None, + "ESI shareable HTML 200 should be eligible" + ); + } + + #[test] + fn inline_mode_never_writes_a_template() { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Inline, + false, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::InlineMode), + "inline has no shared template to write" + ); + } + + #[test] + fn an_authorized_request_is_never_cached() { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::OK, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::AuthorizedRequest), + "an authenticated response must not enter a shared cache" + ); + } + + #[test] + fn cookie_policy_disqualifies_even_without_set_cookie() { + // The dangerous case: session established on an earlier request, so this + // response carries no Set-Cookie, has no Cache-Control at all, is a 200, + // and is HTML — yet is personalized because TS forwarded the Cookie to + // origin unchanged. Every other condition reports it cacheable. + let no_cache_control = edgezero_core::http::HeaderMap::new(); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + true, + StatusCode::OK, + "text/html", + &no_cache_control, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::CookiePolicy), + "cookie-personalized HTML must not become a shared template" + ); + } + + #[test] + fn an_origin_set_cookie_is_never_cached() { + let with_cookie = headers(&[ + (header::CACHE_CONTROL, "max-age=60"), + (header::SET_COOKIE, "sid=abc; Path=/"), + ]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &with_cookie, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginSetCookie), + "caching this would replay one visitor's cookie to the next" + ); + } + + #[test] + fn non_shareable_cache_control_is_refused_case_insensitively() { + for directive in [ + "private", + "no-store", + "no-cache", + "Private, max-age=60", + "NO-STORE", + "public, No-Cache", + ] { + let map = headers(&[(header::CACHE_CONTROL, directive)]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::OriginNotShareable), + "`{directive}` should disqualify the response" + ); + } + } + + #[test] + fn a_bot_protection_block_is_refused_by_the_status_check() { + // A bot protection module replaces the document with a 403. There + // is no separate marker to detect, and none is needed. + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::FORBIDDEN, + "text/html", + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::NonOkStatus), + "a blocked document must not become the shared template" + ); + } + + #[test] + fn non_html_is_refused() { + for content_type in ["text/x-component", "application/json", ""] { + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + content_type, + &shareable(), + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::NotHtml), + "`{content_type}` has no HTML template to transform" + ); + } + } + + #[test] + fn unsupported_content_encoding_is_refused_before_representation_headers_change() { + let map = headers(&[ + (header::CACHE_CONTROL, "public, max-age=60"), + (header::CONTENT_ENCODING, "zstd"), + ]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &map, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::UnsupportedContentEncoding) + ); + + let mut repeated = headers(&[(header::CACHE_CONTROL, "public, max-age=60")]); + repeated.append(header::CONTENT_TYPE, HeaderValue::from_static("text/html")); + repeated.append( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + false, + false, + StatusCode::OK, + "text/html", + &repeated, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::MalformedRepresentationHeaders) + ); + } + + #[test] + fn leak_vectors_are_reported_before_mere_ineligibility() { + // A response that fails several conditions should name the most serious + // one, so an operator reading the log sees the security reason rather + // than a content-type quibble. + let map = headers(&[ + (header::CACHE_CONTROL, "private"), + (header::SET_COOKIE, "sid=abc"), + ]); + assert_eq!( + template_cache_bypass_reason( + AssemblyMode::Esi, + true, + false, + StatusCode::FORBIDDEN, + "application/json", + &map, + ¬hing_covered(), + ), + Some(TemplateCacheBypassReason::AuthorizedRequest), + "authorization is the most serious disqualifier and should win" + ); + } + } + + mod template_neutrality_tests { + //! The gate for #1009's shared-template design. + //! + //! An "absence of per-user values" scan is not sufficient here: the bug + //! that nearly shipped was a *conditionally present* element whose own + //! content was per-URL. These tests assert byte-identity across requests + //! that differ only in the gating decision. + + use super::*; + use crate::creative_opportunities::{ + AssemblyMode, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + + pub(super) fn slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: Some("/99999/example/home".to_string()), + div_id: Some("ad-atf".to_string()), + page_patterns: vec!["/**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + min_viewport_width: None, + max_viewport_width: None, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } + } + + pub(super) fn settings_with_slots() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + // Construct the section rather than mutating it if present: the shared + // fixture does not carry `[creative_opportunities]`, and an `if let + // Some(..)` here would silently no-op and make the inline assertion + // below vacuous. + settings.creative_opportunities = Some(CreativeOpportunitiesConfig { + enabled: true, + gam_network_id: "99999".to_string(), + auction_timeout_ms: Some(500), + price_granularity: Default::default(), + section_root: None, + assembly_mode: None, + template_cache_vary: None, + template_cache_max_age_seconds: None, + template_cache_key_cookies: None, + template_cache_bypass_cookies: None, + origin_is_cookie_independent: None, + origin_readthrough_enabled: None, + section_segment: None, + slot: vec![slot()], + }); + settings + } + + #[test] + fn shared_modes_emit_no_head_script_regardless_of_the_gating_decision() { + let settings = settings_with_slots(); + let slots = [slot()]; + let mode = AssemblyMode::Esi; + let ran = template_ad_slots_script(mode, true, &settings, &slots, "/"); + let did_not_run = template_ad_slots_script(mode, false, &settings, &slots, "/"); + + assert_eq!( + ran, did_not_run, + "{mode:?}: the template must be byte-identical whether or not the ad \ + stack ran; a cached object cannot carry one request's consent, bot, \ + prefetch or kill-switch decision" + ); + assert_eq!( + ran, None, + "{mode:?}: adSlots belongs in the per-request seam, not the template" + ); + } + + #[test] + fn inline_mode_keeps_its_request_dependent_behaviour() { + // Inline responses are per-navigation and never shared, so gating is + // correct there. This guards against "fixing" the shared-mode bug by + // breaking the shipped path. + let settings = settings_with_slots(); + let slots = [slot()]; + + assert!( + template_ad_slots_script(AssemblyMode::Inline, true, &settings, &slots, "/") + .is_some(), + "inline should emit adSlots when the ad stack runs" + ); + assert_eq!( + template_ad_slots_script(AssemblyMode::Inline, false, &settings, &slots, "/"), + None, + "inline should emit nothing when the ad stack does not run" + ); + } + + #[test] + fn shared_modes_are_neutral_across_differing_slot_matches() { + // Slot matching folds in the request path. Under a shared mode even + // that must not reach the template. + let settings = settings_with_slots(); + + let matched = template_ad_slots_script( + AssemblyMode::Esi, + true, + &settings, + &[slot()], + "/news/article", + ); + let unmatched = + template_ad_slots_script(AssemblyMode::Esi, true, &settings, &[], "/other"); + + assert_eq!( + matched, unmatched, + "the template must not vary with slot matching under a shared mode" + ); + } + } + + #[tokio::test] + async fn publisher_asset_cache_policy_applies_to_non_html_response() { + let settings = settings_from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, immutable"), + "matched publisher asset should receive immutable browser policy" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "matched publisher asset should receive Fastly edge policy" + ); + } + + #[tokio::test] + async fn the_origins_answer_beneath_the_route_namespace_loses_the_deployment_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 404, + b"not found".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "text/plain"), + ("x-served-by", "cache-node-1"), + ("x-cache", "MISS"), + ("x-cache-hits", "0"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/_ts/anything") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, &services, request).await { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } + | PublisherResponse::PassThrough { response, .. } => response, + }; + + for name in ["x-served-by", "x-cache", "x-cache-hits"] { + assert!( + response.headers().get(name).is_none(), + "the origin's {name} should not reach a reader" + ); + } + } + + #[tokio::test] + async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { + let settings = settings_from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { mut response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + crate::response_privacy::apply_response_headers_with_cache_privacy( + &settings, + &mut response, + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "publisher asset with Set-Cookie must become private after finalization" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "publisher asset with Set-Cookie must not retain a shared-cache header" + ); + } #[tokio::test] - async fn publisher_asset_cache_policy_applies_to_non_html_response() { - let settings = Settings::from_toml(&format!( + async fn publisher_asset_cache_policy_skips_html_response() { + let settings = settings_from_toml(&format!( r#"{} [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" + id = "broad-publisher-path" enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" + path_glob = "/news/*.html" visibility = "public" browser_ttl_seconds = 31536000 edge_ttl_seconds = 31536000 immutable = true + fingerprint_style = "hex" "#, crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); + )); let stub = Arc::new(StubHttpClient::new()); stub.push_response_with_headers( 200, - b"png".to_vec(), + b"news".to_vec(), vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), (header::CACHE_CONTROL.as_str(), "public, max-age=60"), ], ); @@ -16145,707 +18084,1503 @@ mod tests { ); let request = HttpRequest::builder() .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") + .uri("https://publisher.example/news/story.0123abcd.html") .header(header::HOST, "publisher.example") .body(EdgeBody::empty()) .expect("should build request"); - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { response, .. } = response else { - panic!("should pass through non-HTML asset response"); - }; + let response = run_publisher_proxy(&settings, &services, request).await; + let response = match response { + PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { + response + } + PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { + panic!("should classify HTML response for processing") + } + }; + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=60"), + "asset policy must not cache publisher HTML" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "HTML response must not receive a shared-cache header" + ); + } + + mod ssat_cache_policy_tests { + use super::*; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + use crate::platform::{ + ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, + }; + use crate::test_support::tests::crate_test_settings_str; + + const ORIGIN_ETAG: &str = "\"origin-tag\""; + const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; + const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; + + struct DispatchingTestProvider; + + struct RangeAwareHttpClient { + stub: StubHttpClient, + } + + impl RangeAwareHttpClient { + fn new() -> Self { + Self { + stub: StubHttpClient::new(), + } + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RangeAwareHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + if request.request.headers().contains_key(header::RANGE) { + self.stub.push_response_with_headers( + 206, + b"partial".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-range", "bytes 0-18/39"), + ], + ); + } else { + self.stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + } + self.stub.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.stub.send_async(request).await + } + + async fn select( + &self, + pending_requests: Vec, + ) -> Result> { + self.stub.select(pending_requests).await + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DispatchingTestProvider { + fn provider_name(&self) -> &str { + UNEXPECTED_304_PROVIDER + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let request = PlatformHttpRequest::new( + HttpRequest::builder() + .method(Method::POST) + .uri("https://bidder.example.com/navigation-bids") + .body(EdgeBody::empty()) + .expect("should build test provider request"), + UNEXPECTED_304_BACKEND, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "test provider launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run for an unexpected origin 304"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name( + &self, + _services: &RuntimeServices, + _timeout_ms: u32, + ) -> Option { + Some(UNEXPECTED_304_BACKEND.to_string()) + } + } + + #[derive(Default)] + struct RecordingTelemetrySink { + batches: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionTelemetrySink for RecordingTelemetrySink { + async fn emit_auction_events( + &self, + _services: &RuntimeServices, + batch: AuctionEventBatch, + ) -> Result<(), Report> { + self.batches + .lock() + .expect("should lock telemetry batches") + .push(batch); + Ok(()) + } + } + + fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { + let toml = format!( + "{}\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") + ); + settings_from_toml(&toml) + } + + fn settings_with_disabled_ad_templates() -> Settings { + let toml = format!( + "{}\n\ + [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") + ); + settings_from_toml(&toml) + } + + fn settings_with_disabled_auction() -> Settings { + let toml = format!( + "{}\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = false\n") + ); + settings_from_toml(&toml) + } + + fn settings_without_creative_opportunities() -> Settings { + settings_from_toml(&crate_test_settings_str()) + } + + fn settings_with_dispatching_provider() -> Settings { + let toml = format!( + "{}\n[demand]\nmodules = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n[demand.{UNEXPECTED_304_PROVIDER}]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://unexpected.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") + ); + settings_from_toml(&toml) + } + + fn services_with_telemetry( + http_client: Arc, + telemetry_sink: Arc, + ) -> RuntimeServices { + let telemetry_sink: Arc = telemetry_sink; + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .auction_telemetry_sink(telemetry_sink) + .client_info(ClientInfo::default()) + .build() + } + + fn article_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "article-slot".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/article".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + min_viewport_width: None, + max_viewport_width: None, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + compiled_unit: None, + } + } + + fn conditional_navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, ORIGIN_ETAG) + .header(header::IF_MODIFIED_SINCE, ORIGIN_LAST_MODIFIED) + .body(EdgeBody::empty()) + .expect("should build conditional navigation request") + } + + fn queue_html_response_with_cache_control( + stub: &StubHttpClient, + cache_control: &'static str, + ) { + queue_html_response_with_status_and_cache_control(stub, 200, cache_control); + } - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "matched publisher asset should receive immutable browser policy" - ); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "matched publisher asset should receive Fastly edge policy" - ); - } + fn queue_html_response_with_status_and_cache_control( + stub: &StubHttpClient, + status: u16, + cache_control: &'static str, + ) { + stub.push_response_with_headers( + status, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", cache_control), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ("cdn-cache-control", "max-age=300"), + ("cloudflare-cdn-cache-control", "max-age=300"), + ], + ); + } - #[tokio::test] - async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { - let settings = Settings::from_toml(&format!( - r#"{} + fn non_regulated_consent() -> crate::consent::ConsentContext { + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + } + } - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + async fn run_with_slots( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) + .await + } - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { mut response, .. } = response else { - panic!("should pass through non-HTML asset response"); - }; - crate::response_privacy::apply_response_headers_with_cache_privacy( - &settings, - &mut response, - ); + async fn run_with_slots_and_consent( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); + run_with_orchestrator_and_consent( + settings, + services, + &orchestrator, + slots, + req, + consent, + ) + .await + } - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=0"), - "publisher asset with Set-Cookie must become private after finalization" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "publisher asset with Set-Cookie must not retain a shared-cache header" - ); - } + async fn run_with_orchestrator( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + run_with_orchestrator_and_consent( + settings, + services, + orchestrator, + slots, + req, + non_regulated_consent(), + ) + .await + } - #[tokio::test] - async fn publisher_asset_cache_policy_skips_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} + async fn run_with_orchestrator_and_consent( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + consent: crate::consent::ConsentContext, + ) -> PublisherResponse { + let mut ec_context = EcContext::new_for_test(None, consent); - [[cache.asset_rules]] - id = "broad-publisher-path" - enabled = true - path_glob = "/news/*.html" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - fingerprint_style = "hex" - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"news".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/news/story.0123abcd.html") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + let registry = test_registry(settings); + handle_publisher_request( + AppContext { + settings, + integration_registry: ®istry, + }, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator, + slots, + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request") + } - let response = run_publisher_proxy(&settings, &services, request).await; - let response = match response { - PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { - response - } - PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { - panic!("should classify HTML response for processing") + fn response_head(response: PublisherResponse) -> http::response::Parts { + match response { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } + | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, } - }; + } + + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + fn active_ec_context() -> EcContext { + EcContext::new_for_test_with_ip( + Some(format!("{}.CkId01", "b".repeat(64))), + scheduling_consent(), + Some("203.0.113.7".to_owned()), + ) + } + + #[tokio::test] + async fn pending_origin_wait_failure_abandons_dispatched_auction_once() { + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.set_pending_streaming_responses_supported(true); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + stub.push_response(200, b"unused provider response".to_vec()); + stub.push_wait_error(); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let graph = KvIdentityGraph::new(InMemoryEcKv::new("wait-error-store")); + let mut ec_context = active_ec_context(); + let slots = [article_slot()]; - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=60"), - "asset policy must not cache publisher HTML" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "HTML response must not receive a shared-cache header" - ); - } + let error = match handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + conditional_navigation_request(), + EdgeCacheHeader::SurrogateControl, + ) + .await + { + Ok(_) => panic!("pending origin wait failure should surface as a proxy error"), + Err(error) => error, + }; - mod ssat_cache_policy_tests { - use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; - use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; - use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; - use crate::platform::{ - ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, - }; - use crate::test_support::tests::crate_test_settings_str; + assert!( + format!("{error:?}").contains("Failed to proxy request to origin"), + "should preserve the existing proxy error context: {error:?}" + ); + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + let summaries: Vec<_> = batches + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should emit exactly one terminal summary" + ); + assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); + assert_eq!( + summaries[0].terminal_reason.as_deref(), + Some("origin_proxy_error") + ); + assert_eq!( + summaries[0].publisher_domain, settings.publisher.domain, + "reordered path should retain configured publisher attribution" + ); + } - const ORIGIN_ETAG: &str = "\"origin-tag\""; - const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; - const UNEXPECTED_304_PROVIDER: &str = "example-navigation-bidder"; - const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; + #[tokio::test] + async fn pending_origin_start_failure_skips_kv_auction_and_telemetry() { + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.set_pending_streaming_responses_supported(true); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let lookups = Arc::new(AtomicUsize::new(0)); + let graph = KvIdentityGraph::new(OrderRecordingKv { + inner: InMemoryEcKv::new("start-error-store"), + http: Arc::clone(&stub), + http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), + lookups: Arc::clone(&lookups), + }); + let mut ec_context = active_ec_context(); + let slots = [article_slot()]; - struct DispatchingTestProvider; + let result = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: None, + }, + conditional_navigation_request(), + EdgeCacheHeader::SurrogateControl, + ) + .await; - struct RangeAwareHttpClient { - stub: StubHttpClient, + assert!(result.is_err(), "origin start failure should be returned"); + assert_eq!(lookups.load(Ordering::SeqCst), 0, "should skip KV lookup"); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend"], + "should not dispatch the auction provider after origin start fails" + ); + assert!( + telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches") + .is_empty(), + "should not emit abandonment for an auction that never dispatched" + ); } - impl RangeAwareHttpClient { - fn new() -> Self { - Self { - stub: StubHttpClient::new(), - } - } + #[tokio::test] + async fn signer_admission_failure_continues_publisher_origin_without_provider_io() { + let mut settings = settings_with_dispatching_provider(); + settings + .request_signing + .as_mut() + .expect("should configure request signing stores") + .enabled = true; + let plan = Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile signed navigation auction"), + ); + let orchestrator = crate::auction::build_orchestrator_with_plan(plan) + .expect("should build signed plan-backed orchestrator"); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let slots = [article_slot()]; + + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + assert_eq!(response_head(response).status, StatusCode::OK); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "signer admission failure should skip provider I/O and still fetch the publisher origin" + ); } - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RangeAwareHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - if request.request.headers().contains_key(header::RANGE) { - self.stub.push_response_with_headers( - 206, - b"partial".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("content-range", "bytes 0-18/39"), - ], - ); - } else { - self.stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - } - self.stub.send(request).await + #[tokio::test] + async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + let req = conditional_navigation_request(); + + // Act + let response = run_with_slots(&settings, &services, &slots, req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Bypass], + "eligible publisher navigation should bypass the platform cache" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + None, + "eligible publisher request should not forward If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + None, + "eligible publisher request should not forward If-Modified-Since" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "eligible HTML response should be private and non-storable" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + header::HeaderName::from_static("cdn-cache-control"), + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + ] { + assert!( + !response_head.headers.contains_key(&header_name), + "eligible HTML response should remove {header_name}" + ); } + } - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.stub.send_async(request).await - } + #[tokio::test] + async fn eligible_range_navigation_fetches_complete_html() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let http_client = Arc::new(RangeAwareHttpClient::new()); + let services = build_services_with_http_client( + Arc::clone(&http_client) as Arc + ); + let slots = [article_slot()]; + let mut req = conditional_navigation_request(); + req.headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); + req.headers_mut() + .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - self.stub.select(pending_requests).await + // Act + let response = run_with_slots(&settings, &services, &slots, req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head.status, + StatusCode::OK, + "eligible range navigation should fetch the complete origin document" + ); + let recorded_requests = http_client.stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + for header_name in [header::RANGE, header::IF_RANGE] { + assert_eq!( + recorded_header(outbound_headers, header_name.as_str()), + None, + "eligible publisher request should not forward {header_name}" + ); } } - #[async_trait::async_trait(?Send)] - impl AuctionProvider for DispatchingTestProvider { - fn provider_name(&self) -> &str { - UNEXPECTED_304_PROVIDER - } + #[tokio::test] + async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = conditional_navigation_request(); + req.headers_mut() + .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); + req.headers_mut() + .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request = PlatformHttpRequest::new( - HttpRequest::builder() - .method(Method::POST) - .uri("https://bidder.example.com/navigation-bids") - .body(EdgeBody::empty()) - .expect("should build test provider request"), - UNEXPECTED_304_BACKEND, + // Act + let response = run_with_slots(&settings, &services, &[], req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform caching for non-ad requests while readthrough is disabled", + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "publisher request without matched slots should preserve If-Modified-Since" + ); + assert_eq!( + recorded_header(outbound_headers, header::RANGE.as_str()), + Some("bytes=0-18"), + "publisher request without matched slots should preserve Range" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_RANGE.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-Range" + ); + + for (header_name, expected) in [ + (header::CACHE_CONTROL, "private, max-age=60"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "publisher response without matched slots should preserve {header_name}" ); - context - .services - .http_client() - .send_async(request) - .await - .change_context(TrustedServerError::Auction { - message: "test provider launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) } + } - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("parse_response must not run for an unexpected origin 304"); - } + fn settings_naming_the_service_version() -> Settings { + let mut settings = settings_without_creative_opportunities(); + settings.serve = + crate::middleware::PhaseEntries::new(vec![crate::middleware::PhaseEntry { + media_type: crate::middleware::HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![crate::service_version::MODULE.to_owned()], + }]); + settings + } - fn timeout_ms(&self) -> u32 { - 100 - } + /// The registry of `settings` on a platform reporting `version` for + /// its service. + fn registry_on_a_platform_reporting( + settings: &Settings, + version: &str, + ) -> IntegrationRegistry { + IntegrationRegistry::new(settings) + .expect("should create integration registry") + .with_platform(crate::attestation::PlatformIdentity { + service_version: version.to_owned(), + ..crate::attestation::PlatformIdentity::named("example-platform") + }) + } - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some(UNEXPECTED_304_BACKEND.to_string()) - } + fn service_version_header(response: &Response) -> Option { + response + .headers() + .get("x-ts-service-version") + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) } - #[derive(Default)] - struct RecordingTelemetrySink { - batches: Mutex>, + fn git_version_header(response: &Response) -> Option<&str> { + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()) } - #[async_trait::async_trait(?Send)] - impl AuctionTelemetrySink for RecordingTelemetrySink { - async fn emit_auction_events( - &self, - _services: &RuntimeServices, - batch: AuctionEventBatch, - ) -> Result<(), Report> { - self.batches - .lock() - .expect("should lock telemetry batches") - .push(batch); - Ok(()) - } + /// A page from the origin, as the publisher path hands it to a + /// finalizer, with the services it was fetched through. + async fn origin_page(settings: &Settings) -> (PublisherResponse, RuntimeServices) { + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let response = run_with_slots( + settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + (response, services) } - fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = true\n\n\ - [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + async fn buffered( + settings: &Settings, + registry: &IntegrationRegistry, + ) -> Response { + let (response, services) = origin_page(settings).await; + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), ); - Settings::from_toml(&toml) - .expect("should parse settings with auction and creative opportunities enabled") + buffer_publisher_response_async( + response, + &Method::GET, + settings, + registry, + &orchestrator, + &services, + ) + .await + .expect("should buffer the page") } - fn settings_with_disabled_ad_templates() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = true\n\n\ - [creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + #[tokio::test] + async fn the_service_version_middleware_writes_the_header_on_a_buffered_page() { + let settings = settings_naming_the_service_version(); + let registry = registry_on_a_platform_reporting(&settings, "52"); + + let response = buffered(&settings, ®istry).await; + + assert_eq!( + service_version_header(&response).as_deref(), + Some("52"), + "should write the version the platform reports on the buffered page" + ); + assert_eq!( + git_version_header(&response), + crate::constants::TS_GIT_VERSION, + "should write the git version the build compiled in on the buffered page" + ); + } + + #[tokio::test] + async fn the_service_version_middleware_writes_the_header_on_a_streamed_page() { + let settings = settings_naming_the_service_version(); + let registry = registry_on_a_platform_reporting(&settings, "52"); + let (response, services) = origin_page(&settings).await; + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + + let response = publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::new(settings), + ®istry, + orchestrator, + services, + ) + .await + .expect("should stream the page"); + + assert_eq!( + service_version_header(&response).as_deref(), + Some("52"), + "should write the version the platform reports before the body streams" ); - Settings::from_toml(&toml).expect("should parse settings with disabled ad templates") } - fn settings_with_disabled_auction() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = false\n\n\ - [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + #[tokio::test] + async fn a_page_no_serve_entry_names_the_middleware_on_carries_no_version() { + let settings = settings_without_creative_opportunities(); + let registry = registry_on_a_platform_reporting(&settings, "52"); + + let response = buffered(&settings, ®istry).await; + + assert_eq!( + service_version_header(&response), + None, + "should write nothing where no entry names the middleware" + ); + assert_eq!( + git_version_header(&response), + None, + "should write no git version where no entry names the middleware" ); - Settings::from_toml(&toml).expect("should parse settings with disabled auction") } - fn settings_without_creative_opportunities() -> Settings { - Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without creative opportunities") - } + #[tokio::test] + async fn a_platform_reporting_no_version_writes_no_header() { + let settings = settings_naming_the_service_version(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); - fn settings_with_dispatching_provider() -> Settings { - let toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.{UNEXPECTED_304_PROVIDER}]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://unexpected.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ - [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + let response = buffered(&settings, ®istry).await; + + assert_eq!( + service_version_header(&response), + None, + "should write nothing for a version the platform did not report" ); - Settings::from_toml(&toml) - .expect("should parse settings with the dispatching test provider") } - fn services_with_telemetry( - http_client: Arc, - telemetry_sink: Arc, - ) -> RuntimeServices { - let telemetry_sink: Arc = telemetry_sink; - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .auction_telemetry_sink(telemetry_sink) - .client_info(ClientInfo::default()) - .build() - } + #[tokio::test] + async fn disabled_ad_templates_use_short_browser_cache_policy() { + // Arrange + let mut settings = settings_with_disabled_ad_templates(); + settings.proxy.allowed_domains = + vec!["*.example".to_string(), "*.example.com".to_string()]; + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "public, max-age=300"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; - fn article_slot() -> CreativeOpportunitySlot { - CreativeOpportunitySlot { - id: "article-slot".to_string(), - gam_unit_path: None, - div_id: None, - page_patterns: vec!["/article".to_string()], - formats: vec![CreativeOpportunityFormat { - width: 300, - height: 250, - media_type: MediaType::Banner, - }], - floor_price: None, - targeting: Default::default(), - providers: Default::default(), - compiled_patterns: Vec::new(), - compiled_unit: None, + // Act + let response = run_with_slots( + &settings, + &services, + &slots, + conditional_navigation_request(), + ) + .await; + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let response = buffer_publisher_response_async( + response, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should buffer disabled-template response"); + let (response_head, body) = response.into_parts(); + let body = String::from_utf8( + body.into_bytes() + .expect("should return an in-memory publisher body") + .to_vec(), + ) + .expect("should return UTF-8 publisher HTML"); + + // Assert + assert_eq!( + stub.recorded_cache_intents(), + vec![PlatformCacheIntent::Default], + "should preserve platform caching with disabled ad templates and readthrough" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "disabled server-side ad templates should use the private browser cache policy" + ); + assert!( + !body.contains(".adSlots=JSON.parse"), + "disabled server-side ad templates should not inject ad-slot state" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "disabled server-side ad templates should preserve {header_name}" + ); } } - fn conditional_navigation_request() -> Request { - HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .header(header::IF_NONE_MATCH, ORIGIN_ETAG) - .header(header::IF_MODIFIED_SINCE, ORIGIN_LAST_MODIFIED) - .body(EdgeBody::empty()) - .expect("should build conditional navigation request") - } + #[tokio::test] + async fn disabled_auction_uses_private_browser_cache_policy() { + // Arrange + let settings = settings_with_disabled_auction(); + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); - fn queue_html_response_with_cache_control( - stub: &StubHttpClient, - cache_control: &'static str, - ) { - queue_html_response_with_status_and_cache_control(stub, 200, cache_control); - } + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); - fn queue_html_response_with_status_and_cache_control( - stub: &StubHttpClient, - status: u16, - cache_control: &'static str, - ) { - stub.push_response_with_headers( - status, - b"origin".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("cache-control", cache_control), - ("etag", ORIGIN_ETAG), - ("last-modified", ORIGIN_LAST_MODIFIED), - ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), - ("cdn-cache-control", "max-age=300"), - ("cloudflare-cdn-cache-control", "max-age=300"), - ], + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "disabled auction should use the private browser cache policy" ); } - fn non_regulated_consent() -> crate::consent::ConsentContext { - crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - } - } + #[tokio::test] + async fn navigation_without_matched_slots_replaces_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); - async fn run_with_slots( - settings: &Settings, - services: &RuntimeServices, - slots: &[CreativeOpportunitySlot], - req: Request, - ) -> PublisherResponse { - run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) - .await - } + for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); - async fn run_with_slots_and_consent( - settings: &Settings, - services: &RuntimeServices, - slots: &[CreativeOpportunitySlot], - req: Request, - consent: crate::consent::ConsentContext, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - run_with_orchestrator_and_consent( - settings, - services, - &orchestrator, - slots, - req, - consent, - ) - .await + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "inactive server-side ad templates should replace origin {cache_control} policy" + ); + } } - async fn run_with_orchestrator( - settings: &Settings, - services: &RuntimeServices, - orchestrator: &AuctionOrchestrator, - slots: &[CreativeOpportunitySlot], - req: Request, - ) -> PublisherResponse { - run_with_orchestrator_and_consent( - settings, - services, - orchestrator, - slots, - req, - non_regulated_consent(), - ) - .await - } + #[tokio::test] + async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); - async fn run_with_orchestrator_and_consent( - settings: &Settings, - services: &RuntimeServices, - orchestrator: &AuctionOrchestrator, - slots: &[CreativeOpportunitySlot], - req: Request, - consent: crate::consent::ConsentContext, - ) -> PublisherResponse { - let mut ec_context = EcContext::new_for_test(None, consent); + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, cache_control); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); - handle_publisher_request( - settings, - services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator, - slots, - registry: None, - }, - req, - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request") - } + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); - fn response_head(response: PublisherResponse) -> http::response::Parts { - match response { - PublisherResponse::Buffered(response) - | PublisherResponse::Stream { response, .. } - | PublisherResponse::AssembleTemplate { response, .. } - | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve private origin {cache_control} policy" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cdn-cache-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("cloudflare-cdn-cache-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "inactive server-side ad templates should preserve {header_name}" + ); + } } } - fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { - headers - .iter() - .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) - .map(|(_, value)| value.as_str()) - } + #[tokio::test] + async fn request_scoped_ad_stack_suppression_preserves_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let slots = [article_slot()]; + let mut bot_request = conditional_navigation_request(); + bot_request.headers_mut().insert( + "user-agent", + HeaderValue::from_static("Mozilla/5.0 (compatible; Googlebot/2.1)"), + ); + let mut prefetch_request = conditional_navigation_request(); + prefetch_request + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); - fn active_ec_context() -> EcContext { - EcContext::new_for_test_with_ip( - Some(format!("{}.CkId01", "b".repeat(64))), - scheduling_consent(), - Some("203.0.113.7".to_owned()), - ) + for (skip_reason, request, consent) in [ + ("bot", bot_request, non_regulated_consent()), + ("prefetch", prefetch_request, non_regulated_consent()), + ( + "consent denied", + conditional_navigation_request(), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots_and_consent(&settings, &services, &slots, request, consent) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "{skip_reason} should retain the origin cache policy" + ); + } } #[tokio::test] - async fn pending_origin_wait_failure_abandons_dispatched_auction_once() { - let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + async fn absent_creative_opportunities_use_short_browser_cache_policy() { + // Arrange + let settings = settings_without_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - stub.push_response(200, b"unused provider response".to_vec()); - stub.push_wait_error(); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc ); - let graph = KvIdentityGraph::new(InMemoryEcKv::new("wait-error-store")); - let mut ec_context = active_ec_context(); - let slots = [article_slot()]; - let error = match handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - conditional_navigation_request(), - EdgeCacheHeader::SurrogateControl, - ) - .await - { - Ok(_) => panic!("pending origin wait failure should surface as a proxy error"), - Err(error) => error, - }; + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + let response_head = response_head(response); - assert!( - format!("{error:?}").contains("Failed to proxy request to origin"), - "should preserve the existing proxy error context: {error:?}" - ); - let batches = telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches"); - let summaries: Vec<_> = batches - .iter() - .flat_map(AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!( - summaries.len(), - 1, - "should emit exactly one terminal summary" - ); - assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); - assert_eq!( - summaries[0].terminal_reason.as_deref(), - Some("origin_proxy_error") - ); + // Assert assert_eq!( - summaries[0].publisher_domain, settings.publisher.domain, - "reordered path should retain configured publisher attribution" + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "absent creative opportunities should use the private inactive-stack cache policy" ); } #[tokio::test] - async fn pending_origin_start_failure_skips_kv_auction_and_telemetry() { - let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + async fn inactive_ad_stack_preserves_non_ok_response_cache_policy() { + let settings = settings_with_disabled_ad_templates(); + + for status in [206, 404, 500, 503] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = run_with_slots( + &settings, + &services, + &[article_slot()], + conditional_navigation_request(), + ) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve origin policy on {status}" + ); + } + } + + #[tokio::test] + async fn inactive_ad_stack_preserves_the_cache_privacy_of_a_request_a_module_acts_on() { + // Arrange + let mut settings = settings_with_disabled_ad_templates(); + crate::integrations::registry_test_support::request_fixture::select_and_place( + &mut settings, + ); let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc ); - let lookups = Arc::new(AtomicUsize::new(0)); - let graph = KvIdentityGraph::new(OrderRecordingKv { - inner: InMemoryEcKv::new("start-error-store"), - http: Arc::clone(&stub), - http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), - lookups: Arc::clone(&lookups), - }); - let mut ec_context = active_ec_context(); - let slots = [article_slot()]; + let request = HttpRequest::builder() + .method(Method::GET) + .uri(format!( + "https://ts.example.com/article?{}", + crate::integrations::registry_test_support::request_fixture::QUERY + )) + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build the request the module acts on"); - let result = handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &slots, - registry: None, - }, - conditional_navigation_request(), - EdgeCacheHeader::SurrogateControl, - ) - .await; + // Act + let response = run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); - assert!(result.is_err(), "origin start failure should be returned"); - assert_eq!(lookups.load(Ordering::SeqCst), 0, "should skip KV lookup"); + // Assert assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend"], - "should not dispatch the auction provider after origin start fails" - ); - assert!( - telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches") - .is_empty(), - "should not emit abandonment for an auction that never dispatched" + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a document a module wrote into should stay private when server-side ad templates are inactive" ); } #[tokio::test] - async fn signer_admission_failure_continues_publisher_origin_without_provider_io() { - let mut settings = settings_with_dispatching_provider(); - settings - .request_signing - .as_mut() - .expect("should configure request signing stores") - .enabled = true; - let plan = Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile signed navigation auction"), - ); - let orchestrator = crate::auction::build_orchestrator_with_plan(plan, &settings) - .expect("should build signed plan-backed orchestrator"); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::new(RecordingTelemetrySink::default()), - ); - let slots = [article_slot()]; + async fn inactive_ad_stack_preserves_non_get_and_non_document_cache_policy() { + let settings = settings_with_disabled_ad_templates(); - let response = run_with_orchestrator( - &settings, - &services, - &orchestrator, - &slots, - conditional_navigation_request(), - ) - .await; + for request in [ + HttpRequest::builder() + .method(Method::POST) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build non-GET document request"), + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "empty") + .body(EdgeBody::empty()) + .expect("should build non-document request"), + ] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + queue_html_response_with_cache_control(&stub, "no-cache"); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); - assert_eq!(response_head(response).status, StatusCode::OK); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "signer admission failure should skip provider I/O and still fetch the publisher origin" - ); + // Act + let response = + run_with_slots(&settings, &services, &[article_slot()], request).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-cache"), + "inactive server-side ad templates should preserve non-document request policy" + ); + } + } + + #[tokio::test] + async fn eligible_navigation_rejects_unexpected_origin_304() { + for content_type in [None, Some("text/html; charset=utf-8")] { + // Arrange + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + + // `send_async` consumes the first response before the publisher + // origin request consumes the second response. + stub.push_response(200, b"unused provider response".to_vec()); + let mut origin_headers = vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ]; + if let Some(content_type) = content_type { + origin_headers.push(("content-type", content_type)); + } + stub.push_response_with_headers(304, Vec::new(), origin_headers); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let slots = [article_slot()]; + + // Act + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { + panic!("unexpected origin 304 should return a buffered response") + } + }; + assert_eq!( + response.status(), + StatusCode::BAD_GATEWAY, + "eligible origin 304 should fail closed with or without Content-Type" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "eligible origin 304 should return an explicitly non-storable response" + ); + assert!( + response + .extensions() + .get::() + .is_some(), + "invalid origin 304 response should remain private after late response effects" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + ] { + assert!( + !response.headers().contains_key(&header_name), + "eligible origin 304 should not forward {header_name}" + ); + } + + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + let summary_rows: Vec<_> = batches + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summary_rows.len(), + 1, + "unexpected origin 304 should emit exactly one summary row" + ); + assert_eq!( + summary_rows[0].terminal_status.as_deref(), + Some("abandoned"), + "unexpected origin 304 should abandon the dispatched auction" + ); + assert_eq!( + summary_rows[0].terminal_reason.as_deref(), + Some("unexpected_origin_304"), + "unexpected origin 304 should use the bounded telemetry reason" + ); + } } #[tokio::test] - async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { + async fn inactive_ad_stack_304_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ], + ); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); - let slots = [article_slot()]; - let req = conditional_navigation_request(); // Act - let response = run_with_slots(&settings, &services, &slots, req).await; - let response_head = response_head(response); + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } + | PublisherResponse::Stream { .. } + | PublisherResponse::AssembleTemplate { .. } => { + panic!("noneligible origin 304 should remain buffered") + } + }; + assert_eq!( + response.status(), + StatusCode::NOT_MODIFIED, + "inactive server-side ad templates should preserve the 304 status" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=60"), + "inactive server-side ad templates should apply the browser policy on revalidation" + ); + for (header_name, expected) in [ + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ] { + assert_eq!( + response + .headers() + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "inactive server-side ad templates should preserve {header_name} on revalidation" + ); + } assert_eq!( stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], - "eligible publisher navigation should bypass the platform cache" + vec![PlatformCacheIntent::Default], + "should preserve platform revalidation for non-ad requests with readthrough disabled" ); let recorded_requests = stub.recorded_request_headers(); let outbound_headers = recorded_requests @@ -16853,2551 +19588,3023 @@ mod tests { .expect("should record the outbound publisher request"); assert_eq!( recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), - None, - "eligible publisher request should not forward If-None-Match" + Some(ORIGIN_ETAG), + "noneligible publisher request should preserve If-None-Match" ); assert_eq!( recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), - None, - "eligible publisher request should not forward If-Modified-Since" - ); - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "eligible HTML response should be private and non-storable" + Some(ORIGIN_LAST_MODIFIED), + "noneligible publisher request should preserve If-Modified-Since" ); - for header_name in [ - header::ETAG, - header::LAST_MODIFIED, - header::HeaderName::from_static("surrogate-control"), - header::HeaderName::from_static("fastly-surrogate-control"), - header::HeaderName::from_static("cdn-cache-control"), - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - ] { - assert!( - !response_head.headers.contains_key(&header_name), - "eligible HTML response should remove {header_name}" + } + + #[tokio::test] + async fn inactive_ad_stack_304_preserves_private_origin_cache_policy() { + let settings = settings_with_enabled_auction_and_creative_opportunities(); + + for cache_control in ["private, max-age=0", "No-Store"] { + // Arrange + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![("cache-control", cache_control), ("etag", ORIGIN_ETAG)], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()) + .await; + let response_head = response_head(response); + + // Assert + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(cache_control), + "inactive server-side ad templates should preserve origin {cache_control} policy on revalidation" ); } } + } + + #[tokio::test] + async fn publisher_request_uses_platform_http_client_with_http_types() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, &services, req).await { + PublisherResponse::Buffered(r) => r, + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + response + } + PublisherResponse::Stream { response, .. } + | PublisherResponse::AssembleTemplate { response, .. } => response, + }; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body_string(response), "origin response"); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "should proxy through the platform http client" + ); + } + + #[tokio::test] + async fn personalized_navigation_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-page\"") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "personalized navigations must not forward {header_name}" + ); + } + } + + #[tokio::test] + async fn personalized_iframe_removes_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"frame".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/frame") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "iframe") + .header(header::IF_NONE_MATCH, "\"cached-frame\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-frame\"") + .body(EdgeBody::empty()) + .expect("should build conditional iframe request"); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for header_name in [ + header::IF_NONE_MATCH, + header::IF_MODIFIED_SINCE, + header::RANGE, + header::IF_RANGE, + ] { + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), + "personalized iframe documents must not forward {header_name}" + ); + } + } - #[tokio::test] - async fn eligible_range_navigation_fetches_complete_html() { - // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let http_client = Arc::new(RangeAwareHttpClient::new()); - let services = build_services_with_http_client( - Arc::clone(&http_client) as Arc - ); - let slots = [article_slot()]; - let mut req = conditional_navigation_request(); - req.headers_mut() - .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); - req.headers_mut() - .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); + #[tokio::test] + async fn personalized_subresource_preserves_conditional_and_range_headers() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"video".to_vec(), + vec![("content-type", "video/mp4")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/video.mp4") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "video") + .header(header::IF_NONE_MATCH, "\"cached-video\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::RANGE, "bytes=0-18") + .header(header::IF_RANGE, "\"cached-video\"") + .body(EdgeBody::empty()) + .expect("should build conditional subresource request"); + req.extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); - // Act - let response = run_with_slots(&settings, &services, &slots, req).await; - let response_head = response_head(response); + let _response = run_publisher_proxy(&settings, &services, req).await; - // Assert + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + for (header_name, expected) in [ + (header::IF_NONE_MATCH, "\"cached-video\""), + (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), + (header::RANGE, "bytes=0-18"), + (header::IF_RANGE, "\"cached-video\""), + ] { assert_eq!( - response_head.status, - StatusCode::OK, - "eligible range navigation should fetch the complete origin document" + headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) + .map(|(_, value)| value.as_str()), + Some(expected), + "personalized subresources should preserve {header_name}" ); - let recorded_requests = http_client.stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); - for header_name in [header::RANGE, header::IF_RANGE] { - assert_eq!( - recorded_header(outbound_headers, header_name.as_str()), - None, - "eligible publisher request should not forward {header_name}" - ); - } } + } - #[tokio::test] - async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { - // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut req = conditional_navigation_request(); - req.headers_mut() - .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); - req.headers_mut() - .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); + #[tokio::test] + async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); - // Act - let response = run_with_slots(&settings, &services, &[], req).await; - let response_head = response_head(response); + let _ = run_publisher_proxy(&settings, &services, req).await; - // Assert - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should preserve platform caching for non-ad requests while readthrough is disabled", - ); - let recorded_requests = stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); - assert_eq!( - recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), - Some(ORIGIN_ETAG), - "publisher request without matched slots should preserve If-None-Match" - ); - assert_eq!( - recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), - Some(ORIGIN_LAST_MODIFIED), - "publisher request without matched slots should preserve If-Modified-Since" - ); - assert_eq!( - recorded_header(outbound_headers, header::RANGE.as_str()), - Some("bytes=0-18"), - "publisher request without matched slots should preserve Range" - ); - assert_eq!( - recorded_header(outbound_headers, header::IF_RANGE.as_str()), - Some(ORIGIN_ETAG), - "publisher request without matched slots should preserve If-Range" - ); + assert_eq!( + stub.recorded_stream_response_flags(), + vec![false], + "publisher origin fetch must not request streams when the platform does not support them" + ); + } - for (header_name, expected) in [ - (header::CACHE_CONTROL, "private, max-age=60"), - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cdn-cache-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), - ] { - assert_eq!( - response_head - .headers - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "publisher response without matched slots should preserve {header_name}" - ); - } - } + #[tokio::test] + async fn publisher_origin_fetch_sets_stream_response_when_supported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![true], + "publisher origin fetch should request streams when the platform supports them" + ); + } + + #[tokio::test] + async fn handle_publisher_request_does_not_self_generate_ec() { + // EC generation is the adapter's real-browser-gated responsibility. This + // handler must never mint an EC ID on its own: for a navigation from a + // client the adapter did not pre-generate for (e.g. a non-real browser), + // `ec_value` must stay `None` so no IP-derived identifier reaches the + // auction. Consent allows EC creation and a client IP is present here — + // exactly the conditions under which the old inline call would have + // generated one. + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + assert!( + ec_context.ec_allowed(), + "test precondition: consent must allow EC creation" + ); + + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/article") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let registry = test_registry(&settings); + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); - #[tokio::test] - async fn disabled_ad_templates_use_short_browser_cache_policy() { - // Arrange - let mut settings = settings_with_disabled_ad_templates(); - settings.proxy.allowed_domains = - vec!["*.example".to_string(), "*.example.com".to_string()]; - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let slots = [article_slot()]; + assert_eq!( + ec_context.ec_value(), + None, + "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + ); + } - // Act - let response = run_with_slots( - &settings, - &services, - &slots, - conditional_navigation_request(), - ) - .await; - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let response = buffer_publisher_response_async( - response, - &Method::GET, - &settings, - ®istry, - &orchestrator, - &services, + #[tokio::test] + async fn what_a_request_filter_leaves_survives_into_the_publisher_html_pipeline() { + let mut settings = create_test_settings(); + crate::integrations::registry_test_support::request_fixture::select_and_place( + &mut settings, + ); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with the stand-in"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"content".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .header( + crate::integrations::registry_test_support::request_fixture::FILTER_HEADER, + "1", ) + .body(EdgeBody::empty()) + .expect("should build request"); + + let filter_outcome = registry + .filter_request(crate::integrations::RequestFilterRegistryInput { + settings: &settings, + services: &services, + req: &mut req, + geo_info: None, + permissions: None, + }) .await - .expect("should buffer disabled-template response"); - let (response_head, body) = response.into_parts(); - let body = String::from_utf8( - body.into_bytes() - .expect("should return an in-memory publisher body") - .to_vec(), - ) - .expect("should return UTF-8 publisher HTML"); + .expect("should run the stand-in's filter"); + assert!(matches!( + filter_outcome, + crate::integrations::RequestFilterRegistryOutcome::Continue(_) + )); + let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let response = buffer_publisher_response_async( + publisher_response, + &Method::GET, + &settings, + ®istry, + &AuctionOrchestrator::new(crate::auction::test_support::legacy_auction_config( + &settings, + )), + &services, + ) + .await + .expect("should buffer publisher response"); + let html = response_body_string(response); - // Assert - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should preserve platform caching with disabled ad templates and readthrough" - ); - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "disabled server-side ad templates should use the private browser cache policy" - ); - assert!( - !body.contains(".adSlots=JSON.parse"), - "disabled server-side ad templates should not inject ad-slot state" - ); - for (header_name, expected) in [ - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cdn-cache-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), - ] { - assert_eq!( - response_head - .headers - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "disabled server-side ad templates should preserve {header_name}" - ); - } - } + assert!( + html.contains(crate::integrations::registry_test_support::request_fixture::HEAD_FLAG), + "what the filter left on the request should reach the document" + ); + assert_eq!( + stub.recorded_backend_names().len(), + 1, + "only the publisher origin should be called" + ); + } - #[tokio::test] - async fn disabled_auction_uses_private_browser_cache_policy() { - // Arrange - let settings = settings_with_disabled_auction(); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + #[test] + fn personalized_html_is_private_and_not_shared_cached() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); - // Act - let response = run_with_slots( - &settings, - &services, - &[article_slot()], - conditional_navigation_request(), - ) - .await; - let response_head = response_head(response); + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "disabled auction should use the private browser cache policy" + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "personalized HTML should be private and non-storable" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "personalized HTML should not retain Surrogate-Control" + ); + assert!( + response.headers().get("fastly-surrogate-control").is_none(), + "personalized HTML should not retain Fastly-Surrogate-Control" + ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "personalized HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "personalized HTML should not retain CDN-Cache-Control" + ); + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "personalized HTML should not retain {header_name}" ); } - #[tokio::test] - async fn navigation_without_matched_slots_replaces_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); - - for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, cache_control); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()) - .await; - let response_head = response_head(response); + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_personalized_response_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "personalized HTML should use the exact synthesized-HTML policy" + ); + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "inactive server-side ad templates should replace origin {cache_control} policy" - ); - } - } + #[test] + fn personalized_cache_privacy_does_not_change_non_html_or_ordinary_responses() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable response"); - #[tokio::test] - async fn navigation_without_matched_slots_preserves_private_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + false, + "text/html; charset=utf-8", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "ordinary HTML should retain its existing cache policy" + ); - for cache_control in ["private, max-age=0", "No-Store"] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, cache_control); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/css", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "non-HTML should retain its existing cache policy" + ); + } - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()) - .await; - let response_head = response_head(response); + #[test] + fn a_personalized_response_marker_from_any_integration_forces_private_caching() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .header(header::ETAG, "\"origin-tag\"") + .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some(cache_control), - "inactive server-side ad templates should preserve private origin {cache_control} policy" - ); - for (header_name, expected) in [ - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cdn-cache-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("cloudflare-cdn-cache-control"), - "max-age=300", - ), - ] { - assert_eq!( - response_head - .headers - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "inactive server-side ad templates should preserve {header_name}" - ); - } - } - } + super::apply_personalized_response_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); - #[tokio::test] - async fn request_scoped_ad_stack_suppression_preserves_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let slots = [article_slot()]; - let mut bot_request = conditional_navigation_request(); - bot_request.headers_mut().insert( - "user-agent", - HeaderValue::from_static("Mozilla/5.0 (compatible; Googlebot/2.1)"), + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store, private"), + "a personalized response should be private and non-storable" + ); + for header_name in [ + "surrogate-control", + "fastly-surrogate-control", + "cloudflare-cdn-cache-control", + "cdn-cache-control", + ] { + assert!( + response.headers().get(header_name).is_none(), + "a personalized response should not retain {header_name}" ); - let mut prefetch_request = conditional_navigation_request(); - prefetch_request - .headers_mut() - .insert("sec-purpose", HeaderValue::from_static("prefetch")); - - for (skip_reason, request, consent) in [ - ("bot", bot_request, non_regulated_consent()), - ("prefetch", prefetch_request, non_regulated_consent()), - ( - "consent denied", - conditional_navigation_request(), - crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, - ..Default::default() - }, - ), - ] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - - // Act - let response = - run_with_slots_and_consent(&settings, &services, &slots, request, consent) - .await; - let response_head = response_head(response); - - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-cache"), - "{skip_reason} should retain the origin cache policy" - ); - } } - - #[tokio::test] - async fn absent_creative_opportunities_use_short_browser_cache_policy() { - // Arrange - let settings = settings_without_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + for header_name in [header::ETAG, header::LAST_MODIFIED] { + assert!( + !response.headers().contains_key(&header_name), + "a personalized response should not retain {header_name}" ); + } + } - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; - let response_head = response_head(response); + #[test] + fn the_neutral_marker_alone_marks_a_response_personalized() { + let mut personalized = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build personalized request"); + personalized + .extensions_mut() + .insert(crate::response_privacy::PersonalizedResponse); - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "absent creative opportunities should use the private inactive-stack cache policy" - ); - } + assert!( + super::request_requires_personalized_delivery(&personalized), + "the neutral marker alone should mark a response personalized" + ); - #[tokio::test] - async fn inactive_ad_stack_preserves_non_ok_response_cache_policy() { - let settings = settings_with_disabled_ad_templates(); + let ordinary = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build ordinary request"); - for status in [206, 404, 500, 503] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + assert!( + !super::request_requires_personalized_delivery(&ordinary), + "a request carrying no marker should not be personalized" + ); + } + + #[test] + fn response_carries_body_preserves_bodiless_metadata() { + // A processable GET 200 buffers a body and recomputes Content-Length. + assert!( + super::response_carries_body(&Method::GET, StatusCode::OK), + "a GET 200 publisher response should carry a buffered body" + ); + // HEAD carries no body; recomputing Content-Length to 0 would mislead + // clients/caches about the GET representation length. + assert!( + !super::response_carries_body(&Method::HEAD, StatusCode::OK), + "HEAD publisher responses must not get a recomputed Content-Length" + ); + // Bodiless statuses keep their metadata regardless of method. + assert!( + !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), + "204 responses must not get a recomputed Content-Length" + ); + assert!( + !super::response_carries_body(&Method::GET, StatusCode::RESET_CONTENT), + "205 responses must not get a recomputed Content-Length" + ); + assert!( + !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), + "304 responses must not get a recomputed Content-Length" + ); + } - // Act - let response = run_with_slots( - &settings, - &services, - &[article_slot()], - conditional_navigation_request(), - ) - .await; - let response_head = response_head(response); + #[test] + fn dispatched_auction_guard_stays_armed_until_collection_completes() { + // `take()` hands the dispatched auction to the async collector, but the + // guard must stay armed across the collection await so a drop while + // collection is still pending (a client disconnect at the await point) + // still logs the loss. Only `disarm()` — called once collection reaches + // a terminal result — clears the warning. + let mut guard = DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )); + assert!(guard.armed, "a freshly dispatched guard should be armed"); - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-cache"), - "inactive server-side ad templates should preserve origin policy on {status}" - ); - } - } + let _dispatched = guard + .take() + .expect("guard should yield the dispatched auction for collection"); + assert!( + guard.armed, + "guard must stay armed across the collection await so a drop mid-collection is reported" + ); - #[tokio::test] - async fn inactive_ad_stack_preserves_gpt_diagnostics_cache_privacy() { - // Arrange - let mut settings = settings_with_disabled_ad_templates(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable GPT diagnostics"); - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article?ts_console=1") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build GPT diagnostics request"); + guard.disarm(); + assert!( + !guard.armed, + "guard must disarm once collection reaches a terminal result" + ); + } - // Act - let response = run_with_slots(&settings, &services, &[article_slot()], request).await; - let response_head = response_head(response); + fn response_body_string(response: http::Response) -> String { + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("response body should be valid UTF-8") + } - // Assert + #[test] + fn test_content_type_detection() { + let test_cases = vec![ + ("text/html", true), + ("text/html; charset=utf-8", true), + ("Text/HTML; Charset=utf-8", true), + ("text/css", true), + ("Text/CSS", true), + ("text/javascript", true), + ("application/javascript", true), + ("Application/JavaScript", true), + ("application/json", true), + ("application/json; charset=utf-8", true), + ("Application/JSON; Charset=UTF-8", true), + ("image/jpeg", false), + ("image/png", false), + ("application/pdf", false), + ("video/mp4", false), + ("application/octet-stream", false), + ]; + + for (content_type, expected) in test_cases { assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "active GPT diagnostics should retain cache privacy when server-side ad templates are inactive" + is_processable_content_type(content_type), + expected, + "Content-Type '{content_type}' should_process: expected {expected}", ); } + } - #[tokio::test] - async fn inactive_ad_stack_preserves_non_get_and_non_document_cache_policy() { - let settings = settings_with_disabled_ad_templates(); - - for request in [ - HttpRequest::builder() - .method(Method::POST) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build non-GET document request"), - HttpRequest::builder() - .method(Method::GET) - .uri("https://ts.example.com/article") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "empty") - .body(EdgeBody::empty()) - .expect("should build non-document request"), - ] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + #[test] + fn supported_content_encoding_accepts_known_values() { + assert!(is_supported_content_encoding(""), "should accept empty"); + assert!( + is_supported_content_encoding("identity"), + "should accept identity" + ); + assert!(is_supported_content_encoding("gzip"), "should accept gzip"); + assert!( + is_supported_content_encoding("deflate"), + "should accept deflate" + ); + assert!(is_supported_content_encoding("br"), "should accept br"); + } - // Act - let response = - run_with_slots(&settings, &services, &[article_slot()], request).await; - let response_head = response_head(response); + #[test] + fn supported_content_encoding_rejects_unknown_values() { + assert!(!is_supported_content_encoding("zstd"), "should reject zstd"); + assert!( + !is_supported_content_encoding("compress"), + "should reject compress" + ); + assert!( + !is_supported_content_encoding("snappy"), + "should reject snappy" + ); + } - // Assert - assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-cache"), - "inactive server-side ad templates should preserve non-document request policy" - ); - } - } + #[tokio::test] + async fn parser_seam_loop_collects_before_writing_post_body_chunks() { + let settings = create_test_settings(); + let services = noop_services(); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let dispatched = DispatchedAuction::empty_for_test(test_auction_request(), 500); + let read_count = Arc::new(AtomicUsize::new(0)); + let body_close_processed_at = Arc::new(AtomicUsize::new(0)); + let token = b""; + let reader = ChunkedReader::new( + &[ + b"painted", + b"", + b"", + ], + Arc::clone(&read_count), + ); + let mut processor = RecordingProcessor { + read_count: Arc::clone(&read_count), + body_close_processed_at: Arc::clone(&body_close_processed_at), + }; + let ad_bids_state = AdBidsState::default(); + let ctx = AuctionCollectCtx { + dispatched, + telemetry: AuctionTelemetryCarry { + observation: None, + auction_request: None, + }, + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, + }; + let mut output = Vec::new(); - #[tokio::test] - async fn eligible_navigation_rejects_unexpected_origin_304() { - for content_type in [None, Some("text/html; charset=utf-8")] { - // Arrange - let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); + body_close_hold_loop( + reader, + &mut output, + &mut processor, + ctx, + Some(token.to_vec()), + ) + .await + .expect("should stream body with auction hold"); - // `send_async` consumes the first response before the publisher - // origin request consumes the second response. - stub.push_response(200, b"unused provider response".to_vec()); - let mut origin_headers = vec![ - ("cache-control", "public, max-age=300"), - ("etag", ORIGIN_ETAG), - ("last-modified", ORIGIN_LAST_MODIFIED), - ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), - ]; - if let Some(content_type) = content_type { - origin_headers.push(("content-type", content_type)); - } - stub.push_response_with_headers(304, Vec::new(), origin_headers); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); - let slots = [article_slot()]; + assert_eq!( + body_close_processed_at.load(Ordering::SeqCst), + 1, + "close-body tail should be processed as soon as it is found, before later chunks are read" + ); + let output = std::str::from_utf8(&output).expect("should be utf8"); + let painted = output + .find("painted") + .expect("should preserve body content"); + let bids = output + .find("var b=JSON.parse(") + .expect("should inject collected bids"); + let close = output.find("").expect("should preserve body close"); + let late = output + .find("late()") + .expect("should preserve trailing script"); + assert!( + painted < bids && bids < close && close < late, + "output order: {output}" + ); + } - // Act - let response = run_with_orchestrator( - &settings, - &services, - &orchestrator, - &slots, - conditional_navigation_request(), - ) - .await; + #[tokio::test] + async fn parser_seam_write_failure_abandons_auction_once() { + struct FailingWriter; - // Assert - let response = match response { - PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } - | PublisherResponse::Stream { .. } - | PublisherResponse::AssembleTemplate { .. } => { - panic!("unexpected origin 304 should return a buffered response") - } - }; - assert_eq!( - response.status(), - StatusCode::BAD_GATEWAY, - "eligible origin 304 should fail closed with or without Content-Type" - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "eligible origin 304 should return an explicitly non-storable response" - ); - assert!( - response - .extensions() - .get::() - .is_some(), - "invalid origin 304 response should remain private after late response effects" - ); - for header_name in [ - header::ETAG, - header::LAST_MODIFIED, - header::HeaderName::from_static("surrogate-control"), - header::HeaderName::from_static("fastly-surrogate-control"), - ] { - assert!( - !response.headers().contains_key(&header_name), - "eligible origin 304 should not forward {header_name}" - ); - } + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result { + Err(io::Error::other("injected write failure")) + } - let batches = telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches"); - let summary_rows: Vec<_> = batches - .iter() - .flat_map(AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!( - summary_rows.len(), - 1, - "unexpected origin 304 should emit exactly one summary row" - ); - assert_eq!( - summary_rows[0].terminal_status.as_deref(), - Some("abandoned"), - "unexpected origin 304 should abandon the dispatched auction" - ); - assert_eq!( - summary_rows[0].terminal_reason.as_deref(), - Some("unexpected_origin_304"), - "unexpected origin 304 should use the bounded telemetry reason" - ); + fn flush(&mut self) -> io::Result<()> { + Ok(()) } } - #[tokio::test] - async fn inactive_ad_stack_304_uses_short_browser_cache_policy() { - // Arrange - let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 304, - Vec::new(), - vec![ - ("cache-control", "public, max-age=300"), - ("etag", ORIGIN_ETAG), - ("last-modified", ORIGIN_LAST_MODIFIED), - ("surrogate-control", "max-age=300"), - ("fastly-surrogate-control", "max-age=300"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let settings = create_test_settings(); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let ad_bids_state = AdBidsState::default(); + let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let ctx = AuctionCollectCtx { + dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), + telemetry: AuctionTelemetryCarry { + observation: Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &ec_context, + )), + auction_request: None, + }, + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, + }; + let mut processor = RecordingProcessor { + read_count: Arc::new(AtomicUsize::new(0)), + body_close_processed_at: Arc::new(AtomicUsize::new(0)), + }; - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + let error = body_close_hold_loop( + std::io::Cursor::new(b"ready"), + &mut FailingWriter, + &mut processor, + ctx, + Some(b"".to_vec()), + ) + .await + .expect_err("injected writer failure should surface"); + assert!(format!("{error:?}").contains("Failed to write processed chunk")); - // Assert - let response = match response { - PublisherResponse::Buffered(response) => response, - PublisherResponse::PassThrough { .. } - | PublisherResponse::Stream { .. } - | PublisherResponse::AssembleTemplate { .. } => { - panic!("noneligible origin 304 should remain buffered") + let batches = sink.batches.lock().expect("should lock telemetry batches"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!(summaries.len(), 1, "should emit one terminal summary"); + assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); + assert_eq!( + summaries[0].terminal_reason.as_deref(), + Some("stream_write_error") + ); + } + + #[tokio::test] + async fn parser_seam_async_sink_failures_abandon_auction_once() { + struct FailingWriter { + fail_flush: bool, + } + + impl Write for FailingWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.fail_flush { + Ok(buf.len()) + } else { + Err(io::Error::other("injected write failure")) } - }; - assert_eq!( - response.status(), - StatusCode::NOT_MODIFIED, - "inactive server-side ad templates should preserve the 304 status" - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=60"), - "inactive server-side ad templates should apply the browser policy on revalidation" - ); - for (header_name, expected) in [ - (header::ETAG, ORIGIN_ETAG), - (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), - ( - header::HeaderName::from_static("surrogate-control"), - "max-age=300", - ), - ( - header::HeaderName::from_static("fastly-surrogate-control"), - "max-age=300", - ), - ] { - assert_eq!( - response - .headers() - .get(&header_name) - .and_then(|value| value.to_str().ok()), - Some(expected), - "inactive server-side ad templates should preserve {header_name} on revalidation" - ); } - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should preserve platform revalidation for non-ad requests with readthrough disabled" - ); - let recorded_requests = stub.recorded_request_headers(); - let outbound_headers = recorded_requests - .first() - .expect("should record the outbound publisher request"); - assert_eq!( - recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), - Some(ORIGIN_ETAG), - "noneligible publisher request should preserve If-None-Match" - ); - assert_eq!( - recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), - Some(ORIGIN_LAST_MODIFIED), - "noneligible publisher request should preserve If-Modified-Since" - ); + + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("injected flush failure")) + } } - #[tokio::test] - async fn inactive_ad_stack_304_preserves_private_origin_cache_policy() { - let settings = settings_with_enabled_auction_and_creative_opportunities(); + for fail_flush in [false, true] { + for with_close in [false, true] { + let settings = create_test_settings(); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let ad_bids_state = AdBidsState::default(); + let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let ctx = AuctionCollectCtx { + dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), + telemetry: AuctionTelemetryCarry { + observation: Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &ec_context, + )), + auction_request: None, + }, + deps: AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }, + }; + let mut processor = RecordingProcessor { + read_count: Arc::new(AtomicUsize::new(0)), + body_close_processed_at: Arc::new(AtomicUsize::new(0)), + }; + let html = if with_close { + "ready" + } else { + "ready" + }; + let body = + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( + html.as_bytes(), + )])); + + let error = body_close_hold_loop_stream( + body, + &mut FailingWriter { fail_flush }, + &mut processor, + Compression::None, + Compression::None, + ctx, + settings.publisher.max_buffered_body_bytes, + Some(b"".to_vec()), + ) + .await + .expect_err("should propagate the sink failure before collecting"); + let expected_error = if fail_flush { + "Failed to flush output before auction collection" + } else { + "Failed to write encoded chunk" + }; + assert!( + format!("{error:?}").contains(expected_error), + "should preserve the sink error: {error:?}" + ); - for cache_control in ["private, max-age=0", "No-Store"] { - // Arrange - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 304, - Vec::new(), - vec![("cache-control", cache_control), ("etag", ORIGIN_ETAG)], + let batches = sink.batches.lock().expect("should lock telemetry batches"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should emit one terminal summary for fail_flush={fail_flush}, with_close={with_close}" ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should abandon the uncollected auction" ); - - // Act - let response = - run_with_slots(&settings, &services, &[], conditional_navigation_request()) - .await; - let response_head = response_head(response); - - // Assert assert_eq!( - response_head - .headers - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some(cache_control), - "inactive server-side ad templates should preserve origin {cache_control} policy on revalidation" + summaries[0].terminal_reason.as_deref(), + Some("stream_write_error"), + "should classify write and flush failures consistently" ); } } } #[tokio::test] - async fn publisher_request_uses_platform_http_client_with_http_types() { + async fn hold_step_yields_ready_prefix_before_collecting_auction() { + // A small page whose `` lands in the first source chunk must + // still stream its document prefix immediately. `hold_step_decoded_chunk` + // reports the ready prefix and `close_found` without collecting; only + // `hold_collect_close_tail` awaits collection. Regression guard for the + // #849 FCP objective: the prefix must become ready while collection + // remains pending. let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + let services = noop_services(); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = match run_publisher_proxy(&settings, &services, req).await { - PublisherResponse::Buffered(r) => r, - PublisherResponse::PassThrough { mut response, body } => { - *response.body_mut() = body; - response - } - PublisherResponse::Stream { response, .. } - | PublisherResponse::AssembleTemplate { response, .. } => response, + let ad_bids_state = AdBidsState::default(); + let token = b""; + let mut state = AuctionHoldState::new( + DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + AuctionTelemetryCarry { + observation: None, + auction_request: None, + }, + Some(token.to_vec()), + ); + let collect_refs = AuctionCollectDeps { + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + request_origin: String::new(), + }; + // Passthrough processor: the ordering contract is about collection, not + // HTML rewriting, so keep the emitted bytes verbatim. + let mut processor = RecordingProcessor { + read_count: Arc::new(AtomicUsize::new(0)), + body_close_processed_at: Arc::new(AtomicUsize::new(0)), }; + let mut encoder = BodyStreamEncoder::new(Compression::None); + + let step = hold_step_decoded_chunk( + &mut processor, + &mut encoder, + b"painted", + &mut state, + &collect_refs, + ) + .await + .expect("hold step should succeed"); + + assert!( + step.close_found, + "parser marker in the first chunk must be detected" + ); + let ready: Vec = step.ready.iter().flat_map(|b| b.to_vec()).collect(); + assert_eq!( + std::str::from_utf8(&ready).expect("ready prefix should be utf8"), + "painted", + "the prefix up to must be ready before collection" + ); + assert!( + ad_bids_state + .script_cell() + .lock() + .expect("should lock bid state") + .is_none(), + "auction must not be collected while the ready prefix is emitted" + ); + + let tail = hold_collect_close_tail(&mut encoder, &mut state, &collect_refs) + .await + .expect("collect should succeed"); + let tail_bytes: Vec = tail.iter().flat_map(|b| b.to_vec()).collect(); + let tail = std::str::from_utf8(&tail_bytes).expect("held tail should be utf8"); + assert!( + tail.contains("var b=JSON.parse(") && tail.ends_with(""), + "collected bids and the held close tail must be emitted together: {tail}" + ); + assert!( + ad_bids_state + .script_cell() + .lock() + .expect("should lock bid state") + .is_some(), + "collection must run when the held tail is emitted" + ); + } + + #[test] + fn inline_body_close_seam_holds_tail_in_single_chunk() { + let token = b""; + let mut hold = InlineBodyCloseSeam::new(token.to_vec()); + + let ready = hold.push(b"painted"); + let held = hold.finish(); + + assert_eq!( + std::str::from_utf8(&ready).expect("should be utf8"), + "painted", + "content before should stream before auction collection" + ); + assert_eq!( + std::str::from_utf8(&held).expect("should be utf8"), + "", + "the close-body tag and trailing bytes should be held" + ); + } + + #[test] + fn inline_body_close_seam_holds_tail_across_chunks() { + let token = b""; + let mut hold = InlineBodyCloseSeam::new(token.to_vec()); + + let first = hold.push(b"painted"); + let held = hold.finish(); + + let streamed = [first, second].concat(); + assert_eq!( + std::str::from_utf8(&streamed).expect("should be utf8"), + "painted", + "split bytes must not leak before auction collection" + ); + assert_eq!( + std::str::from_utf8(&held).expect("should be utf8"), + "", + "split close-body tag should be held intact" + ); + } + + #[test] + fn inline_body_close_seam_matches_every_token_split_and_ignores_other_tokens() { + let token = b""; + for split in 0..=token.len() { + let mut seam = InlineBodyCloseSeam::new(token.to_vec()); + let mut ready = seam.push(b""); + ready.extend(seam.push(&token[..split])); + ready.extend(seam.push(&token[split..])); + assert!(seam.found(), "should match token split at {split}"); + assert_eq!( + ready, b"", + "should release all bytes before split {split}" + ); + assert!(seam.finish().is_empty()); + } - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body_string(response), "origin response"); + let other = b""; + let mut seam = InlineBodyCloseSeam::new(token.to_vec()); + let ready = seam.push(other); + assert!(!seam.found()); + assert_eq!([ready, seam.finish()].concat(), other); + } + + #[test] + fn unsupported_encoding_response_is_returned_unmodified() { assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should proxy through the platform http client" + classify_response_route( + StatusCode::OK, + "text/html; charset=utf-8", + "zstd", + "example.com" + ), + ResponseRoute::BufferedUnmodified, ); } - #[tokio::test] - async fn suppressed_navigation_removes_conditional_and_range_headers() { + #[test] + fn test_publisher_origin_host_extraction() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + let origin_host = settings.publisher.origin_host(); + assert_eq!(origin_host, "origin.test-publisher.com"); + + let mut settings_with_port = create_test_settings(); + settings_with_port.publisher.origin_url = "origin.test-publisher.com:8080".to_string(); + assert_eq!( + settings_with_port.publisher.origin_host(), + "origin.test-publisher.com:8080" ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "document") - .header(header::IF_NONE_MATCH, "\"cached-page\"") - .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") - .header(header::RANGE, "bytes=0-18") - .header(header::IF_RANGE, "\"cached-page\"") - .body(EdgeBody::empty()) - .expect("should build conditional request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + } - let _response = run_publisher_proxy(&settings, &services, req).await; + #[test] + fn test_invalid_utf8_handling() { + let invalid_utf8_bytes = vec![0xFF, 0xFE, 0xFD]; + assert!(String::from_utf8(invalid_utf8_bytes.clone()).is_err()); + } - let headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should record one outbound request"); - for header_name in [ - header::IF_NONE_MATCH, - header::IF_MODIFIED_SINCE, - header::RANGE, - header::IF_RANGE, - ] { - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed navigations must not forward {header_name}" + #[test] + fn test_utf8_conversion_edge_cases() { + let test_cases = vec![ + (vec![0xE2, 0x98, 0x83], true), + (vec![0xF0, 0x9F, 0x98, 0x80], true), + (vec![0xFF, 0xFE], false), + (vec![0xC0, 0x80], false), + (vec![0xED, 0xA0, 0x80], false), + ]; + + for (bytes, should_be_valid) in test_cases { + let result = String::from_utf8(bytes.clone()); + assert_eq!( + result.is_ok(), + should_be_valid, + "UTF-8 validation failed for bytes: {:?}", + bytes ); } } - #[tokio::test] - async fn suppressed_iframe_removes_conditional_and_range_headers() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"frame".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + #[test] + fn route_streams_2xx_html_without_post_processors() { + assert_eq!( + classify_response_route( + StatusCode::OK, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/frame") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "iframe") - .header(header::IF_NONE_MATCH, "\"cached-frame\"") - .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") - .header(header::RANGE, "bytes=0-18") - .header(header::IF_RANGE, "\"cached-frame\"") - .body(EdgeBody::empty()) - .expect("should build conditional iframe request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + } - let _response = run_publisher_proxy(&settings, &services, req).await; + #[test] + fn route_streams_mixed_case_html_content_type() { + assert_eq!( + classify_response_route( + StatusCode::OK, + "Text/HTML; Charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, + "HTML MIME type matching must be case-insensitive", + ); + } - let headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should record one outbound request"); - for header_name in [ - header::IF_NONE_MATCH, - header::IF_MODIFIED_SINCE, - header::RANGE, - header::IF_RANGE, - ] { - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case(header_name.as_str())), - "suppressed iframe documents must not forward {header_name}" - ); - } + #[test] + fn route_streams_html_with_post_processors() { + assert_eq!( + classify_response_route( + StatusCode::OK, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, + ); } - #[tokio::test] - async fn suppressed_subresource_preserves_conditional_and_range_headers() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"video".to_vec(), - vec![("content-type", "video/mp4")], + #[test] + fn route_streams_non_html_even_with_post_processors_registered() { + assert_eq!( + classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), + ResponseRoute::Stream, ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + } + + #[test] + fn route_buffers_unmodified_on_unsupported_encoding() { + assert_eq!( + classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), + ResponseRoute::BufferedUnmodified, ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/video.mp4") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "video") - .header(header::IF_NONE_MATCH, "\"cached-video\"") - .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") - .header(header::RANGE, "bytes=0-18") - .header(header::IF_RANGE, "\"cached-video\"") - .body(EdgeBody::empty()) - .expect("should build conditional subresource request"); - req.extensions_mut() - .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + } - let _response = run_publisher_proxy(&settings, &services, req).await; + #[test] + fn route_passes_through_non_processable_2xx() { + assert_eq!( + classify_response_route(StatusCode::OK, "image/png", "", "example.com"), + ResponseRoute::PassThrough, + ); + } - let headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should record one outbound request"); - for (header_name, expected) in [ - (header::IF_NONE_MATCH, "\"cached-video\""), - (header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT"), - (header::RANGE, "bytes=0-18"), - (header::IF_RANGE, "\"cached-video\""), - ] { - assert_eq!( - headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(header_name.as_str())) - .map(|(_, value)| value.as_str()), - Some(expected), - "suppressed subresources should preserve {header_name}" - ); - } + #[test] + fn route_buffers_non_processable_error_responses() { + assert_eq!( + classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), + ResponseRoute::BufferedUnmodified, + ); } - #[tokio::test] - async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + #[test] + fn route_excludes_204_from_pass_through() { + assert_eq!( + classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), + ResponseRoute::BufferedUnmodified, ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + } + + #[test] + fn route_excludes_205_from_pass_through() { + assert_eq!( + classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), + ResponseRoute::BufferedUnmodified, ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + } - let _ = run_publisher_proxy(&settings, &services, req).await; + #[test] + fn route_excludes_204_for_processable_content_types() { + assert_eq!( + classify_response_route( + StatusCode::NO_CONTENT, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::BufferedUnmodified, + "204 + HTML must not route to Stream", + ); + assert_eq!( + classify_response_route( + StatusCode::NO_CONTENT, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::BufferedUnmodified, + "204 + HTML + post-processors must not route to Stream", + ); + } + #[test] + fn route_excludes_205_for_processable_content_types() { assert_eq!( - stub.recorded_stream_response_flags(), - vec![false], - "publisher origin fetch must not request streams when the platform does not support them" + classify_response_route( + StatusCode::RESET_CONTENT, + "application/json", + "", + "example.com" + ), + ResponseRoute::BufferedUnmodified, + "205 + JSON must not route to Stream", ); } - #[tokio::test] - async fn publisher_origin_fetch_sets_stream_response_when_supported() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], + #[test] + fn route_streams_non_2xx_processable_content() { + assert_eq!( + classify_response_route( + StatusCode::NOT_FOUND, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + assert_eq!( + classify_response_route( + StatusCode::INTERNAL_SERVER_ERROR, + "application/json", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let _ = run_publisher_proxy(&settings, &services, req).await; + } + #[test] + fn route_streams_non_2xx_html_with_post_processors() { assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "publisher origin fetch should request streams when the platform supports them" + classify_response_route( + StatusCode::NOT_FOUND, + "text/html; charset=utf-8", + "gzip", + "example.com" + ), + ResponseRoute::Stream, ); } - #[tokio::test] - async fn handle_publisher_request_does_not_self_generate_ec() { - // EC generation is the adapter's real-browser-gated responsibility. This - // handler must never mint an EC ID on its own: for a navigation from a - // client the adapter did not pre-generate for (e.g. a non-real browser), - // `ec_value` must stay `None` so no IP-derived identifier reaches the - // auction. Consent allows EC creation and a client IP is present here — - // exactly the conditions under which the old inline call would have - // generated one. - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + #[test] + fn route_passes_through_non_processable_even_with_empty_request_host() { + assert_eq!( + classify_response_route(StatusCode::OK, "image/png", "", ""), + ResponseRoute::PassThrough, ); + } - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = - EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); - assert!( - ec_context.ec_allowed(), - "test precondition: consent must allow EC creation" + #[test] + fn route_buffers_processable_content_with_empty_request_host() { + assert_eq!( + classify_response_route(StatusCode::OK, "text/html", "gzip", ""), + ResponseRoute::BufferedUnmodified, ); + } - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/article") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build request"); + #[test] + fn pass_through_preserves_body_and_content_length() { + let image_bytes: Vec = (0..=255).cycle().take(4096).collect(); - let _ = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - req, - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should proxy publisher request"); + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "image/png") + .header(header::CONTENT_LENGTH, image_bytes.len() as u64) + .body(EdgeBody::from(image_bytes.clone())) + .expect("should build test response"); + // Simulate PassThrough: take body then reattach + let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); + // Body is unmodified — Content-Length stays correct assert_eq!( - ec_context.ec_value(), - None, - "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .expect("should have content-length"), + "4096", + "Content-Length should be preserved for pass-through" ); - } - #[tokio::test] - async fn datadome_filter_marker_survives_into_publisher_html_pipeline() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "datadome", - &serde_json::json!({ - "enabled": true, - "enable_protection": true, - "server_side_key_secret_name": "server-side-key", - "protection_excluded_ip_cidrs": ["192.0.2.0/24"], - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"content".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_secret_http_client_and_client_ip( - NoopSecretStore, - Arc::clone(&stub) as Arc, - Some("192.0.2.10".parse().expect("should parse client IP")), + // Reattach and verify body content + *response.body_mut() = body; + let (_, final_body) = response.into_parts(); + let output = final_body.into_bytes().unwrap_or_default(); + assert_eq!( + output, image_bytes, + "pass-through should preserve body byte-for-byte" ); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "document") - .body(EdgeBody::empty()) - .expect("should build request"); + } - let filter_outcome = registry - .filter_request(crate::integrations::RequestFilterRegistryInput { - settings: &settings, - services: &services, - req: &mut req, - geo_info: None, - }) - .await - .expect("should run DataDome filter"); - assert!(matches!( - filter_outcome, - crate::integrations::RequestFilterRegistryOutcome::Continue(_) - )); - let publisher_response = run_publisher_proxy(&settings, &services, req).await; - let response = buffer_publisher_response_async( - publisher_response, - &Method::GET, - &settings, - ®istry, - &AuctionOrchestrator::new(settings.auction.clone()), - &services, - ) - .await - .expect("should buffer publisher response"); - let html = response_body_string(response); + #[test] + fn test_content_encoding_detection() { + let test_encodings = vec!["gzip", "deflate", "br", "identity", ""]; - assert!(!html.contains("window.ddjskey")); - assert!(!html.contains("/integrations/datadome/tags.js")); - assert_eq!( - stub.recorded_backend_names().len(), - 1, - "only the publisher origin should be called" - ); + for encoding in test_encodings { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip, deflate, br"), + ); + + if !encoding.is_empty() { + req.headers_mut().insert( + header::CONTENT_ENCODING, + http::HeaderValue::from_str(encoding) + .expect("content encoding should be valid"), + ); + } + + let content_encoding = req + .headers() + .get(header::CONTENT_ENCODING) + .map(|h| h.to_str().unwrap_or_default()) + .unwrap_or_default(); + + assert_eq!(content_encoding, encoding); + } } #[test] - fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "datadome", - &serde_json::json!({ - "enabled": true, - "client_side_key": "test-client-key", - }), - ) - .expect("should configure DataDome integration"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create integration registry with DataDome"); - let mut params = make_stream_params(&settings, "identity"); - params.content_type = "text/html; charset=utf-8".to_string(); - params.suppress_datadome_client_side_tag = true; - let mut output = Vec::new(); + fn publisher_proxy_does_not_add_accept_encoding_when_absent() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + // No Accept-Encoding header set by the client. - stream_publisher_body( - EdgeBody::from(b"content".to_vec()), - &mut output, - ¶ms, - &settings, - ®istry, - ) - .expect("should process suppressed HTML"); + restrict_accept_encoding(&mut req); - let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); - assert!( - !html.contains("window.ddjskey"), - "publisher processing should omit the DataDome client configuration" - ); - assert!( - !html.contains("/integrations/datadome/tags.js"), - "publisher processing should omit the DataDome client tag URL" + assert_eq!( + req.headers().get(header::ACCEPT_ENCODING), + None, + "publisher proxy should not inject Accept-Encoding when the client sent none" ); } #[test] - fn suppressed_datadome_html_is_private_and_not_shared_cached() { - let mut response = Response::builder() - .status(StatusCode::OK) - .header(header::CACHE_CONTROL, "public, max-age=600") - .header("surrogate-control", "max-age=600") - .header("fastly-surrogate-control", "max-age=600") - .header("cloudflare-cdn-cache-control", "max-age=600") - .header("cdn-cache-control", "max-age=600") - .header(header::ETAG, "\"origin-tag\"") - .header(header::LAST_MODIFIED, "Wed, 21 Oct 2015 07:28:00 GMT") - .body(EdgeBody::empty()) - .expect("should build cacheable HTML response"); - - super::apply_datadome_client_tag_cache_privacy( - &mut response, - &Method::GET, - true, - "text/html; charset=utf-8", + fn publisher_proxy_limits_accept_encoding_to_supported_values() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip, deflate, br, zstd"), ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "suppressed HTML should be private and non-storable" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "suppressed HTML should not retain Surrogate-Control" - ); - assert!( - response.headers().get("fastly-surrogate-control").is_none(), - "suppressed HTML should not retain Fastly-Surrogate-Control" - ); - assert!( - response - .headers() - .get("cloudflare-cdn-cache-control") - .is_none(), - "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" - ); - assert!( - response.headers().get("cdn-cache-control").is_none(), - "suppressed HTML should not retain CDN-Cache-Control" - ); - for header_name in [header::ETAG, header::LAST_MODIFIED] { - assert!( - !response.headers().contains_key(&header_name), - "suppressed HTML should not retain {header_name}" - ); - } + restrict_accept_encoding(&mut req); - let mut no_store_response = Response::builder() - .status(StatusCode::OK) - .header(header::CACHE_CONTROL, "no-store") - .body(EdgeBody::empty()) - .expect("should build no-store HTML response"); - super::apply_datadome_client_tag_cache_privacy( - &mut no_store_response, - &Method::GET, - true, - "text/html; charset=utf-8", - ); assert_eq!( - no_store_response - .headers() - .get(header::CACHE_CONTROL) + req.headers() + .get(header::ACCEPT_ENCODING) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), - "suppressed HTML should use the exact synthesized-HTML policy" + Some("gzip, deflate, br"), + "publisher fallback should only advertise encodings the rewrite pipeline supports" ); } #[test] - fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { - let mut response = Response::builder() - .status(StatusCode::OK) - .header(header::CACHE_CONTROL, "public, max-age=600") - .header("surrogate-control", "max-age=600") - .body(EdgeBody::empty()) - .expect("should build cacheable response"); - - super::apply_datadome_client_tag_cache_privacy( - &mut response, - &Method::GET, - false, - "text/html; charset=utf-8", - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=600"), - "unsuppressed HTML should retain its existing cache policy" + fn publisher_proxy_preserves_identity_only_accept_encoding() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("identity"), ); - super::apply_datadome_client_tag_cache_privacy( - &mut response, - &Method::GET, - true, - "text/css", - ); + restrict_accept_encoding(&mut req); + assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) + req.headers() + .get(header::ACCEPT_ENCODING) .and_then(|value| value.to_str().ok()), - Some("public, max-age=600"), - "non-HTML should retain its existing cache policy" + Some("identity"), + "publisher fallback should preserve identity-only clients" ); } #[test] - fn response_carries_body_preserves_bodiless_metadata() { - // A processable GET 200 buffers a body and recomputes Content-Length. - assert!( - super::response_carries_body(&Method::GET, StatusCode::OK), - "a GET 200 publisher response should carry a buffered body" - ); - // HEAD carries no body; recomputing Content-Length to 0 would mislead - // clients/caches about the GET representation length. - assert!( - !super::response_carries_body(&Method::HEAD, StatusCode::OK), - "HEAD publisher responses must not get a recomputed Content-Length" - ); - // Bodiless statuses keep their metadata regardless of method. - assert!( - !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), - "204 responses must not get a recomputed Content-Length" - ); - assert!( - !super::response_carries_body(&Method::GET, StatusCode::RESET_CONTENT), - "205 responses must not get a recomputed Content-Length" + fn publisher_proxy_respects_supported_client_subset() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("br, gzip;q=0, zstd"), ); - assert!( - !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), - "304 responses must not get a recomputed Content-Length" + + restrict_accept_encoding(&mut req); + + assert_eq!( + req.headers() + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("br"), + "publisher fallback should only advertise the supported encodings the client accepts" ); } #[test] - fn dispatched_auction_guard_stays_armed_until_collection_completes() { - // `take()` hands the dispatched auction to the async collector, but the - // guard must stay armed across the collection await so a drop while - // collection is still pending (a client disconnect at the await point) - // still logs the loss. Only `disarm()` — called once collection reaches - // a terminal result — clears the warning. - let mut guard = DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( - test_auction_request(), - 10, - )); - assert!(guard.armed, "a freshly dispatched guard should be armed"); - - let _dispatched = guard - .take() - .expect("guard should yield the dispatched auction for collection"); - assert!( - guard.armed, - "guard must stay armed across the collection await so a drop mid-collection is reported" + fn publisher_proxy_falls_back_to_identity_for_unsupported_client_encodings() { + let mut req = build_request(Method::GET, "https://test.example.com/page"); + req.headers_mut().insert( + header::ACCEPT_ENCODING, + http::HeaderValue::from_static("zstd"), ); - guard.disarm(); - assert!( - !guard.armed, - "guard must disarm once collection reaches a terminal result" - ); - } + restrict_accept_encoding(&mut req); - fn response_body_string(response: http::Response) -> String { - String::from_utf8( - response - .into_body() - .into_bytes() - .unwrap_or_default() - .to_vec(), - ) - .expect("response body should be valid UTF-8") + assert_eq!( + req.headers() + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("identity"), + "publisher fallback should request identity when the client only accepts unsupported encodings" + ); } #[test] - fn test_content_type_detection() { - let test_cases = vec![ - ("text/html", true), - ("text/html; charset=utf-8", true), - ("Text/HTML; Charset=utf-8", true), - ("text/css", true), - ("Text/CSS", true), - ("text/javascript", true), - ("application/javascript", true), - ("Application/JavaScript", true), - ("application/json", true), - ("application/json; charset=utf-8", true), - ("Application/JSON; Charset=UTF-8", true), - ("image/jpeg", false), - ("image/png", false), - ("application/pdf", false), - ("video/mp4", false), - ("application/octet-stream", false), - ]; + fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { + let headers = |values: &[&str]| { + let mut headers = edgezero_core::http::HeaderMap::new(); + for value in values { + headers.append( + header::ACCEPT_ENCODING, + HeaderValue::from_str(value).expect("should build accept-encoding"), + ); + } + headers + }; - for (content_type, expected) in test_cases { + assert_eq!( + negotiate_reader_compression(&headers(&[])), + Ok(Compression::None) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), + Ok(Compression::Gzip) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip, br"])), + Ok(Compression::Brotli), + "server preference breaks an equal-quality tie" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=0.5"])), + Ok(Compression::None), + "implicit identity has q=1" + ); + assert_eq!( + negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), + Err(ReaderEncodingError::NoAcceptableEncoding) + ); + assert_eq!( + negotiate_reader_compression(&headers(&["gzip;q=invalid"])), + Err(ReaderEncodingError::Malformed) + ); + for malformed in [ + "gzip;q=1e-1", + "gzip;q=0.1234", + "gzip;q=1.001", + "not a coding;q=1", + ] { assert_eq!( - is_processable_content_type(content_type), - expected, - "Content-Type '{content_type}' should_process: expected {expected}", + negotiate_reader_compression(&headers(&[malformed])), + Err(ReaderEncodingError::Malformed), + "{malformed} is not valid Accept-Encoding syntax" ); } } #[test] - fn supported_content_encoding_accepts_known_values() { - assert!(is_supported_content_encoding(""), "should accept empty"); - assert!( - is_supported_content_encoding("identity"), - "should accept identity" - ); - assert!(is_supported_content_encoding("gzip"), "should accept gzip"); - assert!( - is_supported_content_encoding("deflate"), - "should accept deflate" + fn tsjs_dynamic_returns_not_found_for_unknown_filename() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + "https://publisher.example/static/tsjs=unknown.js", ); - assert!(is_supported_content_encoding("br"), "should accept br"); - } - #[test] - fn supported_content_encoding_rejects_unknown_values() { - assert!(!is_supported_content_encoding("zstd"), "should reject zstd"); - assert!( - !is_supported_content_encoding("compress"), - "should reject compress" - ); - assert!( - !is_supported_content_encoding("snappy"), - "should reject snappy" - ); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); } - #[tokio::test] - async fn parser_seam_loop_collects_before_writing_post_body_chunks() { + #[test] + fn tsjs_dynamic_serves_unified_bundle_for_known_filename() { let settings = create_test_settings(); - let services = noop_services(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let dispatched = DispatchedAuction::empty_for_test(test_auction_request(), 500); - let read_count = Arc::new(AtomicUsize::new(0)); - let body_close_processed_at = Arc::new(AtomicUsize::new(0)); - let token = b""; - let reader = ChunkedReader::new( - &[ - b"painted", - b"", - b"", - ], - Arc::clone(&read_count), + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-unified.min.js", ); - let mut processor = RecordingProcessor { - read_count: Arc::clone(&read_count), - body_close_processed_at: Arc::clone(&body_close_processed_at), - }; - let ad_bids_state = AdBidsState::default(); - let ctx = AuctionCollectCtx { - dispatched, - telemetry: AuctionTelemetryCarry { - observation: None, - auction_request: None, - }, - deps: AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }, - }; - let mut output = Vec::new(); - body_close_hold_loop( - reader, - &mut output, - &mut processor, - ctx, - Some(token.to_vec()), - ) - .await - .expect("should stream body with auction hold"); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!(response.status(), StatusCode::OK); + } - assert_eq!( - body_close_processed_at.load(Ordering::SeqCst), - 1, - "close-body tail should be processed as soon as it is found, before later chunks are read" + #[test] + fn tsjs_dynamic_serves_a_standalone_module_without_cookie_variance() { + let mut settings = create_test_settings(); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, ); - let output = std::str::from_utf8(&output).expect("should be utf8"); - let painted = output - .find("painted") - .expect("should preserve body content"); - let bids = output - .find("var b=JSON.parse(") - .expect("should inject collected bids"); - let close = output.find("").expect("should preserve body close"); - let late = output - .find("late()") - .expect("should preserve trailing script"); - assert!( - painted < bids && bids < close && close < late, - "output order: {output}" + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let mut req = build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs={}", + crate::integrations::registry_test_support::request_fixture::MODULE_FILE + ), + ); + req.headers_mut().insert( + header::COOKIE, + HeaderValue::from_static("ts-request-fixture=1"), ); - } - - #[tokio::test] - async fn parser_seam_write_failure_abandons_auction_once() { - struct FailingWriter; - - impl Write for FailingWriter { - fn write(&mut self, _buf: &[u8]) -> io::Result { - Err(io::Error::other("injected write failure")) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - let settings = create_test_settings(); - let sink = Arc::new(RecordingTelemetrySink::default()); - let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = AdBidsState::default(); - let ec_context = EcContext::new_for_test(None, ConsentContext::default()); - let ctx = AuctionCollectCtx { - dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), - telemetry: AuctionTelemetryCarry { - observation: Some(AuctionObservationContext::from_parts( - AuctionSource::InitialNavigation, - "proxy.example.com", - "/article", - 1, - None, - &ec_context, - )), - auction_request: None, - }, - deps: AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }, - }; - let mut processor = RecordingProcessor { - read_count: Arc::new(AtomicUsize::new(0)), - body_close_processed_at: Arc::new(AtomicUsize::new(0)), - }; - let error = body_close_hold_loop( - std::io::Cursor::new(b"ready"), - &mut FailingWriter, - &mut processor, - ctx, - Some(b"".to_vec()), - ) - .await - .expect_err("injected writer failure should surface"); - assert!(format!("{error:?}").contains("Failed to write processed chunk")); + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); - let batches = sink.batches.lock().expect("should lock telemetry batches"); - let summaries: Vec<_> = batches - .iter() - .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!(summaries.len(), 1, "should emit one terminal summary"); - assert_eq!(summaries[0].terminal_status.as_deref(), Some("abandoned")); - assert_eq!( - summaries[0].terminal_reason.as_deref(), - Some("stream_write_error") + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::SET_COOKIE)); + assert!( + !response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("private") || value.contains("no-store")), + "standalone module should remain cookie-independent and publicly cacheable" ); } - #[tokio::test] - async fn parser_seam_async_sink_failures_abandon_auction_once() { - struct FailingWriter { - fail_flush: bool, - } - - impl Write for FailingWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - if self.fail_flush { - Ok(buf.len()) - } else { - Err(io::Error::other("injected write failure")) - } - } - - fn flush(&mut self) -> io::Result<()> { - Err(io::Error::other("injected flush failure")) - } - } + #[test] + fn parse_single_module_filename_extracts_known_id() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); - for fail_flush in [false, true] { - for with_close in [false, true] { - let settings = create_test_settings(); - let sink = Arc::new(RecordingTelemetrySink::default()); - let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = AdBidsState::default(); - let ec_context = EcContext::new_for_test(None, ConsentContext::default()); - let ctx = AuctionCollectCtx { - dispatched: DispatchedAuction::empty_for_test(test_auction_request(), 500), - telemetry: AuctionTelemetryCarry { - observation: Some(AuctionObservationContext::from_parts( - AuctionSource::InitialNavigation, - "proxy.example.com", - "/article", - 1, - None, - &ec_context, - )), - auction_request: None, - }, - deps: AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }, - }; - let mut processor = RecordingProcessor { - read_count: Arc::new(AtomicUsize::new(0)), - body_close_processed_at: Arc::new(AtomicUsize::new(0)), - }; - let html = if with_close { - "ready" - } else { - "ready" - }; - let body = - EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( - html.as_bytes(), - )])); + assert_eq!( + parse_single_module_filename("tsjs-creative.min.js", ®istry), + Some("creative"), + "should extract the module id from a minified filename" + ); + assert_eq!( + parse_single_module_filename("tsjs-creative.js", ®istry), + Some("creative"), + "should extract the module id from an unminified filename" + ); + } - let error = body_close_hold_loop_stream( - body, - &mut FailingWriter { fail_flush }, - &mut processor, - Compression::None, - Compression::None, - ctx, - settings.publisher.max_buffered_body_bytes, - Some(b"".to_vec()), - ) - .await - .expect_err("should propagate the sink failure before collecting"); - let expected_error = if fail_flush { - "Failed to flush output before auction collection" - } else { - "Failed to write encoded chunk" - }; - assert!( - format!("{error:?}").contains(expected_error), - "should preserve the sink error: {error:?}" - ); + #[test] + fn parse_single_module_filename_resolves_a_carried_module_id() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried module"); - let batches = sink.batches.lock().expect("should lock telemetry batches"); - let summaries: Vec<_> = batches - .iter() - .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) - .filter(|row| row.event_kind == "summary") - .collect(); - assert_eq!( - summaries.len(), - 1, - "should emit one terminal summary for fail_flush={fail_flush}, with_close={with_close}" - ); - assert_eq!( - summaries[0].terminal_status.as_deref(), - Some("abandoned"), - "should abandon the uncollected auction" - ); - assert_eq!( - summaries[0].terminal_reason.as_deref(), - Some("stream_write_error"), - "should classify write and flush failures consistently" - ); - } - } + assert_eq!( + parse_single_module_filename("tsjs-probe.min.js", ®istry), + Some("probe"), + "should resolve a module id trusted-server-js has never heard of" + ); } - #[tokio::test] - async fn hold_step_yields_ready_prefix_before_collecting_auction() { - // A small page whose `` lands in the first source chunk must - // still stream its document prefix immediately. `hold_step_decoded_chunk` - // reports the ready prefix and `close_found` without collecting; only - // `hold_collect_close_tail` awaits collection. Regression guard for the - // #849 FCP objective: the prefix must become ready while collection - // remains pending. + #[test] + fn parse_single_module_filename_rejects_unknown_ids() { let settings = create_test_settings(); - let services = noop_services(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let ad_bids_state = AdBidsState::default(); - let token = b""; - let mut state = AuctionHoldState::new( - DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( - test_auction_request(), - 500, - )), - AuctionTelemetryCarry { - observation: None, - auction_request: None, - }, - Some(token.to_vec()), + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + + assert_eq!( + parse_single_module_filename("tsjs-evil.min.js", ®istry), + None, + "should reject unknown module names" ); - let collect_refs = AuctionCollectDeps { - price_granularity: PriceGranularity::default(), - ad_bids_state: &ad_bids_state, - orchestrator: &orchestrator, - services: &services, - settings: &settings, - request_origin: String::new(), - }; - // Passthrough processor: the ordering contract is about collection, not - // HTML rewriting, so keep the emitted bytes verbatim. - let mut processor = RecordingProcessor { - read_count: Arc::new(AtomicUsize::new(0)), - body_close_processed_at: Arc::new(AtomicUsize::new(0)), - }; - let mut encoder = BodyStreamEncoder::new(Compression::None); + assert_eq!( + parse_single_module_filename("tsjs-core.min.js", ®istry), + None, + "should not resolve core, which is only ever served inside the bundle" + ); + assert_eq!( + parse_single_module_filename("creative.min.js", ®istry), + None, + "should reject without tsjs- prefix" + ); + assert_eq!( + parse_single_module_filename("tsjs-creative.txt", ®istry), + None, + "should reject non-js extension" + ); + } - let step = hold_step_decoded_chunk( - &mut processor, - &mut encoder, - b"painted", - &mut state, - &collect_refs, - ) - .await - .expect("hold step should succeed"); + #[test] + fn tsjs_dynamic_serves_a_deferred_module_when_it_is_selected() { + use crate::integrations::registry_test_support::deferred_fixture; - assert!( - step.close_found, - "parser marker in the first chunk must be detected" + let mut settings = create_test_settings(); + settings.select_module("testing", deferred_fixture::MODULE); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs={}", + deferred_fixture::MODULE_FILE + ), ); - let ready: Vec = step.ready.iter().flat_map(|b| b.to_vec()).collect(); + + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); assert_eq!( - std::str::from_utf8(&ready).expect("ready prefix should be utf8"), - "painted", - "the prefix up to must be ready before collection" + response.status(), + StatusCode::OK, + "should serve the deferred module when its integration is selected" ); - assert!( - ad_bids_state - .script_cell() - .lock() - .expect("should lock bid state") - .is_none(), - "auction must not be collected while the ready prefix is emitted" + } + + #[test] + fn tsjs_dynamic_returns_not_found_for_a_module_that_is_not_named() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs={}", + crate::integrations::registry_test_support::deferred_fixture::MODULE_FILE + ), ); - let tail = hold_collect_close_tail(&mut encoder, &mut state, &collect_refs) - .await - .expect("collect should succeed"); - let tail_bytes: Vec = tail.iter().flat_map(|b| b.to_vec()).collect(); - let tail = std::str::from_utf8(&tail_bytes).expect("held tail should be utf8"); - assert!( - tail.contains("var b=JSON.parse(") && tail.ends_with(""), - "collected bids and the held close tail must be emitted together: {tail}" + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should return 404 for a deferred module that is not named" ); - assert!( - ad_bids_state - .script_cell() - .lock() - .expect("should lock bid state") - .is_some(), - "collection must run when the held tail is emitted" + } + + #[test] + fn tsjs_dynamic_returns_not_found_for_arbitrary_module_name() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); + let req = build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-evil.min.js", + ); + + let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should reject unknown module names" ); } #[test] - fn inline_body_close_seam_holds_tail_in_single_chunk() { - let token = b""; - let mut hold = InlineBodyCloseSeam::new(token.to_vec()); + fn tsjs_dynamic_uses_immutable_cache_for_matching_hash() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let module_ids = registry.js_module_ids_immediate(); + let hash = trusted_server_js::concatenated_hash(&module_ids); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + ); - let ready = hold.push(b"painted"); - let held = hold.finish(); + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SurrogateControl) + .expect("should handle tsjs request"); + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - std::str::from_utf8(&ready).expect("should be utf8"), - "painted", - "content before should stream before auction collection" + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, immutable"), + "matching content-versioned bundle should be immutable" ); assert_eq!( - std::str::from_utf8(&held).expect("should be utf8"), - "", - "the close-body tag and trailing bytes should be held" + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "matching content-versioned bundle should set the Fastly edge TTL" ); } #[test] - fn inline_body_close_seam_holds_tail_across_chunks() { - let token = b""; - let mut hold = InlineBodyCloseSeam::new(token.to_vec()); + fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let module_ids = registry.js_module_ids_immediate(); + let hash = trusted_server_js::concatenated_hash(&module_ids); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + ); - let first = hold.push(b"painted"); - let held = hold.finish(); + let response = handle_tsjs_dynamic( + &request, + ®istry, + EdgeCacheHeader::CloudflareCdnCacheControl, + ) + .expect("should handle tsjs request"); - let streamed = [first, second].concat(); assert_eq!( - std::str::from_utf8(&streamed).expect("should be utf8"), - "painted", - "split bytes must not leak before auction collection" + response + .headers() + .get("cloudflare-cdn-cache-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "Cloudflare requests should use its edge cache header" ); - assert_eq!( - std::str::from_utf8(&held).expect("should be utf8"), - "", - "split close-body tag should be held intact" + assert!( + response.headers().get("surrogate-control").is_none(), + "Cloudflare requests should not emit Fastly's edge cache header" ); } - #[test] - fn inline_body_close_seam_matches_every_token_split_and_ignores_other_tokens() { - let token = b""; - for split in 0..=token.len() { - let mut seam = InlineBodyCloseSeam::new(token.to_vec()); - let mut ready = seam.push(b""); - ready.extend(seam.push(&token[..split])); - ready.extend(seam.push(&token[split..])); - assert!(seam.found(), "should match token split at {split}"); - assert_eq!( - ready, b"", - "should release all bytes before split {split}" - ); - assert!(seam.finish().is_empty()); - } + fn body_text(response: http::Response) -> String { + let bytes = response + .into_body() + .into_bytes() + .expect("should read the tsjs response body") + .to_vec(); + String::from_utf8(bytes).expect("should serve UTF-8 JavaScript") + } - let other = b""; - let mut seam = InlineBodyCloseSeam::new(token.to_vec()); - let ready = seam.push(other); - assert!(!seam.found()); - assert_eq!([ready, seam.finish()].concat(), other); + fn cache_control_text(response: &http::Response) -> String { + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string() + } + + fn carried_deferred_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_deferred_js() + .build(), + )) } #[test] - fn unsupported_encoding_response_is_returned_unmodified() { - assert_eq!( - classify_response_route( - StatusCode::OK, - "text/html; charset=utf-8", - "zstd", - "example.com" - ), - ResponseRoute::BufferedUnmodified, + fn tsjs_dynamic_serves_a_carried_module_in_the_unified_bundle_under_the_composed_hash() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried module"); + let parts = registry.js_parts_immediate(); + let expected_hash = crate::tsjs_bundle::compose_hash(&parts); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={expected_hash}"), ); - } - #[test] - fn test_publisher_origin_host_extraction() { - let settings = create_test_settings(); - let origin_host = settings.publisher.origin_host(); - assert_eq!(origin_host, "origin.test-publisher.com"); + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); - let mut settings_with_port = create_test_settings(); - settings_with_port.publisher.origin_url = "origin.test-publisher.com:8080".to_string(); - assert_eq!( - settings_with_port.publisher.origin_host(), - "origin.test-publisher.com:8080" + assert_eq!(response.status(), StatusCode::OK, "should serve the bundle"); + assert!( + cache_control_text(&response).contains("immutable"), + "should treat the composed hash as the matching version" + ); + let body = body_text(response); + let core = trusted_server_js::module_bundle("core").expect("should have compiled core in"); + assert!( + body.starts_with(core), + "should put the compile-time core first in the bundle" + ); + assert!( + body.contains("window.__probe=1"), + "should serve the carried module inside the unified bundle" ); } #[test] - fn test_invalid_utf8_handling() { - let invalid_utf8_bytes = vec![0xFF, 0xFE, 0xFD]; - assert!(String::from_utf8(invalid_utf8_bytes.clone()).is_err()); - } - - #[test] - fn test_utf8_conversion_edge_cases() { - let test_cases = vec![ - (vec![0xE2, 0x98, 0x83], true), - (vec![0xF0, 0x9F, 0x98, 0x80], true), - (vec![0xFF, 0xFE], false), - (vec![0xC0, 0x80], false), - (vec![0xED, 0xA0, 0x80], false), - ]; + fn tsjs_dynamic_serves_a_carried_deferred_module_standalone() { + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "seam-probe", + carried_deferred_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried deferred module"); + let request = build_request( + Method::GET, + &format!("https://publisher.example/static/tsjs=tsjs-probe.min.js?v={PROBE_JS_SHA256}"), + ); - for (bytes, should_be_valid) in test_cases { - let result = String::from_utf8(bytes.clone()); - assert_eq!( - result.is_ok(), - should_be_valid, - "UTF-8 validation failed for bytes: {:?}", - bytes - ); - } - } + let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SMaxageFallback) + .expect("should handle tsjs request"); - #[test] - fn route_streams_2xx_html_without_post_processors() { assert_eq!( - classify_response_route( - StatusCode::OK, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, + response.status(), + StatusCode::OK, + "should serve a carried deferred module as its own file" ); - } - - #[test] - fn route_streams_mixed_case_html_content_type() { - assert_eq!( - classify_response_route( - StatusCode::OK, - "Text/HTML; Charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, - "HTML MIME type matching must be case-insensitive", + assert!( + cache_control_text(&response).contains("immutable"), + "should treat the carried module's own hash as the matching version" ); - } - - #[test] - fn route_streams_html_with_post_processors() { assert_eq!( - classify_response_route( - StatusCode::OK, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, + body_text(response), + PROBE_JS, + "should serve the carried source verbatim" ); } - #[test] - fn route_streams_non_html_even_with_post_processors_registered() { - assert_eq!( - classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), - ResponseRoute::Stream, - ); + fn carried_standalone_probe_registration( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder("probe") + .with_js_module(CarriedJsModule { + source: PROBE_JS, + sha256: PROBE_JS_SHA256, + }) + .with_standalone_js() + .build(), + )) } - #[test] - fn route_buffers_unmodified_on_unsupported_encoding() { - assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), - ResponseRoute::BufferedUnmodified, - ); - } + /// The settings of a deployment running the middleware fixture, with the + /// `[[fetch]]` entry given. + fn settings_running_the_middleware_fixture(fetch: Vec) -> Settings { + use crate::integrations::registry_test_support::middleware_fixture as fixture; - #[test] - fn route_passes_through_non_processable_2xx() { - assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "example.com"), - ResponseRoute::PassThrough, - ); + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(fetch); + settings } - #[test] - fn route_buffers_non_processable_error_responses() { - assert_eq!( - classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), - ResponseRoute::BufferedUnmodified, - ); + /// A text body of `content_type` streamed through the publisher path, as + /// a reader gets it. + fn text_body_through_the_publisher( + settings: &Settings, + content_type: &str, + body: &str, + ) -> String { + let registry = IntegrationRegistry::new(settings).expect("should build a registry"); + let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: content_type.to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + let mut output = Vec::new(); + stream_publisher_body( + EdgeBody::from(body.as_bytes().to_vec()), + &mut output, + ¶ms, + settings, + ®istry, + ) + .expect("should stream a text body"); + String::from_utf8(output).expect("text") } #[test] - fn route_excludes_204_from_pass_through() { - assert_eq!( - classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), - ResponseRoute::BufferedUnmodified, - ); - } + fn a_text_body_passes_through_the_middleware_of_the_entry_covering_it() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + let body = "User-agent: *\nSee https://origin.example.com/more\n"; + + let output = text_body_through_the_publisher(&settings, "text/plain; charset=utf-8", body); - #[test] - fn route_excludes_205_from_pass_through() { assert_eq!( - classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), - ResponseRoute::BufferedUnmodified, + output, + format!( + "User-agent: *\nSee https://proxy.example.com/more\n{}", + fixture::TEXT_LINE + ), + "should move the origin's address and then run the entry's middleware" ); } - #[test] - fn route_excludes_204_for_processable_content_types() { - assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::BufferedUnmodified, - "204 + HTML must not route to Stream", + /// The same through the response path a streamed body takes, which + /// builds its processor apart from the buffered path's. + #[tokio::test] + async fn a_streamed_text_body_passes_through_the_middleware_of_the_entry_covering_it() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::platform::test_support::noop_services; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), ); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: "text/plain; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"User-agent: *\nSee https://origin.example.com/"), + bytes::Bytes::from_static(b"more\n"), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should stream a text body"); + assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "text/html; charset=utf-8", - "gzip", - "example.com" + String::from_utf8(output).expect("text"), + format!( + "User-agent: *\nSee https://proxy.example.com/more\n{}", + fixture::TEXT_LINE ), - ResponseRoute::BufferedUnmodified, - "204 + HTML + post-processors must not route to Stream", + "should move the origin's address and then run the entry's middleware" ); } - #[test] - fn route_excludes_205_for_processable_content_types() { - assert_eq!( - classify_response_route( - StatusCode::RESET_CONTENT, - "application/json", - "", - "example.com" - ), - ResponseRoute::BufferedUnmodified, - "205 + JSON must not route to Stream", - ); + /// The parameters of a text response at `/a.txt`, from the origin as + /// `content_type` says, for a reader of `proxy.example.com`. + fn text_params(content_type: &str) -> OwnedProcessResponseParams { + OwnedProcessResponseParams { + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + public_prefix: crate::prefix::PublicPrefix::default(), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "identity".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/a.txt".to_owned(), + content_type: content_type.to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + } } - #[test] - fn route_streams_non_2xx_processable_content() { - assert_eq!( - classify_response_route( - StatusCode::NOT_FOUND, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, - ); - assert_eq!( - classify_response_route( - StatusCode::INTERNAL_SERVER_ERROR, - "application/json", - "gzip", - "example.com" - ), - ResponseRoute::Stream, - ); + /// Settings running the fixture with a serve entry on `text/plain` + /// naming its header-writing middleware. + fn settings_serving_the_text_header() -> Settings { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let mut settings = settings_running_the_middleware_fixture(Vec::new()); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT_HEADER.to_owned()], + }]); + settings } - #[test] - fn route_streams_non_2xx_html_with_post_processors() { - assert_eq!( - classify_response_route( - StatusCode::NOT_FOUND, - "text/html; charset=utf-8", - "gzip", - "example.com" - ), - ResponseRoute::Stream, - ); + fn named_headers(headers: &[ResponseHeader]) -> Vec<(&str, &[u8])> { + headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect() } #[test] - fn route_passes_through_non_processable_even_with_empty_request_host() { + fn a_text_body_s_serve_middleware_write_their_headers_on_the_response() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_serving_the_text_header(); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let params = text_params("text/plain; charset=utf-8"); + let mut output = Vec::new(); + + let headers = stream_publisher_body( + EdgeBody::from(b"plain\n".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should stream a text body"); + assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", ""), - ResponseRoute::PassThrough, + named_headers(&headers), + [( + fixture::TEXT_HEADER_NAME, + fixture::TEXT_HEADER_VALUE.as_bytes() + )], + "should hand back the header the serve middleware writes on a text response" ); + assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); } - #[test] - fn route_buffers_processable_content_with_empty_request_host() { + #[tokio::test] + async fn a_streamed_text_body_s_serve_middleware_write_their_headers_on_the_response() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::platform::test_support::noop_services; + + let settings = settings_serving_the_text_header(); + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let services = noop_services(); + let mut params = text_params("text/plain; charset=utf-8"); + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"pla"), + bytes::Bytes::from_static(b"in\n"), + ])); + let mut output = Vec::new(); + + let headers = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("should stream a text body"); + assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "gzip", ""), - ResponseRoute::BufferedUnmodified, + named_headers(&headers), + [( + fixture::TEXT_HEADER_NAME, + fixture::TEXT_HEADER_VALUE.as_bytes() + )], + "should hand back the header the serve middleware writes on a streamed text response" ); + assert_eq!(String::from_utf8(output).expect("text"), "plain\n"); } #[test] - fn pass_through_preserves_body_and_content_length() { - let image_bytes: Vec = (0..=255).cycle().take(4096).collect(); + fn a_text_body_no_entry_covers_is_left_as_it_was() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; - let mut response = Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "image/png") - .header(header::CONTENT_LENGTH, image_bytes.len() as u64) - .body(EdgeBody::from(image_bytes.clone())) - .expect("should build test response"); + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: Some("/elsewhere/".to_owned()), + middleware: vec![fixture::TEXT.to_owned()], + }]); - // Simulate PassThrough: take body then reattach - let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); - // Body is unmodified — Content-Length stays correct - assert_eq!( - response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .expect("should have content-length"), - "4096", - "Content-Length should be preserved for pass-through" - ); + let output = text_body_through_the_publisher(&settings, "text/plain", "plain\n"); - // Reattach and verify body content - *response.body_mut() = body; - let (_, final_body) = response.into_parts(); - let output = final_body.into_bytes().unwrap_or_default(); assert_eq!( - output, image_bytes, - "pass-through should preserve body byte-for-byte" + output, "plain\n", + "should run nothing on a path the entry does not cover" ); } - #[test] - fn test_content_encoding_detection() { - let test_encodings = vec!["gzip", "deflate", "br", "identity", ""]; - - for encoding in test_encodings { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("gzip, deflate, br"), - ); + /// What the publisher path answered one request with. + struct Proxied { + settings: Settings, + registry: IntegrationRegistry, + services: crate::platform::RuntimeServices, + orchestrator: AuctionOrchestrator, + response: PublisherResponse, + } - if !encoding.is_empty() { - req.headers_mut().insert( - header::CONTENT_ENCODING, - http::HeaderValue::from_str(encoding) - .expect("content encoding should be valid"), - ); - } + /// A request through the publisher path, the origin answering `status`. + async fn proxy(fetch: Vec, method: Method, path: &str, status: u16) -> Proxied { + proxy_with( + settings_running_the_middleware_fixture(fetch), + method, + path, + status, + ) + .await + } - let content_encoding = req - .headers() - .get(header::CONTENT_ENCODING) - .map(|h| h.to_str().unwrap_or_default()) - .unwrap_or_default(); + /// As [`proxy`], with the settings given. + async fn proxy_with(settings: Settings, method: Method, path: &str, status: u16) -> Proxied { + proxy_origin( + settings, + method, + path, + status, + b"the origin's page", + "text/html; charset=utf-8", + ) + .await + } - assert_eq!(content_encoding, encoding); + /// As [`proxy_with`], with the origin answering `status` with `body` as + /// `content_type`. + async fn proxy_origin( + settings: Settings, + method: Method, + path: &str, + status: u16, + body: &[u8], + content_type: &'static str, + ) -> Proxied { + let registry = IntegrationRegistry::new(&settings).expect("should build a registry"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + status, + body.to_vec(), + vec![("content-type", content_type)], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test_with_ip(None, consent, None); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); + let req = HttpRequest::builder() + .method(method) + .uri(format!("https://publisher.example{path}")) + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + let response = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: ®istry, + }, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy the request"); + Proxied { + settings, + registry, + services, + orchestrator, + response, } } - #[test] - fn publisher_proxy_does_not_add_accept_encoding_when_absent() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - // No Accept-Encoding header set by the client. + /// One entry naming the fixture's text middleware, for `path` or for + /// every path. + fn text_entry_for(path: Option<&str>) -> Vec { + use crate::integrations::registry_test_support::middleware_fixture as fixture; - restrict_accept_encoding(&mut req); + vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: path.map(str::to_owned), + middleware: vec![fixture::TEXT.to_owned()], + }] + } - assert_eq!( - req.headers().get(header::ACCEPT_ENCODING), - None, - "publisher proxy should not inject Accept-Encoding when the client sent none" - ); + /// The response of a proxied request, however it is to be sent. + fn response_of(response: PublisherResponse) -> Response { + match response { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::PassThrough { response, .. } => response, + PublisherResponse::AssembleTemplate { .. } => { + panic!("a template is not an answer to these requests") + } + } } - #[test] - fn publisher_proxy_limits_accept_encoding_to_supported_values() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("gzip, deflate, br, zstd"), - ); + fn content_type_of(response: &Response) -> Option<&str> { + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + } - restrict_accept_encoding(&mut req); + /// Settings refusing every crawler, with the entry that places core's + /// robots.txt middleware on the file. + fn settings_refusing_every_crawler() -> Settings { + use crate::robots_txt::middleware as robots_txt; - assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("gzip, deflate, br"), - "publisher fallback should only advertise encodings the rewrite pipeline supports" - ); + let mut settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: robots_txt::MEDIA_TYPE.to_owned(), + path: Some(crate::robots_txt::ROBOTS_TXT_PATH.to_owned()), + middleware: vec![robots_txt::MODULE.to_owned()], + }]); + let mut refusing = crate::robots_txt::RobotsTxtConfig::refuse_all(); + refusing.always_allow = vec!["/ads.txt".to_owned()]; + settings.robots_txt = Some(refusing); + settings } - #[test] - fn publisher_proxy_preserves_identity_only_accept_encoding() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("identity"), + /// The file a proxied robots.txt request streams, with its status and + /// content type checked. + async fn streamed_robots_txt(proxied: Proxied) -> String { + let PublisherResponse::Stream { + response, + body, + params, + } = proxied.response + else { + panic!("the file should stream through the entry's middleware"); + }; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should be a document of the entry's media type" ); + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the file"); + String::from_utf8(output).expect("the file is text") + } - restrict_accept_encoding(&mut req); + /// The file the middleware makes replaces the origin's own. + #[tokio::test] + async fn robots_txt_replaces_the_origins_file() { + let proxied = proxy_origin( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 200, + b"User-agent: *\nAllow: /\n", + "text/plain; charset=utf-8", + ) + .await; assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("identity"), - "publisher fallback should preserve identity-only clients" + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the refusal replaces the origin's file" ); } - #[test] - fn publisher_proxy_respects_supported_client_subset() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("br, gzip;q=0, zstd"), - ); - - restrict_accept_encoding(&mut req); + /// An origin that answers every path with its page, as a single-page + /// application's catch-all does, has no file, so the middleware makes it + /// rather than the crawler being handed the page. + #[tokio::test] + async fn robots_txt_replaces_a_page_the_origin_answers_at_its_path() { + let proxied = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 200, + ) + .await; assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("br"), - "publisher fallback should only advertise the supported encodings the client accepts" + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file replaces the page" ); } - #[test] - fn publisher_proxy_falls_back_to_identity_for_unsupported_client_encodings() { - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - http::HeaderValue::from_static("zstd"), - ); - - restrict_accept_encoding(&mut req); + /// Where the origin has no file, the creating rule makes it. + #[tokio::test] + async fn robots_txt_is_made_where_the_origin_has_none() { + let proxied = proxy_with( + settings_refusing_every_crawler(), + Method::GET, + crate::robots_txt::ROBOTS_TXT_PATH, + 404, + ) + .await; assert_eq!( - req.headers() - .get(header::ACCEPT_ENCODING) - .and_then(|value| value.to_str().ok()), - Some("identity"), - "publisher fallback should request identity when the client only accepts unsupported encodings" + streamed_robots_txt(proxied).await, + "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the file is made" ); } - #[test] - fn esi_reader_encoding_negotiation_honours_quality_identity_and_repeated_fields() { - let headers = |values: &[&str]| { - let mut headers = edgezero_core::http::HeaderMap::new(); - for value in values { - headers.append( - header::ACCEPT_ENCODING, - HeaderValue::from_str(value).expect("should build accept-encoding"), - ); - } - headers - }; + #[tokio::test] + async fn a_file_the_origin_does_not_have_is_made_by_the_middleware_named_for_its_path() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + let proxied = proxy( + text_entry_for(Some("/ads.txt")), + Method::GET, + "/ads.txt", + 404, + ) + .await; + let PublisherResponse::Stream { + response, + body, + params, + } = proxied.response + else { + panic!("a made file should stream through the entry's middleware"); + }; + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - negotiate_reader_compression(&headers(&[])), - Ok(Compression::None) + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should be a document of the entry's media type" ); + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the made file"); assert_eq!( - negotiate_reader_compression(&headers(&["gzip;q=0.8", "br;q=0.4, identity;q=0.1"])), - Ok(Compression::Gzip) + String::from_utf8(output).expect("text"), + fixture::TEXT_LINE, + "should be what the middleware writes on an empty document" ); + } + + #[tokio::test] + async fn a_head_request_for_a_made_file_gets_the_same_answer() { + let proxied = proxy( + text_entry_for(Some("/ads.txt")), + Method::HEAD, + "/ads.txt", + 404, + ) + .await; + let response = buffer_publisher_response_async( + proxied.response, + &Method::HEAD, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should buffer the HEAD answer"); + + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - negotiate_reader_compression(&headers(&["gzip, br"])), - Ok(Compression::Brotli), - "server preference breaks an equal-quality tie" + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should answer a HEAD as the GET is answered" ); + assert!( + response + .into_body() + .into_bytes() + .expect("should hold the HEAD answer in memory") + .is_empty(), + "should answer a HEAD with no body" + ); + } + + #[tokio::test] + async fn an_html_entry_with_a_path_makes_no_missing_page() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let proxied = proxy( + vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/news/".to_owned()), + middleware: vec![fixture::HEAD.to_owned()], + }], + Method::GET, + "/news/missing", + 404, + ) + .await; + assert_eq!( - negotiate_reader_compression(&headers(&["gzip;q=0.5"])), - Ok(Compression::None), - "implicit identity has q=1" + response_of(proxied.response).status(), + StatusCode::NOT_FOUND, + "should keep the origin's answer for a page it does not have" ); + } + + #[tokio::test] + async fn an_html_entry_ahead_does_not_shadow_the_entry_for_a_missing_file() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let mut fetch = vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: Some("/".to_owned()), + middleware: vec![fixture::HEAD.to_owned()], + }]; + fetch.extend(text_entry_for(Some("/ads.txt"))); + let proxied = proxy(fetch, Method::GET, "/ads.txt", 404).await; + let response = response_of(proxied.response); + + assert_eq!(response.status(), StatusCode::OK); assert_eq!( - negotiate_reader_compression(&headers(&["zstd, identity;q=0"])), - Err(ReaderEncodingError::NoAcceptableEncoding) + content_type_of(&response), + Some("text/plain; charset=utf-8"), + "should make the file from the text/plain entry, not as HTML" ); + } + + #[tokio::test] + async fn an_entry_covering_every_path_makes_no_missing_page() { + let proxied = proxy(text_entry_for(None), Method::GET, "/missing.txt", 404).await; + assert_eq!( - negotiate_reader_compression(&headers(&["gzip;q=invalid"])), - Err(ReaderEncodingError::Malformed) + response_of(proxied.response).status(), + StatusCode::NOT_FOUND, + "should keep the origin's answer, because a page the origin does not have is its answer" ); - for malformed in [ - "gzip;q=1e-1", - "gzip;q=0.1234", - "gzip;q=1.001", - "not a coding;q=1", - ] { - assert_eq!( - negotiate_reader_compression(&headers(&[malformed])), - Err(ReaderEncodingError::Malformed), - "{malformed} is not valid Accept-Encoding syntax" - ); - } } - #[test] - fn tsjs_dynamic_returns_not_found_for_unknown_filename() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let req = build_request( + #[tokio::test] + async fn a_missing_file_outside_the_entry_path_keeps_the_origin_answer() { + let proxied = proxy( + text_entry_for(Some("/ads.txt")), Method::GET, - "https://publisher.example/static/tsjs=unknown.js", - ); + "/robots.txt", + 404, + ) + .await; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); - assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response_of(proxied.response).status(), + StatusCode::NOT_FOUND + ); } - #[test] - fn tsjs_dynamic_serves_unified_bundle_for_known_filename() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let req = build_request( + #[tokio::test] + async fn an_origin_error_on_the_entry_path_keeps_the_origin_answer() { + let proxied = proxy( + text_entry_for(Some("/ads.txt")), Method::GET, - "https://publisher.example/static/tsjs=tsjs-unified.min.js", - ); + "/ads.txt", + 500, + ) + .await; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); - assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_of(proxied.response).status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should make a file only where the origin has none" + ); } - #[test] - fn tsjs_dynamic_serves_diagnostics_standalone_without_cookie_variance() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let mut req = build_request( + #[tokio::test] + async fn what_a_middleware_prepares_reaches_its_create() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let proxied = proxy( + vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::PREPARING.to_owned()], + }], Method::GET, - "https://publisher.example/static/tsjs=tsjs-gpt_diagnostics.min.js", + "/article", + 200, + ) + .await; + let PublisherResponse::Stream { body, params, .. } = proxied.response else { + panic!("a page should stream through the entry's middleware"); + }; + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the page"); + let page = String::from_utf8(output).expect("html"); + + assert!( + page.contains(&fixture::prepared_marker( + fixture::PREPARING, + "publisher.example" + )), + "should write what the preparation answered, called as the middleware for the \ + reader's host: {page}" ); - req.headers_mut().insert( - header::COOKIE, - HeaderValue::from_static("__Host-ts-console=1"), + assert!( + !page.contains(fixture::NOTHING_PREPARED), + "should not ask create before the preparation" ); + } - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); + #[tokio::test] + async fn a_serve_middleware_prepares_for_each_reader() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let mut settings = settings_running_the_middleware_fixture(Vec::new()); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::PREPARING.to_owned()], + }]); + let proxied = proxy_with(settings, Method::GET, "/article", 200).await; + let PublisherResponse::Stream { body, params, .. } = proxied.response else { + panic!("a page should stream through the entry's middleware"); + }; + let mut params = *params; + let mut output = Vec::new(); + stream_publisher_body_async( + body, + &mut output, + &mut params, + &proxied.settings, + &proxied.registry, + &proxied.orchestrator, + &proxied.services, + ) + .await + .expect("should stream the page"); + let page = String::from_utf8(output).expect("html"); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::SET_COOKIE)); assert!( - !response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.contains("private") || value.contains("no-store")), - "standalone module should remain cookie-independent and publicly cacheable" + page.contains(&fixture::prepared_marker( + fixture::PREPARING, + "publisher.example" + )), + "should prepare in the serve phase for this reader: {page}" ); } - #[test] - fn parse_single_module_filename_extracts_known_id() { + #[tokio::test] + async fn a_refusal_answers_the_request_with_its_status_and_stores_nothing() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::HTML_MEDIA_TYPE; + + let proxied = proxy( + vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::REFUSING.to_owned()], + }], + Method::GET, + "/article", + 200, + ) + .await; + let PublisherResponse::Buffered(response) = proxied.response else { + panic!("a refusal is answered whole, with no document to stream"); + }; + let header_value = |name: header::HeaderName| { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(header_value(header::RETRY_AFTER).as_deref(), Some("30")); assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.min.js"), - Some("sourcepoint"), - "should extract sourcepoint from minified filename" + header_value(header::CONTENT_TYPE).as_deref(), + Some("text/plain; charset=utf-8") ); assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.js"), - Some("sourcepoint"), - "should extract sourcepoint from unminified filename" + header_value(header::CACHE_CONTROL).as_deref(), + Some("no-store, private"), + "should be stored and shared by nothing" + ); + assert!( + response + .extensions() + .get::() + .is_some(), + "should be marked as this request's alone" ); + let body = response.into_body().into_bytes().expect("a whole body"); + assert_eq!(String::from_utf8_lossy(&body), fixture::REFUSAL_MESSAGE); } #[test] - fn parse_single_module_filename_rejects_unknown_ids() { - assert_eq!( - parse_single_module_filename("tsjs-evil.min.js"), - None, - "should reject unknown module names" - ); - assert_eq!( - parse_single_module_filename("tsjs-core.min.js"), - Some("core"), - "should accept any known module ID (deferred check happens in caller)" - ); - assert_eq!( - parse_single_module_filename("prebid.min.js"), - None, - "should reject without tsjs- prefix" - ); + fn a_text_body_in_a_charset_other_than_utf8_is_left_as_it_was() { + use crate::integrations::registry_test_support::middleware_fixture as fixture; + + let settings = settings_running_the_middleware_fixture(vec![PhaseEntry { + media_type: fixture::TEXT_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::TEXT.to_owned()], + }]); + + let output = + text_body_through_the_publisher(&settings, "text/plain; charset=utf-16", "plain\n"); + assert_eq!( - parse_single_module_filename("tsjs-sourcepoint.txt"), - None, - "should reject non-js extension" + output, "plain\n", + "should hand a middleware no body it would corrupt by writing UTF-8" ); } #[test] - fn tsjs_dynamic_serves_prebid_shim_when_enabled() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-prebid.min.js", - ); + fn a_refusal_rounds_retry_after_up_to_whole_seconds() { + let response = refusal_response(&Refusal::service_unavailable( + std::time::Duration::from_millis(1_500), + "later", + )) + .expect("should build the refusal"); - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); assert_eq!( - response.status(), - StatusCode::OK, - "should serve the deferred prebid shim module when prebid is enabled" + response + .headers() + .get(header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()), + Some("2"), + "should not ask the caller back before the refusal's time" ); } #[test] - fn tsjs_dynamic_returns_not_found_for_disabled_deferred_module() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &serde_json::json!({ - "enabled": false, - "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", - }), - ) - .expect("should update prebid config"); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), + fn a_fetch_preparation_of_a_document_that_may_be_stored_is_not_handed_the_reader() { + use crate::module_context::{EdgeCookie, FromModuleContext}; + + let ec_context = EcContext::new_for_test( + Some(crate::ec::tests::CANONICAL_COOKIE_VALUE.to_owned()), + ConsentContext::default(), ) - .expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-prebid.min.js", - ); + .with_module_for_test(Arc::new(crate::ec::tests::CanonicalizingModule)); + let services = noop_services(); + let settings = settings_running_the_middleware_fixture(Vec::new()); + let method = Method::GET; + let handed = |stored: bool| { + let request = ModuleRequest::new(&method, "publisher.example", "https", "/article"); + let context = preparation_context(request, stored, &settings, &ec_context, &services); + let call = context.call("testing.preparation", PermissionSet::none()); + ( + <&PermissionState>::from_module_context(&call).is_ok(), + EdgeCookie::from_module_context(&call).is_ok(), + ) + }; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); assert_eq!( - response.status(), - StatusCode::NOT_FOUND, - "should return 404 for disabled deferred module" + handed(false), + (true, true), + "should hand a document that is not stored the reader's permissions and Edge Cookie" + ); + assert_eq!( + handed(true), + (false, false), + "should hand a document that may be stored neither the reader's permissions nor Edge Cookie" ); } #[test] - fn tsjs_dynamic_returns_not_found_for_arbitrary_module_name() { - let settings = create_test_settings(); - let registry = IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), + fn the_serve_phase_of_a_document_that_may_be_stored_is_prepared_for_the_reader() { + use crate::module_context::{EdgeCookie, FromModuleContext}; + + let ec_context = EcContext::new_for_test( + Some(crate::ec::tests::CANONICAL_COOKIE_VALUE.to_owned()), + ConsentContext::default(), ) - .expect("should create integration registry"); - let req = build_request( - Method::GET, - "https://publisher.example/static/tsjs=tsjs-evil.min.js", - ); + .with_module_for_test(Arc::new(crate::ec::tests::CanonicalizingModule)); + let services = noop_services(); + let settings = settings_running_the_middleware_fixture(Vec::new()); + let method = Method::GET; + let handed = |context: &ModuleContext<'_>| { + let call = context.call("testing.preparation", PermissionSet::none()); + ( + <&PermissionState>::from_module_context(&call).is_ok(), + EdgeCookie::from_module_context(&call).is_ok(), + ) + }; - let response = handle_tsjs_dynamic(&req, ®istry, EdgeCacheHeader::SMaxageFallback) - .expect("should handle tsjs request"); - assert_eq!( - response.status(), - StatusCode::NOT_FOUND, - "should reject unknown module names" - ); + for stored in [true, false] { + let (fetch, serve) = preparation_contexts( + || ModuleRequest::new(&method, "publisher.example", "https", "/article"), + stored, + &settings, + &ec_context, + &services, + ); + + assert_eq!( + handed(&fetch), + (!stored, !stored), + "the fetch phase should see the reader only when the document is not stored" + ); + assert_eq!( + handed(&serve), + (true, true), + "the serve phase should see the reader whether or not the document is stored" + ); + } } #[test] - fn tsjs_dynamic_uses_immutable_cache_for_matching_hash() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let module_ids = registry.js_module_ids_immediate(); - let hash = trusted_server_js::concatenated_hash(&module_ids); - let request = build_request( - Method::GET, - &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + fn tsjs_dynamic_serves_a_standalone_module_by_registration_flag_not_by_name() { + // The module the registration marks standalone is one a vendor crate + // carries, so nothing in the binary knows its name. A branch that named + // an integration instead of reading the flag could not serve it, and a + // reintroduced constant would fail this test rather than pass it. + let mut settings = create_test_settings(); + settings.select_module("testing", "testing.probe"); + let extra = [IntegrationBuilder::new( + "probe", + "standalone-probe", + carried_standalone_probe_registration, + validate_nothing, + ) + .with_module_name("testing.probe")]; + let registry = IntegrationRegistry::with_registrations(&settings, &extra) + .expect("should build a registry with a carried standalone module"); + assert!( + registry.js_module_ids_immediate().contains(&"creative"), + "fixture should put core's creative module in the unified bundle" + ); + assert!( + !registry.js_module_ids().contains(&"probe"), + "a standalone module should stay out of the bundled module ids" ); - let response = handle_tsjs_dynamic(&request, ®istry, EdgeCacheHeader::SurrogateControl) - .expect("should handle tsjs request"); - - assert_eq!(response.status(), StatusCode::OK); + let standalone = handle_tsjs_dynamic( + &build_request( + Method::GET, + &format!( + "https://publisher.example/static/tsjs=tsjs-probe.min.js?v={PROBE_JS_SHA256}" + ), + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "matching content-versioned bundle should be immutable" + standalone.status(), + StatusCode::OK, + "should serve a module its registration marks standalone" + ); + assert!( + cache_control_text(&standalone).contains("immutable"), + "should treat the carried module's own hash as the matching version" ); assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "matching content-versioned bundle should set the Fastly edge TTL" + body_text(standalone), + PROBE_JS, + "should serve the standalone module as its own file" ); - } - #[test] - fn tsjs_dynamic_uses_cloudflare_edge_header_when_selected() { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let module_ids = registry.js_module_ids_immediate(); - let hash = trusted_server_js::concatenated_hash(&module_ids); - let request = build_request( - Method::GET, - &format!("https://publisher.example/static/tsjs=tsjs-unified.min.js?v={hash}"), + let unified = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-unified.min.js", + ), + ®istry, + EdgeCacheHeader::SMaxageFallback, + ) + .expect("should handle tsjs request"); + assert!( + !body_text(unified).contains(PROBE_JS), + "a standalone module should stay out of the unified bundle" ); - let response = handle_tsjs_dynamic( - &request, + let bundled_only = handle_tsjs_dynamic( + &build_request( + Method::GET, + "https://publisher.example/static/tsjs=tsjs-lockr.min.js", + ), ®istry, - EdgeCacheHeader::CloudflareCdnCacheControl, + EdgeCacheHeader::SMaxageFallback, ) .expect("should handle tsjs request"); - assert_eq!( - response - .headers() - .get("cloudflare-cdn-cache-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "Cloudflare requests should use its edge cache header" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "Cloudflare requests should not emit Fastly's edge cache header" + bundled_only.status(), + StatusCode::NOT_FOUND, + "should not serve a bundle-only module as a single file" ); } @@ -19493,49 +22700,346 @@ mod tests { encoder.finish().expect("should finish compression"); } - let body = EdgeBody::from(compressed); + let body = EdgeBody::from(compressed); + let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: "gzip".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/".to_owned(), + content_type: "text/css".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }; + + let mut output = Vec::new(); + stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) + .expect("should process gzip CSS"); + + // Decompress output + use flate2::read::GzDecoder; + use std::io::Read; + let mut decoder = GzDecoder::new(&output[..]); + let mut decompressed = String::new(); + decoder + .read_to_string(&mut decompressed) + .expect("should decompress output"); + + assert!( + decompressed.contains("proxy.example.com"), + "should rewrite origin to proxy. Got: {decompressed}" + ); + assert!( + !decompressed.contains("origin.example.com"), + "should not contain original host. Got: {decompressed}" + ); + } + + /// A selector is chosen for each document, so one that does not parse + /// cannot be refused when the settings load. The response is refused + /// instead of the parser panicking. + #[test] + fn a_middleware_whose_selector_does_not_parse_refuses_the_response() { + use crate::error::IntoHttpResponse as _; + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::{HTML_MEDIA_TYPE, PhaseEntries, PhaseEntry}; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::BROKEN.to_owned()], + }]); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), - content_encoding: "gzip".to_string(), + content_encoding: String::new(), origin_host: "origin.example.com".to_string(), origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), - content_type: "text/css".to_string(), + request_path: "/".to_owned(), + content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); - stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) - .expect("should process gzip CSS"); - - // Decompress output - use flate2::read::GzDecoder; - use std::io::Read; - let mut decoder = GzDecoder::new(&output[..]); - let mut decompressed = String::new(); - decoder - .read_to_string(&mut decompressed) - .expect("should decompress output"); + let error = stream_publisher_body( + EdgeBody::from(b"x".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect_err("should refuse a page whose middleware cannot run"); + let message = error.to_string(); assert!( - decompressed.contains("proxy.example.com"), - "should rewrite origin to proxy. Got: {decompressed}" + message.contains(&format!( + "the fetch middleware `{}` cannot run", + fixture::BROKEN + )) && message.contains(&format!( + "`{}` is not a usable selector", + fixture::BROKEN_SELECTOR + )), + "should name the middleware and the selector: {message}" ); - assert!( - !decompressed.contains("origin.example.com"), - "should not contain original host. Got: {decompressed}" + assert_eq!( + error.current_context().status_code(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer as a fault of the deployment's own" ); + assert!(output.is_empty(), "should write nothing of the page"); + } + + /// The same refusal through each finalizer, so an adapter answers with an + /// error where it would otherwise send the headers of a page it cannot + /// deliver. + #[tokio::test] + async fn a_middleware_whose_selector_does_not_parse_fails_the_response_through_each_finalizer() + { + use crate::error::IntoHttpResponse as _; + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::{HTML_MEDIA_TYPE, PhaseEntries, PhaseEntry}; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.fetch = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::BROKEN.to_owned()], + }]); + let settings = Arc::new(settings); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let services = noop_services(); + let page = || PublisherResponse::Stream { + response: Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(EdgeBody::empty()) + .expect("should build the response"), + body: EdgeBody::from( + b"x".to_vec(), + ), + params: Box::new(OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/".to_owned(), + content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }), + }; + + let streamed = publisher_response_into_streaming_response( + page(), + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await; + let buffered = buffer_publisher_response_async( + page(), + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await; + + for (finalizer, result) in [("streaming", streamed), ("buffered", buffered)] { + let error = result + .err() + .unwrap_or_else(|| panic!("should fail the response in the {finalizer} finalizer")); + assert!( + error.to_string().contains(&format!( + "the fetch middleware `{}` cannot run", + fixture::BROKEN + )), + "should name the middleware in the {finalizer} finalizer: {error}" + ); + assert_eq!( + error.current_context().status_code(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer as a fault of the deployment's own in the {finalizer} finalizer" + ); + } + } + + /// A serve entry that cannot run fails the response at each place a page + /// leaves from, being a page that is not stored and a page from the + /// store, through each finalizer. + #[tokio::test] + async fn a_serve_middleware_whose_selector_does_not_parse_fails_the_response_wherever_it_leaves() + { + use crate::error::IntoHttpResponse as _; + use crate::integrations::registry_test_support::middleware_fixture as fixture; + use crate::middleware::{HTML_MEDIA_TYPE, PhaseEntries, PhaseEntry}; + + let mut settings = create_test_settings(); + settings.select_module("testing", fixture::MODULE); + settings.serve = PhaseEntries::new(vec![PhaseEntry { + media_type: HTML_MEDIA_TYPE.to_owned(), + path: None, + middleware: vec![fixture::BROKEN_READER.to_owned()], + }]); + let settings = Arc::new(settings); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); + let services = noop_services(); + let response = || { + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(EdgeBody::empty()) + .expect("should build the response") + }; + let params = || { + Box::new(OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), + csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + request_path: "/".to_owned(), + content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + request_state: IntegrationRequestState::default(), + }) + }; + let from_the_origin = || PublisherResponse::Stream { + response: response(), + body: EdgeBody::from( + b"x".to_vec(), + ), + params: params(), + }; + let from_the_store = || PublisherResponse::AssembleTemplate { + response: response(), + template: format!( + "x{AD_ASSEMBLY_SEAM}" + ) + .into_bytes(), + params: params(), + }; + + let mut results = Vec::new(); + let pages: [(&str, &dyn Fn() -> PublisherResponse); 2] = [ + ("from the origin", &from_the_origin), + ("from the store", &from_the_store), + ]; + for (page, publisher_response) in pages { + results.push(( + page, + "streaming", + publisher_response_into_streaming_response( + publisher_response(), + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await, + )); + results.push(( + page, + "buffered", + buffer_publisher_response_async( + publisher_response(), + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await, + )); + } + + for (page, finalizer, result) in results { + let error = result.err().unwrap_or_else(|| { + panic!("should fail a page {page} in the {finalizer} finalizer") + }); + assert!( + error.to_string().contains(&format!( + "the serve middleware `{}` cannot run", + fixture::BROKEN_READER + )), + "should name the middleware for a page {page} in the {finalizer} finalizer: \ + {error}" + ); + assert_eq!( + error.current_context().status_code(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer as a fault of the deployment's own for a page {page} in the \ + {finalizer} finalizer" + ); + } } /// Empty origin body on the streaming route must produce no output @@ -19554,7 +23058,9 @@ mod tests { .expect("should create integration registry"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19563,15 +23069,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -19602,7 +23109,9 @@ mod tests { ) .expect("should create integration registry"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19611,15 +23120,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -19635,8 +23145,8 @@ mod tests { ); } - #[test] - fn body_chunk_source_yields_once_body_in_chunks() { + #[tokio::test] + async fn body_chunk_source_yields_once_body_in_chunks() { futures::executor::block_on(async { let body = EdgeBody::from_bytes(bytes::Bytes::from_static(b"abcdef")); let mut source = BodyChunkSource::new(body, 3).with_max_bytes(16); @@ -19658,8 +23168,8 @@ mod tests { }); } - #[test] - fn body_chunk_source_preserves_stream_chunks() { + #[tokio::test] + async fn body_chunk_source_preserves_stream_chunks() { futures::executor::block_on(async { let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"first"), @@ -19684,8 +23194,8 @@ mod tests { }); } - #[test] - fn body_chunk_source_enforces_cumulative_raw_cap() { + #[tokio::test] + async fn body_chunk_source_enforces_cumulative_raw_cap() { futures::executor::block_on(async { let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"1234"), @@ -19713,8 +23223,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19725,10 +23235,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19737,15 +23251,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -19777,8 +23292,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_gzip_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_gzip_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19789,10 +23304,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19801,15 +23320,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -19844,8 +23364,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_deflate_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_deflate_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19856,10 +23376,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19868,15 +23392,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -19911,8 +23436,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_brotli_stream_without_auction() { + #[tokio::test] + async fn stream_publisher_body_async_processes_brotli_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19923,10 +23448,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -19935,15 +23464,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -19978,8 +23508,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_rejects_truncated_brotli_stream() { + #[tokio::test] + async fn stream_publisher_body_async_rejects_truncated_brotli_stream() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -19990,10 +23520,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20002,15 +23536,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -20042,7 +23577,9 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20051,20 +23588,21 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } - #[test] - fn stream_publisher_body_async_rejects_truncated_gzip_stream() { + #[tokio::test] + async fn stream_publisher_body_async_rejects_truncated_gzip_stream() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20075,7 +23613,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = non_html_stream_params("gzip"); let compressed = @@ -20106,8 +23646,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_rejects_truncated_deflate_stream() { + #[tokio::test] + async fn stream_publisher_body_async_rejects_truncated_deflate_stream() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20118,7 +23658,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = non_html_stream_params("deflate"); let compressed = @@ -20150,8 +23692,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_enforces_decoded_byte_cap() { + #[tokio::test] + async fn stream_publisher_body_async_enforces_decoded_byte_cap() { futures::executor::block_on(async { let mut settings = create_test_settings(); // Raw compressed input stays tiny (well under the cap); only the @@ -20166,7 +23708,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = non_html_stream_params("gzip"); let compressed = gzip_encode(&vec![b'a'; 64 * 1024]); @@ -20197,8 +23741,8 @@ mod tests { }); } - #[test] - fn body_chunk_source_resumes_after_cancelled_poll() { + #[tokio::test] + async fn body_chunk_source_resumes_after_cancelled_poll() { futures::executor::block_on(async { let mut pending_once = true; let mut yielded = false; @@ -20243,8 +23787,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_stream_with_auction_hold() { + #[tokio::test] + async fn stream_publisher_body_async_processes_stream_with_auction_hold() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20255,11 +23799,15 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20268,7 +23816,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -20281,8 +23831,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -20318,8 +23867,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered() { + #[tokio::test] + async fn stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20330,11 +23879,15 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20343,7 +23896,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -20356,8 +23911,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -20396,8 +23950,8 @@ mod tests { }); } - #[test] - fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { + #[tokio::test] + async fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { futures::executor::block_on(async { let settings = create_test_settings(); let registry = IntegrationRegistry::with_plan( @@ -20408,10 +23962,14 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); let mut params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20420,7 +23978,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, @@ -20430,8 +23990,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -20474,7 +24033,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -20482,7 +24043,9 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20491,15 +24054,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/css".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let publisher_response = PublisherResponse::Stream { response, @@ -20606,7 +24170,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -20635,7 +24201,9 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -20644,7 +24212,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -20654,24 +24224,29 @@ mod tests { auction_request: dispatched_auction.as_ref().map(|_| test_auction_request()), dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } } #[test] - fn streaming_finalize_emits_gam_attribution_head_before_origin_eof() { + fn streaming_finalize_emits_an_integration_s_head_before_origin_eof() { + use crate::integrations::registry_test_support::tag_fixture as tag; + let mut settings = create_test_settings(); settings - .integrations - .insert_config( - "gpt", + .insert_module_config( + "testing", + tag::MODULE, &serde_json::json!({ - "enabled": true, - "gam_attribution_enabled": true + "mark_bundle": true }), ) - .expect("should insert GPT config"); + .expect("should insert the stand-in's settings"); + crate::html_processor::test_support::place_on_every_page( + &mut settings, + crate::middleware::MiddlewarePhase::Fetch, + &[tag::MODULE], + ); let body = streaming_finalize_response_with_settings( html_stream_params("", None), @@ -20684,12 +24259,12 @@ mod tests { .expect("should emit UTF-8 HTML"); assert!( - html.contains("__tsjs_gam_attribution_enabled=true"), - "first rewritten head chunk should carry the primary activation flag: {html}" + html.contains(tag::HEAD_FLAG), + "first rewritten head chunk should carry the integration's head insert: {html}" ); assert!( - html.contains("data-ts-gam-attribution=\"true\""), - "first rewritten head chunk should authorize the bundle fallback: {html}" + html.contains(&format!("{}=\"true\"", tag::BUNDLE_ATTRIBUTE)), + "first rewritten head chunk should carry the bundle tag's attribute: {html}" ); } @@ -20737,18 +24312,11 @@ mod tests { fn streaming_finalize_auction_hold_emits_prefix_before_origin_eof() { // A body-close literal in script data must not stop streaming. Only the // request token emitted by lol_html at the structural end is a seam. - let page = br#"
still streaming
"#; + let page = br#"
still streaming
"#; let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should enable Next.js"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let params = html_stream_params( "", Some(DispatchedAuction::empty_for_test( @@ -20766,11 +24334,11 @@ mod tests { let html = String::from_utf8(first.to_vec()).expect("should be valid UTF-8"); assert!( html.contains("") && html.contains("still streaming"), - "RSC script data and later article bytes must stream before EOF. Got: {html}" + "script data and later article bytes must stream before EOF. Got: {html}" ); assert!( html.contains("proxy.example.com/app") && !html.contains("origin.example.com/app"), - "Next.js rewriting must complete before the parser seam: {html}" + "the stand-in's rewriting must complete before the parser seam: {html}" ); assert!( html.contains(".adSlots=JSON.parse"), @@ -20871,7 +24439,9 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for GatedAuctionProvider { fn provider_name(&self) -> &'static str { - "seam-test" + // A demand name is snake_case, and it has to match the + // `[demand]` entry the test selects for the auction to route to it. + "seam_test" } async fn request_bids( @@ -20914,24 +24484,15 @@ mod tests { } #[test] - fn parser_confirmed_auction_seam_streams_nextjs_for_every_encoding() { + fn parser_confirmed_auction_seam_streams_a_two_pass_integration_for_every_encoding() { for encoding in ["", "gzip", "deflate", "br"] { let mut settings = create_test_settings(); settings.auction.enabled = true; - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); + settings.auction.provider_names = vec!["seam_test".to_owned()]; settings.auction.timeout_ms = 60_000; - settings.auction.mediator = None; - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should enable Next.js"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let client = Arc::new(GatedAuctionHttpClient { inner: StubHttpClient::new(), released: std::sync::atomic::AtomicBool::new(false), @@ -20958,15 +24519,10 @@ mod tests { else { panic!("should dispatch a pending auction"); }; - let payload = r#"{"url":"https://origin.example.com/path","text":""}"#; - let split = payload.find("/path").expect("should find payload split"); - let first_payload = format!("1:T{:x},{}", payload.len(), &payload[..split]); - let first_script = - serde_json::to_string(&first_payload).expect("should encode first payload"); - let second_script = - serde_json::to_string(&payload[split..]).expect("should encode second payload"); - let prefix = format!( - "

before RSC

between scripts
still streaming
" + // One group split across two scripts, so the stand-in holds its + // output from the first script until the second arrives. + let prefix = String::from( + "

before the group

between scripts
still streaming
", ); let page = format!("{prefix}"); let encoded = match encoding { @@ -21029,7 +24585,7 @@ mod tests { ); assert!( prefix.contains("proxy.example.com") && !prefix.contains("origin.example.com"), - "should rewrite the split RSC group for {encoding}: {prefix}" + "should rewrite the split group for {encoding}: {prefix}" ); assert!( !prefix.contains("var b=JSON.parse("), @@ -21077,7 +24633,8 @@ mod tests { "should inject once" ); assert!( - !html.contains("ts-inline-body-close-") && !html.contains("__ts_rsc_"), + !html.contains("ts-inline-body-close-") + && !html.contains(crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX), "should remove internal markers for {encoding}: {html}" ); assert_eq!( @@ -21159,7 +24716,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); for (method, status, expected_length, expected_transfer_encoding) in BODILESS_FRAMING_CASES { @@ -21224,7 +24783,9 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let services = noop_services(); for (method, status, expected_length, expected_transfer_encoding) in BODILESS_FRAMING_CASES @@ -21310,13 +24871,17 @@ mod tests { ), ) .expect("should create integration registry"); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let make_params = || { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21325,7 +24890,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: Some(AuctionObservationContext::from_parts( @@ -21342,8 +24909,7 @@ mod tests { 10, )), price_granularity: PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), } }; let make_stream_response = || PublisherResponse::Stream { @@ -21491,7 +25057,9 @@ mod tests { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new(AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + )); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -21511,7 +25079,9 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21520,7 +25090,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -21533,8 +25105,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let publisher_response = PublisherResponse::Stream { response, @@ -21592,7 +25163,9 @@ mod tests { r#""#; let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21601,7 +25174,9 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "Text/HTML; Charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: Some( r#""# .to_string(), @@ -21611,8 +25186,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -21656,7 +25230,9 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21665,15 +25241,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -21731,18 +25308,12 @@ mod tests { /// routes through `Stream`, and the shared processor pipeline applies it. #[test] fn streaming_html_with_stream_processors_rewrites_body() { - // Configure nextjs so a stream processor is registered. + // Select the stand-in and place its middleware, so the page has a + // stream processor. let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let registry = IntegrationRegistry::with_plan( &settings, @@ -21754,8 +25325,15 @@ mod tests { .expect("should create integration registry"); assert!( - !registry.html_stream_processor_factories().is_empty(), - "nextjs integration must register an HTML stream processor" + !registry + .middleware_chain( + &settings.fetch, + crate::middleware::MiddlewarePhase::Fetch, + crate::middleware::HTML_MEDIA_TYPE, + "/", + ) + .is_empty(), + "the stand-in's middleware, which supplies an HTML stream processor, must run on the page" ); assert_eq!( classify_response_route( @@ -21774,7 +25352,9 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21783,15 +25363,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html; charset=utf-8".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -21812,23 +25393,15 @@ mod tests { ); } - /// Document-state survives from the parser pass into the stream processor. - /// `NextJsRscPlaceholderRewriter` writes into `IntegrationDocumentState` - /// during parsing; the request-local stream processor reads it and substitutes. + /// What a text handler captures during parsing reaches the stream + /// processor of the same document, which substitutes it. /// Regression test: placeholders must be inserted and removed from final output. #[test] - fn document_state_placeholders_substitute_through_streaming_path() { + fn a_handler_s_placeholders_are_substituted_through_the_streaming_path() { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); + crate::integrations::registry_test_support::payload_fixture::select_and_place( + &mut settings, + ); let registry = IntegrationRegistry::with_plan( &settings, Arc::new( @@ -21838,10 +25411,12 @@ mod tests { ) .expect("should create integration registry"); - // Small, single-fragment RSC script — placeholder path (not fallback). - let html = br#""#; + // One small script whose payload is swapped for a placeholder. + let html = br#""#; let params = OwnedProcessResponseParams { + public_prefix: crate::prefix::PublicPrefix::default(), csp_nonce_observed: None, + prepared: Arc::new(Prepared::default()), template_cache_key: None, seam_ad_slots: None, policy_headers: Vec::new(), @@ -21850,15 +25425,16 @@ mod tests { origin_url: "https://origin.example.com".to_string(), request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), + request_path: "/".to_owned(), content_type: "text/html".to_string(), + permissions_json: String::new(), ad_slots_script: None, ad_bids_state: AdBidsState::default(), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, + request_state: IntegrationRequestState::default(), }; let mut output = Vec::new(); @@ -21869,11 +25445,13 @@ mod tests { &settings, ®istry, ) - .expect("should process RSC push"); + .expect("should process the payload script"); let processed = String::from_utf8(output).expect("valid UTF-8"); assert!( - !processed.contains("__ts_rsc_payload_"), + !processed.contains( + crate::integrations::registry_test_support::payload_fixture::PLACEHOLDER_PREFIX + ), "placeholder must be substituted before reaching output. Got: {processed}" ); assert!( @@ -21889,11 +25467,11 @@ mod tests { #[cfg(test)] mod creative_opportunities_tests { use super::super::{ - AdBidsState, MatchedSlotsContext, build_ad_slots_script, build_auction_request, - build_bid_map, build_bids_script, diagnostics_auction_id, html_escape_for_script, + AdBidsState, MatchedSlotsContext, auction_token, build_ad_slots_script, + build_auction_request, build_bid_map, build_bids_script, html_escape_for_script, write_bids_to_state, }; - use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; + use crate::auction::types::{Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; use crate::creative_opportunities::{ CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, @@ -21939,6 +25517,8 @@ mod tests { width: 300, height: 250, media_type: MediaType::Banner, + min_viewport_width: None, + max_viewport_width: None, }], floor_price: Some(0.50), targeting: [("pos".to_string(), "atf".to_string())] @@ -22006,8 +25586,53 @@ mod tests { "must NOT contain request_id" ); assert_eq!( - slot_json["targeting"]["ts"], "operator-value", - "should forward operator-provided ts targeting verbatim" + slot_json["targeting"]["ts"], "operator-value", + "should forward operator-provided ts targeting verbatim" + ); + } + + #[test] + fn slot_json_carries_viewport_sizes_only_when_a_format_is_bounded() { + let config = make_config(); + let plain = crate::publisher::build_slot_json(&make_slot(), &config, "example") + .expect("should build slot JSON"); + assert!( + plain.get("viewport_sizes").is_none(), + "a slot without bounds should carry no viewport sizes" + ); + + let mut slot = make_slot(); + slot.formats = vec![ + CreativeOpportunityFormat { + width: 300, + height: 600, + media_type: MediaType::Banner, + min_viewport_width: Some(1040), + max_viewport_width: None, + }, + CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + min_viewport_width: None, + max_viewport_width: None, + }, + ]; + let banded = crate::publisher::build_slot_json(&slot, &config, "example") + .expect("should build slot JSON"); + + assert_eq!( + banded["viewport_sizes"], + serde_json::json!([ + [[1040, 0], [[300, 600], [300, 250]]], + [[0, 0], [[300, 250]]] + ]), + "should carry each band as a viewport and its sizes, widest first" + ); + assert_eq!( + banded["formats"], + serde_json::json!([[300, 600], [300, 250]]), + "formats should still list every size" ); } @@ -22159,28 +25784,28 @@ mod tests { } /// Guards the browser-visible token every auction path shares: it must - /// be fresh per auction and absent unless diagnostics can consume it. + /// be fresh per auction and absent unless a module reads it. #[test] - fn diagnostics_auction_id_is_fresh_and_gated() { - let mut settings = test_settings(); + fn auction_token_is_fresh_and_made_only_where_a_module_reads_it() { + let settings = test_settings(); + let orchestrator = crate::auction::orchestrator::AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); assert_eq!( - diagnostics_auction_id(&settings), + auction_token(&orchestrator), None, - "no token should be minted without the diagnostics integration" + "no token should be minted where no module reads one" ); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); + let orchestrator = orchestrator.publishing_auction_token(); let first = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + auction_token(&orchestrator).expect("should mint a token where a module reads one"); let second = - diagnostics_auction_id(&settings).expect("enabled diagnostics should mint a token"); + auction_token(&orchestrator).expect("should mint a token where a module reads one"); assert!( first.starts_with("ts-auc-"), - "token should use the diagnostics prefix, got `{first}`" + "token should use the auction token prefix, got `{first}`" ); assert_ne!(first, second, "each auction should mint its own token"); } @@ -23063,38 +26688,40 @@ mod tests { } #[test] - fn bid_map_exposes_aps_renderer_and_selected_bid_id() { + fn bid_map_exposes_a_typed_renderer_and_the_bid_it_picks() { // Sanitization is opt-in, so enable it: the script-only creative // below is what drives this bid onto the renderer path. Left at the // default it would survive processing as an ordinary creative. let mut settings = test_settings(); settings.auction.sanitize_creatives = true; - let mut bid = make_bid("atf_sidebar_ad", 1.50, "aps", "fallback-ad", "", ""); + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "fallback-ad", "", ""); bid.bid_id = Some("selected-bid".to_string()); bid.creative = Some("".to_string()); bid.nurl = None; bid.burl = None; - bid.renderer = Some(BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - })); + bid.renderer = Some( + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "selected-bid", + "creativeUrl": "https://creative.example/render", + "envelope": "fictional-base64", + }), + ) + .expect("should build the example renderer descriptor") + .picking_bid_by("bidId"), + ); let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let obj = map["atf_sidebar_ad"] .as_object() - .expect("should include APS bid"); + .expect("should include the renderer bid"); - assert_eq!(obj["hb_bidder"], "aps"); + assert_eq!(obj["hb_bidder"], "example"); assert_eq!(obj["hb_adid"], "selected-bid"); - assert_eq!(obj["renderer"]["type"], "aps"); + assert_eq!(obj["renderer"]["type"], "example"); assert_eq!(obj["renderer"]["bidId"], "selected-bid"); assert!(obj.get("adm").is_none()); @@ -23103,6 +26730,67 @@ mod tests { assert!(script.contains("\\u003C/script\\u003E")); } + #[test] + fn bid_map_prefers_the_renderer_bid_id_over_ad_id_and_the_openrtb_bid_id() { + // Every hb_adid source carries a different value, so the assertion + // below passes only when the renderer field is the one read. The + // envelope is oversized on purpose: reading this field must not + // copy it. + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); + bid.bid_id = Some("openrtb-bid-id".to_string()); + bid.cache_id = None; + bid.creative = Some("".to_string()); + bid.renderer = Some( + BidRenderer::new( + "example", + serde_json::json!({ + "version": 1, + "bidId": "renderer-bid-id", + "creativeUrl": "https://creative.example/render", + "envelope": "A".repeat(200 * 1024), + }), + ) + .expect("should build the example renderer descriptor") + .picking_bid_by("bidId"), + ); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include the renderer bid"); + + assert_eq!( + obj["hb_adid"], "renderer-bid-id", + "should read hb_adid from the renderer payload rather than ad_id or bid_id" + ); + } + + #[test] + fn bid_map_ignores_a_renderer_bid_id_its_builder_did_not_state() { + let mut settings = test_settings(); + settings.auction.sanitize_creatives = true; + let mut bid = make_bid("atf_sidebar_ad", 1.50, "example", "ad-id-value", "", ""); + bid.cache_id = None; + bid.renderer = Some( + BidRenderer::new("example", serde_json::json!({ "bidId": "renderer-bid-id" })) + .expect("should build renderer descriptor"), + ); + let winning_bids = HashMap::from([("atf_sidebar_ad".to_string(), bid)]); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); + let obj = map["atf_sidebar_ad"] + .as_object() + .expect("should include the bid"); + + assert_eq!( + obj["hb_adid"], "ad-id-value", + "should ignore a bidId the code that built the renderer did not state it picks its bid by" + ); + } + #[test] fn bid_map_omits_creative_rejected_by_processing_without_renderer() { // Sanitization is opt-in, so enable it: script-only markup is what @@ -23595,8 +27283,8 @@ mod tests { fn settings_with_co() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } @@ -23604,23 +27292,26 @@ mod tests { /// Settings for a deployment that has no `[creative_opportunities]` /// section, so page-bids answers `404`. fn settings_without_co() -> Settings { - let toml = format!("{}\n[auction]\nenabled = true\n", crate_test_settings_str()); + let toml = format!( + "{}\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") + ); Settings::from_toml(&toml) .expect("should parse settings without creative_opportunities") } fn settings_with_co_auction_disabled() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = false\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = false\n") ); Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") } fn settings_with_co_templates_disabled() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + "{}\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); Settings::from_toml(&toml).expect("should parse settings with disabled templates") } @@ -23675,6 +27366,8 @@ mod tests { width: 300, height: 250, media_type: crate::auction::types::MediaType::Banner, + min_viewport_width: None, + max_viewport_width: None, }], floor_price: Some(0.50), targeting: Default::default(), @@ -23691,7 +27384,9 @@ mod tests { #[tokio::test] async fn page_bids_format_absent_or_json_returns_json() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); for path_and_format in ["/2024/article", "/2024/article&format=json"] { let response = run_page_bids_response( &settings, @@ -23723,7 +27418,9 @@ mod tests { #[tokio::test] async fn page_bids_format_rejects_removed_unknown_and_empty_values() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); for format in ["fragment", "scrpit", ""] { let response = run_page_bids_response( &settings, @@ -23802,23 +27499,34 @@ mod tests { captured_request: Arc>>, winning_bid: bool, ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); orchestrator.register_provider(Arc::new(AuctionIdTestProvider { captured_request, winning_bid, })); + // The plan compiled for a deployment publishes a token when a + // selected module reads one, and the stand-in is such a module. + if settings + .selects_module(crate::integrations::registry_test_support::request_fixture::MODULE) + { + orchestrator = orchestrator.publishing_auction_token(); + } orchestrator } #[tokio::test] async fn page_bids_response_includes_auction_id_only_for_winning_bids() { let mut settings = settings_with_co(); - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); + settings.demand = crate::auction::test_support::demand_named( + crate::auction::test_support::plain_fixture::MODULE, + &[AUCTION_ID_TEST_PROVIDER], + ); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let slots = article_slot(); let winning_stub = Arc::new(StubHttpClient::new()); winning_stub.push_response(200, b"winner".to_vec()); @@ -23873,7 +27581,7 @@ mod tests { .to_string(); assert!( winning_auction_id.starts_with("ts-auc-"), - "page-bids should expose a freshly minted diagnostics token, got `{winning_auction_id}`" + "page-bids should expose a freshly minted auction token, got `{winning_auction_id}`" ); assert_ne!( winning_auction_id, auction_request.id, @@ -23923,10 +27631,10 @@ mod tests { } /// The browser-visible auction ID is minted per auction and only for - /// deployments that run the diagnostics integration, so it can neither + /// deployments that run a module that reads it, so it can neither /// carry EC identity across auctions nor reach pages that ignore it. #[tokio::test] - async fn page_bids_auction_id_is_per_auction_and_gated_on_diagnostics() { + async fn page_bids_auction_id_is_per_auction_and_made_only_where_a_module_reads_it() { async fn winning_auction_id(settings: &Settings) -> Option { let slots = article_slot(); let stub = Arc::new(StubHttpClient::new()); @@ -23970,32 +27678,34 @@ mod tests { } let mut settings = settings_with_co(); - settings.auction.providers = - crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) - .expect("should enable diagnostics"); + settings.demand = crate::auction::test_support::demand_named( + crate::auction::test_support::plain_fixture::MODULE, + &[AUCTION_ID_TEST_PROVIDER], + ); + settings.select_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); let first = winning_auction_id(&settings) .await - .expect("first auction should expose a diagnostics token"); + .expect("first auction should expose an auction token"); let second = winning_auction_id(&settings) .await - .expect("second auction should expose a diagnostics token"); + .expect("second auction should expose an auction token"); assert_ne!( first, second, "each auction for the same visitor should mint its own token" ); - settings - .integrations - .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": false })) - .expect("should disable diagnostics"); + settings.remove_module( + "testing", + crate::integrations::registry_test_support::request_fixture::MODULE, + ); assert_eq!( winning_auction_id(&settings).await, None, - "no auction metadata should reach the page without the diagnostics integration" + "no auction metadata should reach the page where no module reads it" ); } @@ -24010,7 +27720,9 @@ mod tests { #[tokio::test] async fn deprecated_alias_response_matches_canonical_path() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let canonical = run_page_bids_response( &settings, @@ -24047,7 +27759,9 @@ mod tests { #[tokio::test] async fn deprecated_alias_response_is_marked_deprecated() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let canonical = run_page_bids_response( &settings, @@ -24091,7 +27805,9 @@ mod tests { settings.creative_opportunities.is_none(), "test settings should have no creative opportunities configured" ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let response = run_page_bids_response( &settings, @@ -24118,7 +27834,9 @@ mod tests { #[tokio::test] async fn cross_site_request_is_denied_before_configuration_is_revealed() { let settings = settings_without_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = Request::builder() .method(Method::GET) .uri(format!("https://test-publisher.com{PAGE_BIDS_PATH}?path=/")) @@ -24138,7 +27856,9 @@ mod tests { #[tokio::test] async fn cross_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-fetch-site", "cross-site"); @@ -24155,7 +27875,9 @@ mod tests { #[tokio::test] async fn missing_fetch_metadata_without_tsjs_header_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); req.headers_mut().remove("sec-fetch-site"); @@ -24172,7 +27894,9 @@ mod tests { #[tokio::test] async fn missing_fetch_metadata_with_tsjs_header_is_allowed() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); req.headers_mut().remove("sec-fetch-site"); set_test_header(&mut req, "x-tsjs-page-bids", "1"); @@ -24190,7 +27914,9 @@ mod tests { #[tokio::test] async fn same_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut req = make_page_bids_request("/2024/01/my-article/"); // `same-site` admits sibling origins under the same registrable // domain — not trusted to spend SSP quota. @@ -24211,7 +27937,9 @@ mod tests { // Spec §8 kill-switch: creative-opportunities.toml with zero slots disables // all server-side auction activity and injection. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let req = make_page_bids_request("/2024/01/my-article/"); let body = run_page_bids(&settings, &orchestrator, &[], req).await; @@ -24240,7 +27968,9 @@ mod tests { // but the server must not burn SSP request quota running a real auction // for them. Same gate the publisher path applies. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header( @@ -24274,7 +28004,9 @@ mod tests { // Navigations triggered by Sec-Purpose=prefetch should not fire real // SSP auctions — the user has not yet visited the page. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-purpose", "prefetch"); @@ -24302,7 +28034,9 @@ mod tests { #[tokio::test] async fn page_bids_omits_only_over_limit_dynamic_slot() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let mut over_limit = article_slot() .into_iter() .next() @@ -24343,7 +28077,9 @@ mod tests { async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match @@ -24405,7 +28141,9 @@ mod tests { // the auction is off. Consent is allowed here so the test isolates // the kill switch. let settings = settings_with_co_auction_disabled(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -24434,7 +28172,9 @@ mod tests { // The dedicated template switch must suppress publisher/page-bids // delivery without using the global auction switch. let settings = settings_with_co_templates_disabled(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -24465,7 +28205,9 @@ mod tests { // hook does not create GPT slots client-side — matching the publisher // navigation path's shared `evaluate_ad_stack_gate` gate. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(&settings), + ); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -24535,6 +28277,11 @@ mod tests { /// tests drive the real handlers with a divergent edge host and assert on /// the auction request the orchestrator dispatched and on the telemetry rows /// the handler emitted. + /// + /// The same capturing setup also covers how both paths key the identity + /// graph. A module whose canonical form differs from the cookie value has + /// its row read under the canonical key, so the partner ID stored there + /// reaches the dispatched auction request. mod navigation_publisher_domain_tests { use super::*; use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; @@ -24542,6 +28289,7 @@ mod tests { use crate::auction::types::AuctionRequest; use crate::auction::{AuctionContext, AuctionOrchestrator}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::ec::tests::{CANONICAL_COOKIE_VALUE, CANONICAL_KV_KEY, CanonicalizingModule}; use crate::platform::test_support::{ NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, }; @@ -24556,7 +28304,7 @@ mod tests { /// `[publisher] domain` from [`crate_test_settings_str`]. const CONFIGURED_DOMAIN: &str = "test-publisher.com"; - const CAPTURING_PROVIDER: &str = "request-capturing-provider"; + const CAPTURING_PROVIDER: &str = "request_capturing_provider"; /// Records the [`AuctionRequest`] the orchestrator dispatched, then /// fails its launch so no real transport handle is needed. @@ -24625,11 +28373,11 @@ mod tests { fn settings_with_capturing_provider() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.{CAPTURING_PROVIDER}]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ + "{}\n[demand]\nmodules = [\"{CAPTURING_PROVIDER}\"]\n\n[demand.{CAPTURING_PROVIDER}]\nimplementation = \"auction.plain-fixture\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ [creative_opportunities]\ngam_network_id = \"12345\"\n", - crate_test_settings_str() + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n") ); - Settings::from_toml(&toml).expect("should parse settings with a capturing provider") + settings_from_toml(&toml) } fn article_slot() -> Vec { @@ -24642,6 +28390,8 @@ mod tests { width: 300, height: 250, media_type: MediaType::Banner, + min_viewport_width: None, + max_viewport_width: None, }], floor_price: Some(0.50), targeting: Default::default(), @@ -24713,7 +28463,9 @@ mod tests { settings: &Settings, captured: &Arc>>, ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut orchestrator = AuctionOrchestrator::new( + crate::auction::test_support::legacy_auction_config(settings), + ); orchestrator.register_provider(Arc::new(RequestCapturingProvider { captured: Arc::clone(captured), })); @@ -24798,8 +28550,12 @@ mod tests { .body(EdgeBody::empty()) .expect("should build test request"); + let registry = test_registry(&settings); let _ = handle_publisher_request( - &settings, + AppContext { + settings: &settings, + integration_registry: ®istry, + }, &services, None, &mut ec_context, @@ -24882,8 +28638,12 @@ mod tests { .expect("should build test request"); let slots = slots_with_over_limit_dynamic_sibling(); + let registry = test_registry(&settings); let _ = handle_publisher_request( - &settings, + AppContext { + settings: &settings, + integration_registry: ®istry, + }, &services, None, &mut ec_context, @@ -24944,5 +28704,284 @@ mod tests { assert_only_renderable_slot_was_auctioned(&captured); } + + /// The bidstream partner whose stored ID the canonical row carries. + const CANONICAL_ROW_PARTNER: &str = "ssp.example.com"; + + /// The partner ID the canonical row stores for [`CANONICAL_ROW_PARTNER`]. + const CANONICAL_ROW_UID: &str = "partner-uid-123"; + + /// An identity graph holding one live row under [`CANONICAL_KV_KEY`], + /// and a registry that forwards that row's partner ID as an EID. + fn canonical_row_graph_and_registry() -> (KvIdentityGraph, PartnerRegistry) { + let graph = KvIdentityGraph::in_memory("navigation-canonical-store"); + graph + .create( + CANONICAL_KV_KEY, + &crate::ec::kv_types::KvEntry::minimal( + CANONICAL_ROW_PARTNER, + CANONICAL_ROW_UID, + 1_741_824_000, + ), + ) + .expect("should seed the row under the canonical key"); + let registry = PartnerRegistry::from_config(&[crate::settings::EcPartner { + name: "Canonical row partner".to_owned(), + source_domain: CANONICAL_ROW_PARTNER.to_owned(), + openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(crate::redacted::Redacted::new( + "canonical-row-partner-token-32-bytes".to_owned(), + )), + batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), + pull_sync_enabled: false, + pull_sync_url: None, + pull_sync_allowed_domains: vec![], + pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), + pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), + ts_pull_token: None, + }]) + .expect("should build a registry with one bidstream partner"); + (graph, registry) + } + + /// A returning visitor carrying the identifier [`CanonicalizingModule`] + /// creates, with consent that permits the server-side auction. + fn canonicalizing_returning_visitor() -> EcContext { + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + EcContext::new_for_test(Some(CANONICAL_COOKIE_VALUE.to_owned()), consent) + .with_module_for_test(Arc::new(CanonicalizingModule)) + } + + /// Asserts the dispatched auction request carried the partner ID the + /// canonical row stores, and the request snapshot is bound to the + /// canonical key. + fn assert_auction_used_the_canonical_row( + captured: &Arc>>, + ec_context: &EcContext, + ) { + let request = captured + .lock() + .expect("should lock captured request") + .clone() + .expect("should dispatch an auction request"); + let eids = request + .user + .eids + .expect("the auction should carry server-side EIDs from the identity graph"); + assert!( + eids.iter().any(|eid| eid.source == CANONICAL_ROW_PARTNER + && eid.uids.iter().any(|uid| uid.id == CANONICAL_ROW_UID)), + "the auction should carry the partner ID stored under the canonical key, got {eids:?}" + ); + assert!( + ec_context + .kv_snapshot() + .entry_for(CANONICAL_KV_KEY) + .is_some(), + "the request snapshot should be bound to the canonical key" + ); + } + + #[tokio::test] + async fn initial_navigation_reads_the_identity_row_under_the_canonical_key() { + // The identity graph stores a row under the owning module's + // canonical form of the identifier. Preloaded and resolved under + // the identifier as issued, a module whose canonical form differs + // from the cookie value found no row, so the auction carried no + // server-side EIDs. + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = services_with( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let (graph, registry) = canonical_row_graph_and_registry(); + let mut ec_context = canonicalizing_returning_visitor(); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://{EDGE_HOST}/2024/01/my-article/")) + .header(header::HOST, EDGE_HOST) + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build test request"); + + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &article_slot(), + registry: Some(®istry), + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + assert_auction_used_the_canonical_row(&captured, &ec_context); + } + + #[tokio::test] + async fn initial_navigation_keeps_a_new_identifiers_cookie() { + // Generation binds the request snapshot to the canonical key. The + // navigation preload that follows read under the identifier as + // issued, found nothing there, and replaced that snapshot with a + // miss, so EC finalization skipped the cookie for the identifier + // this request had just created. The second store's first point + // read misses the row generation just wrote, as an eventually + // consistent store can right after a write. The preload keeps + // generation's snapshot in that case only when it compares both + // snapshots under the canonical key. + let stores = [ + ( + "a consistent store", + KvIdentityGraph::in_memory("navigation-new-identifier-store"), + ), + ( + "a store whose first point read misses", + KvIdentityGraph::stale_lookup("navigation-stale-read-store", 1), + ), + ]; + for (store, graph) in stores { + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = services_with( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent) + .with_module_for_test(Arc::new(CanonicalizingModule)); + ec_context + .generate_if_needed( + &settings, + Some(&graph), + &crate::platform::test_support::noop_services(), + ) + .await + .expect("should create the identifier through the module"); + assert_eq!( + ec_context.ec_value(), + Some(CANONICAL_COOKIE_VALUE), + "test precondition: the module should create its identifier" + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://{EDGE_HOST}/2024/01/my-article/")) + .header(header::HOST, EDGE_HOST) + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build test request"); + + let _ = handle_publisher_request( + AppContext { + settings: &settings, + integration_registry: &test_registry(&settings), + }, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + let mut response = Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + &crate::platform::test_support::noop_services(), + ) + .await; + + let cookies: Vec<&str> = response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + assert!( + cookies + .iter() + .any(|cookie| cookie.starts_with("ts-ec=") && !cookie.contains("Max-Age=0")), + "the identifier this request created should reach the browser with {store}, \ + got {cookies:?}" + ); + } + } + + #[tokio::test] + async fn page_bids_reads_the_identity_row_under_the_canonical_key() { + // The same keying for the SPA re-auction endpoint, which loads the + // row itself only once a live auction will run. + let settings = settings_with_capturing_provider(); + let captured = Arc::new(Mutex::new(None)); + let orchestrator = orchestrator_capturing_request(&settings, &captured); + let services = services_with( + Arc::new(crate::platform::test_support::NoopHttpClient), + Arc::new(RecordingTelemetrySink::default()), + ); + let (graph, registry) = canonical_row_graph_and_registry(); + let mut ec_context = canonicalizing_returning_visitor(); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri(format!( + "https://{EDGE_HOST}/_ts/page-bids?path=/2024/01/my-article/" + )) + .header(header::HOST, EDGE_HOST) + .body(EdgeBody::empty()) + .expect("should build test request"); + req.headers_mut().insert( + header::HeaderName::from_static("sec-fetch-site"), + HeaderValue::from_static("same-origin"), + ); + + let _ = handle_page_bids( + &settings, + &services, + Some(&graph), + AuctionDispatch { + orchestrator: &orchestrator, + slots: &article_slot(), + registry: Some(®istry), + }, + &mut ec_context, + req, + ) + .await + .expect("should return ok response"); + + assert_auction_used_the_canonical_row(&captured, &ec_context); + } } } diff --git a/crates/trusted-server-core/src/redacted.rs b/crates/trusted-server-core/src/redacted.rs index 3389c4c817..df17f43708 100644 --- a/crates/trusted-server-core/src/redacted.rs +++ b/crates/trusted-server-core/src/redacted.rs @@ -1,14 +1,89 @@ -// NOTE: This file is also included in build.rs via #[path]. -// It must remain self-contained (no `crate::` imports). - -//! A wrapper type that redacts sensitive values in [`Debug`] and [`fmt::Display`] output. +//! A wrapper type that redacts sensitive values in [`Debug`] and [`fmt::Display`] output, +//! and the marking that hides sensitive settings from the configuration view. //! //! Use [`Redacted`] for secrets, passwords, API keys, and other sensitive values //! that must never appear in logs or error messages. +//! +//! A settings field holding a value that is sensitive but is not a +//! [`Redacted`] carries `#[serde(serialize_with = "crate::redacted::sensitive")]`. +use core::cell::Cell; use core::fmt; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeMap as _; +use serde::{Deserialize, Serialize, Serializer}; + +/// The key a sensitive value is wrapped under while [`to_marked_value`] +/// runs. It starts with a control character, so no setting can share it. +pub const SENSITIVE_MARKER: &str = "\u{1}sensitive"; + +std::thread_local! { + static MARKING: Cell = const { Cell::new(false) }; +} + +/// Restores the marking flag when a marked serialization ends, including +/// by a panic. +struct MarkingGuard { + previous: bool, +} + +impl MarkingGuard { + fn begin() -> Self { + Self { + previous: MARKING.with(|marking| marking.replace(true)), + } + } +} + +impl Drop for MarkingGuard { + fn drop(&mut self) { + MARKING.with(|marking| marking.set(self.previous)); + } +} + +/// Serializes `value` to JSON with every sensitive value wrapped as +/// `{SENSITIVE_MARKER: value}`. +/// +/// Only this call marks. Every other serialization, the configuration push +/// included, writes sensitive values as they are. +/// +/// # Errors +/// +/// When `value` cannot be represented as JSON. +pub fn to_marked_value(value: &T) -> Result +where + T: Serialize + ?Sized, +{ + let _guard = MarkingGuard::begin(); + serde_json::to_value(value) +} + +/// Whether a [`to_marked_value`] call is running on this thread. +#[must_use] +pub fn is_marking() -> bool { + MARKING.with(Cell::get) +} + +/// Serializes a sensitive settings field, for +/// `#[serde(serialize_with = "crate::redacted::sensitive")]`, marked as +/// [`to_marked_value`] says. +/// +/// # Errors +/// +/// As the serializer's own. +pub fn sensitive(value: &T, serializer: S) -> Result +where + T: Serialize + ?Sized, + S: Serializer, +{ + if is_marking() { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry(SENSITIVE_MARKER, value)?; + map.end() + } else { + value.serialize(serializer) + } +} /// Wraps a value so that [`Debug`] and [`fmt::Display`] print `[REDACTED]` /// instead of the inner contents. @@ -16,6 +91,8 @@ use serde::{Deserialize, Serialize}; /// Access the real value via [`expose`](Redacted::expose). Callers must /// never log or display the returned reference. /// +/// Serializes as the inner value, marked as [`to_marked_value`] says. +/// /// # Examples /// /// ``` @@ -25,16 +102,21 @@ use serde::{Deserialize, Serialize}; /// assert_eq!(format!("{:?}", secret), "[REDACTED]"); /// assert_eq!(secret.expose(), "my-secret-key"); /// ``` -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Deserialize)] #[serde(transparent)] pub struct Redacted(T); +impl Serialize for Redacted { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + sensitive(&self.0, serializer) + } +} + impl Redacted { /// Creates a new [`Redacted`] value. - #[allow( - dead_code, - reason = "used by the library crate but not by build.rs path inclusion" - )] pub fn new(value: T) -> Self { Self(value) } @@ -143,6 +225,55 @@ mod tests { ); } + #[test] + fn serialization_is_transparent_outside_a_marked_serialization() { + let secret = Redacted::new("serialize-me".to_owned()); + + let value = serde_json::to_value(&secret).expect("should serialize"); + + assert_eq!(value, serde_json::json!("serialize-me")); + assert!(!is_marking(), "should not be marking outside the call"); + } + + #[test] + fn a_marked_serialization_wraps_redacted_and_sensitive_values() { + #[derive(Serialize)] + struct Fixture { + plain: String, + #[serde(serialize_with = "sensitive")] + marked: String, + secret: Redacted, + } + let fixture = Fixture { + plain: "open".to_owned(), + marked: "hidden".to_owned(), + secret: Redacted::new("secret".to_owned()), + }; + + let marked = to_marked_value(&fixture).expect("should serialize"); + let normal = serde_json::to_value(&fixture).expect("should serialize"); + + assert_eq!( + marked, + serde_json::json!({ + "plain": "open", + "marked": { SENSITIVE_MARKER: "hidden" }, + "secret": { SENSITIVE_MARKER: "secret" }, + }), + "should wrap only the marked values" + ); + assert_eq!( + normal, + serde_json::json!({ + "plain": "open", + "marked": "hidden", + "secret": "secret", + }), + "should leave an ordinary serialization unchanged" + ); + assert!(!is_marking(), "should clear the flag when the call ends"); + } + #[test] fn struct_with_redacted_field_debug() { #[derive(Debug)] diff --git a/crates/trusted-server-core/src/request_signing/endpoints.rs b/crates/trusted-server-core/src/request_signing/endpoints.rs index d35e51a151..6308d6d049 100644 --- a/crates/trusted-server-core/src/request_signing/endpoints.rs +++ b/crates/trusted-server-core/src/request_signing/endpoints.rs @@ -1,18 +1,17 @@ //! HTTP endpoint handlers for request signing operations. //! -//! This module provides endpoint handlers for JWKS retrieval, signature verification, -//! key rotation, and key deactivation operations. +//! This module provides endpoint handlers for JWKS retrieval and signature +//! verification, and the key ID rules key creation follows. use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{Request, Response, StatusCode, header}; use serde::{Deserialize, Serialize}; -use crate::error::{IntoHttpResponse, TrustedServerError}; +use crate::error::TrustedServerError; use crate::http_util::enforce_max_body_size; use crate::platform::RuntimeServices; use crate::request_signing::discovery::TrustedServerDiscovery; -use crate::request_signing::rotation::KeyRotationManager; use crate::request_signing::signing; use crate::settings::Settings; @@ -99,7 +98,6 @@ pub struct VerifySignatureResponse { } const VERIFY_MAX_BODY_BYTES: usize = 4096; -const ADMIN_MAX_BODY_BYTES: usize = 4096; /// Will verify a signature given a payload and kid /// Useful for testing integration with signatures @@ -160,62 +158,11 @@ pub fn handle_verify_signature( Ok(json_response(StatusCode::OK, response_json)) } -/// JSON request body for the key-rotation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct RotateKeyRequest { - /// Optional explicit key identifier for the new signing key. - #[serde(skip_serializing_if = "Option::is_none")] - pub kid: Option, -} - -/// JSON response body for the key-rotation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct RotateKeyResponse { - /// Whether the rotation operation succeeded. - pub success: bool, - /// Human-readable summary of the rotation result. - pub message: String, - /// Newly generated or supplied key identifier. - pub new_kid: String, - /// Previously active key identifier, if one existed. - pub previous_kid: Option, - /// Active key identifiers after the rotation completes. - pub active_kids: Vec, - /// Public JWK associated with the newly active key. - pub jwk: serde_json::Value, - /// Error detail when rotation fails. - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -struct SigningStoreIds<'a> { - config_store_id: &'a str, - secret_store_id: &'a str, -} - const MAX_KID_LENGTH: usize = 128; -fn signing_store_ids( - settings: &Settings, -) -> Result, Report> { - settings - .request_signing - .as_ref() - .map(|setting| SigningStoreIds { - config_store_id: setting.config_store_id.as_str(), - secret_store_id: setting.secret_store_id.as_str(), - }) - .ok_or_else(|| { - TrustedServerError::Configuration { - message: "missing signing storage configuration".to_string(), - } - .into() - }) -} - -/// Validates the structural constraints every kid must satisfy regardless of -/// operation: non-empty, length-bounded, and limited to a safe charset so a kid -/// cannot smuggle a CSV separator or other control characters into storage. +/// Validates the structural constraints every kid must satisfy: non-empty, +/// length-bounded, and limited to a safe charset so a kid cannot smuggle a CSV +/// separator or other control characters into storage. fn validate_kid_format(kid: &str) -> Result<(), Report> { if kid.is_empty() || kid.len() > MAX_KID_LENGTH { return Err(Report::new(TrustedServerError::BadRequest { @@ -235,27 +182,23 @@ fn validate_kid_format(kid: &str) -> Result<(), Report> { Ok(()) } -/// Validates a kid for creation/rotation. +/// Validates a kid for a new key. /// -/// Enforces the **portable-KID contract**: a kid created through any adapter must -/// be storable by every platform's key-name encoder, so create/rotate validates -/// against the strictest common denominator. Concretely the kid must start with a -/// lowercase ASCII letter, on top of the structural [`validate_kid_format`] -/// charset checks. +/// Enforces the **portable-KID contract**: a new kid must be storable by every +/// platform's key-name encoder, so creation validates against the strictest +/// common denominator. Concretely the kid must start with a lowercase ASCII +/// letter, on top of the structural [`validate_kid_format`] charset checks. /// /// That floor is currently set by the Fermyon Spin variable encoder /// (`spin_variable_name` in the Spin adapter's `platform.rs`), which requires a /// lowercase ASCII leading character: it would otherwise alias digit-leading keys /// (`1foo` and `n1foo` both map to `v_n1foo`) or reject uppercase/punctuation /// leading keys (`KidA`, `_kid`, `-kid`, `.kid`, `:kid`) at storage time. -/// Enforcing the floor here turns those into a client-correctable 400 on every -/// adapter instead of a runtime 5xx on Spin. [`kid_is_creatable`] exposes this -/// predicate so adapter crates can pin the contract with a test — core must never -/// accept a kid its encoder rejects. System-generated KIDs (`ts-YYYY-MM-DD`) -/// start with `t` and are unaffected. -/// -/// Deactivation/deletion deliberately uses the looser [`validate_kid_format`] so -/// legacy KIDs created under earlier validation rules remain removable. +/// Checking the floor here refuses such a kid for every platform rather than +/// leaving it to fail at storage time on Spin. [`kid_is_creatable`] exposes +/// this predicate so adapter crates can pin the contract with a test, because +/// core must never accept a kid its encoder rejects. System-generated KIDs +/// (`ts-YYYY-MM-DD`) start with `t` and are unaffected. fn validate_kid(kid: &str) -> Result<(), Report> { validate_kid_format(kid)?; @@ -268,233 +211,28 @@ fn validate_kid(kid: &str) -> Result<(), Report> { Ok(()) } -/// Returns whether `kid` satisfies the create/rotate portable-KID contract -/// enforced by [`validate_kid`]. +/// Returns whether `kid` satisfies the portable-KID contract that +/// [`validate_kid`] checks for a new key. /// -/// Exposed so platform adapter crates can assert their key-name encoder accepts -/// every kid this validation admits, pinning the cross-adapter contract against -/// silent drift between this validation floor and an adapter's storage encoder. +/// Exposed for the tool that creates keys, which asks before it writes +/// anything, and so platform adapter crates can assert their key-name encoder +/// accepts every kid this validation admits, pinning the cross-adapter contract +/// against silent drift between this validation floor and an adapter's storage +/// encoder. #[must_use] pub fn kid_is_creatable(kid: &str) -> bool { validate_kid(kid).is_ok() } -/// Rotates the current active kid by generating and saving a new one. -/// -/// # Response contract -/// -/// Returns `200 OK` with `success: true` on success, `400 Bad Request` for an -/// invalid operator-supplied `kid`, or `500 Internal Server Error` when rotation -/// fails. Failure responses include `success: false` and a populated `error` -/// field. Unlike [`handle_verify_signature`], the error field contains internal -/// detail — this is intentional because this endpoint is auth-gated and -/// operator-facing only. -/// -/// # Errors -/// -/// Returns an error if the request signing settings are missing or JSON parsing fails. -pub fn handle_rotate_key( - settings: &Settings, - services: &RuntimeServices, - req: Request, -) -> Result, Report> { - let SigningStoreIds { - config_store_id, - secret_store_id, - } = signing_store_ids(settings)?; - - let body = request_body_bytes(req.into_body(), "rotate-key")?; - enforce_max_body_size(&body, ADMIN_MAX_BODY_BYTES, "rotate-key")?; - let rotate_req: RotateKeyRequest = if body.is_empty() { - RotateKeyRequest { kid: None } - } else { - serde_json::from_slice(&body).change_context(TrustedServerError::Configuration { - message: "invalid JSON request body".into(), - })? - }; - - let manager = KeyRotationManager::new(config_store_id, secret_store_id); - let validation_result = if let Some(kid) = rotate_req.kid.as_deref() { - validate_kid(kid) - } else { - Ok(()) - }; - let result = validation_result.and_then(|()| manager.rotate_key(services, rotate_req.kid)); - - match result { - Ok(result) => { - let jwk_value = serde_json::to_value(&result.jwk).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize JWK: {}", e), - }) - })?; - - let response = RotateKeyResponse { - success: true, - message: "Key rotated successfully".to_string(), - new_kid: result.new_kid, - previous_kid: result.previous_kid, - active_kids: result.active_kids, - jwk: jwk_value, - error: None, - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(StatusCode::OK, response_json)) - } - Err(e) => { - let status = e.current_context().status_code(); - let response = RotateKeyResponse { - success: false, - message: "Key rotation failed".to_string(), - new_kid: String::new(), - previous_kid: None, - active_kids: vec![], - jwk: serde_json::json!({}), - error: Some(format!("{}", e)), - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(status, response_json)) - } - } -} - -/// JSON request body for the key-deactivation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct DeactivateKeyRequest { - /// Key identifier to deactivate or delete. - pub kid: String, - /// Whether the key should be deleted from storage after deactivation. - #[serde(default)] - pub delete: bool, -} - -/// JSON response body for the key-deactivation endpoint. -#[derive(Debug, Deserialize, Serialize)] -pub struct DeactivateKeyResponse { - /// Whether the deactivation or deletion succeeded. - pub success: bool, - /// Human-readable summary of the operation result. - pub message: String, - /// Key identifier that was deactivated or deleted. - pub deactivated_kid: String, - /// Whether the key was deleted from storage. - pub deleted: bool, - /// Active key identifiers remaining after the operation. - pub remaining_active_kids: Vec, - /// Error detail when the operation fails. - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -/// Deactivates or deletes an active signing key. -/// -/// # Response contract +/// Returns whether `kid` has the shape of a stored key id, which +/// [`validate_kid_format`] checks. /// -/// Returns `200 OK` with `success: true` on success, `400 Bad Request` for an -/// invalid operator-supplied `kid`, or `500 Internal Server Error` when -/// deactivation fails. Failure responses include `success: false` and a populated -/// `error` field. Like [`handle_rotate_key`] and unlike -/// [`handle_verify_signature`], the error field contains internal detail — this -/// is intentional because this endpoint is auth-gated and operator-facing only. -/// -/// # Errors -/// -/// Returns an error if the request signing settings are missing or JSON parsing fails. -pub fn handle_deactivate_key( - settings: &Settings, - services: &RuntimeServices, - req: Request, -) -> Result, Report> { - let SigningStoreIds { - config_store_id, - secret_store_id, - } = signing_store_ids(settings)?; - - let body = request_body_bytes(req.into_body(), "deactivate-key")?; - enforce_max_body_size(&body, ADMIN_MAX_BODY_BYTES, "deactivate-key")?; - let deactivate_req: DeactivateKeyRequest = - serde_json::from_slice(&body).change_context(TrustedServerError::Configuration { - message: "invalid JSON request body".into(), - })?; - - let manager = KeyRotationManager::new(config_store_id, secret_store_id); - - // Use the looser format validation here so legacy KIDs created under earlier - // validation rules (e.g. digit- or uppercase-leading) can still be - // deactivated or deleted. The stricter validate_kid only gates new key - // creation/rotation. - let result = validate_kid_format(&deactivate_req.kid).and_then(|()| { - if deactivate_req.delete { - manager.delete_key(services, &deactivate_req.kid) - } else { - manager.deactivate_key(services, &deactivate_req.kid) - } - }); - - match result { - Ok(()) => { - let remaining_keys = manager.list_active_keys(services).unwrap_or_else(|e| { - log::warn!("failed to list active keys after deactivation: {}", e); - vec![] - }); - - let response = DeactivateKeyResponse { - success: true, - message: if deactivate_req.delete { - "Key deleted successfully".to_string() - } else { - "Key deactivated successfully".to_string() - }, - deactivated_kid: deactivate_req.kid, - deleted: deactivate_req.delete, - remaining_active_kids: remaining_keys, - error: None, - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(StatusCode::OK, response_json)) - } - Err(e) => { - let status = e.current_context().status_code(); - let response = DeactivateKeyResponse { - success: false, - message: if deactivate_req.delete { - "Key deletion failed".to_string() - } else { - "Key deactivation failed".to_string() - }, - deactivated_kid: deactivate_req.kid.clone(), - deleted: false, - remaining_active_kids: vec![], - error: Some(format!("{}", e)), - }; - - let response_json = serde_json::to_string(&response).map_err(|e| { - Report::new(TrustedServerError::Configuration { - message: format!("failed to serialize response: {}", e), - }) - })?; - - Ok(json_response(status, response_json)) - } - } +/// Looser than [`kid_is_creatable`], so that a key created under an earlier +/// rule can still be deactivated and deleted. Exposed for the tool that +/// retires keys, which asks before it reads or writes anything. +#[must_use] +pub fn kid_is_well_formed(kid: &str) -> bool { + validate_kid_format(kid).is_ok() } #[cfg(test)] @@ -719,266 +457,6 @@ mod tests { assert!(result.is_err(), "Malformed JSON should error"); } - #[test] - fn test_handle_rotate_key_with_empty_body() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_request(Method::POST, "https://test.com/admin/keys/rotate", None); - - let resp = handle_rotate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when store writes fail" - ); - - let body = response_body_string(resp); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - - assert!( - !response.success, - "should report failure when store writes fail" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_rotate_key_with_custom_kid() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = RotateKeyRequest { - kid: Some("test-custom-key".to_string()), - }; - - let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some(&body_json), - ); - - let resp = handle_rotate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when store writes fail" - ); - - let body = response_body_string(resp); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - - assert!( - !response.success, - "should report failure when store writes fail" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_rotate_key_invalid_json() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some("invalid json"), - ); - - let result = handle_rotate_key(&settings, &noop_services(), req); - assert!(result.is_err(), "Invalid JSON should return error"); - } - - #[test] - fn test_handle_rotate_key_rejects_invalid_kid() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = RotateKeyRequest { - kid: Some("bad,kid".to_string()), - }; - - let body_json = serde_json::to_string(&req_body).expect("should serialize rotate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some(&body_json), - ); - - let resp = handle_rotate_key(&settings, &noop_services(), req) - .expect("should return a response for invalid kid"); - - assert_eq!( - resp.status(), - StatusCode::BAD_REQUEST, - "should reject malformed kid as a bad request" - ); - - let body = response_body_string(resp); - let response: RotateKeyResponse = - serde_json::from_str(&body).expect("should deserialize rotate response"); - - assert!( - !response.success, - "should report failure when supplied kid is invalid" - ); - assert!( - response - .error - .as_deref() - .is_some_and(|error| error.contains("kid must contain only")), - "should explain the kid character restrictions" - ); - } - - #[test] - fn test_handle_deactivate_key_request() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = DeactivateKeyRequest { - kid: "test-old-key".to_string(), - delete: false, - }; - - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when active-kids cannot be read" - ); - - let body = response_body_string(resp); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - - assert!( - !response.success, - "should report failure when store reads fail" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_deactivate_key_with_delete() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = DeactivateKeyRequest { - kid: "test-old-key".to_string(), - delete: true, - }; - - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); - - assert_eq!( - resp.status(), - StatusCode::INTERNAL_SERVER_ERROR, - "should return 500 when active-kids cannot be read" - ); - - let body = response_body_string(resp); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - - assert!( - !response.success, - "should report failure when store reads fail" - ); - assert!( - !response.deleted, - "should not report deletion when the operation failed" - ); - assert!( - response.error.is_some(), - "should include error detail in failure response" - ); - } - - #[test] - fn test_handle_deactivate_key_invalid_json() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some("invalid json"), - ); - - let result = handle_deactivate_key(&settings, &noop_services(), req); - assert!(result.is_err(), "Invalid JSON should return error"); - } - - #[test] - fn test_handle_deactivate_key_rejects_invalid_kid() { - let settings = crate::test_support::tests::create_test_settings(); - - let req_body = DeactivateKeyRequest { - kid: "bad kid".to_string(), - delete: false, - }; - - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response for invalid kid"); - - assert_eq!( - resp.status(), - StatusCode::BAD_REQUEST, - "should reject malformed kid as a bad request" - ); - - let body = response_body_string(resp); - let response: DeactivateKeyResponse = - serde_json::from_str(&body).expect("should deserialize deactivate response"); - - assert!( - !response.success, - "should report failure when supplied kid is invalid" - ); - assert!( - response - .error - .as_deref() - .is_some_and(|error| error.contains("kid must contain only")), - "should explain the kid character restrictions" - ); - } - #[test] fn verify_signature_rejects_oversized_body() { let settings = crate::test_support::tests::create_test_settings(); @@ -1018,63 +496,6 @@ mod tests { ); } - #[test] - fn rotate_key_rejects_oversized_body() { - let settings = crate::test_support::tests::create_test_settings(); - let oversized = "x".repeat(ADMIN_MAX_BODY_BYTES + 1); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/rotate", - Some(&oversized), - ); - let err = handle_rotate_key(&settings, &noop_services(), req) - .expect_err("should reject oversized body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::PAYLOAD_TOO_LARGE, - "should return 413 for rotate-key body over limit" - ); - } - - #[test] - fn rotate_key_rejects_streaming_body() { - let settings = crate::test_support::tests::create_test_settings(); - let req = build_streaming_request(Method::POST, "https://test.com/admin/keys/rotate"); - let err = handle_rotate_key(&settings, &noop_services(), req) - .expect_err("should reject streaming rotate body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::BAD_REQUEST, - "should return 400 for streaming rotate body" - ); - assert!( - matches!( - err.current_context(), - TrustedServerError::BadRequest { message } - if message == "rotate-key request body must be buffered, not streamed" - ), - "should explain that rotate bodies must be buffered" - ); - } - - #[test] - fn deactivate_key_rejects_oversized_body() { - let settings = crate::test_support::tests::create_test_settings(); - let oversized = "x".repeat(ADMIN_MAX_BODY_BYTES + 1); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&oversized), - ); - let err = handle_deactivate_key(&settings, &noop_services(), req) - .expect_err("should reject oversized body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::PAYLOAD_TOO_LARGE, - "should return 413 for deactivate-key body over limit" - ); - } - #[test] fn validate_kid_accepts_allowed_operator_supplied_ids() { validate_kid("azAZ09-_.:").expect("should accept allowed kid characters"); @@ -1115,7 +536,7 @@ mod tests { fn validate_kid_rejects_non_lowercase_leading_ids() { // The Spin variable encoder requires a lowercase ASCII leading character, // so uppercase- and punctuation-leading KIDs that pass validate_kid_format - // must be rejected on the create/rotate path to avoid a runtime 5xx on Spin. + // must be rejected for a new key, which Spin could not store. for kid in &["KidA", "_kid", "-kid", ".kid", ":kid"] { assert!( validate_kid(kid).is_err(), @@ -1127,63 +548,32 @@ mod tests { } #[test] - fn validate_kid_format_allows_digit_leading_for_legacy_removal() { - // Deactivation/deletion uses validate_kid_format so digit-leading KIDs - // created under earlier rules stay removable, while the stricter - // validate_kid still blocks them on the create/rotate path. + fn validate_kid_format_allows_digit_leading_ids() { + // The leading-letter rule belongs to validate_kid alone, so the + // structural check accepts a digit-leading kid that a new key may not + // take. for kid in &["2026-key", "0abc", "9xyz", "1-key"] { validate_kid_format(kid) - .unwrap_or_else(|e| panic!("format check should accept legacy kid {kid}: {e:?}")); + .unwrap_or_else(|e| panic!("format check should accept kid {kid}: {e:?}")); assert!( validate_kid(kid).is_err(), - "create/rotate validation must still reject digit-leading kid: {kid}" + "new-key validation must still reject digit-leading kid: {kid}" ); } } #[test] - fn deactivate_allows_legacy_digit_leading_kid_past_validation() { - // A digit-leading legacy kid must pass validation and reach storage - // (which fails with the noop test stores -> 500) rather than being - // rejected up front as a 400 bad request. - let settings = crate::test_support::tests::create_test_settings(); - let req_body = DeactivateKeyRequest { - kid: "1legacy-key".to_string(), - delete: true, - }; - let body_json = - serde_json::to_string(&req_body).expect("should serialize deactivate request"); - let req = build_request( - Method::POST, - "https://test.com/admin/keys/deactivate", - Some(&body_json), - ); - - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response for a legacy digit-leading kid"); - - assert_ne!( - resp.status(), - StatusCode::BAD_REQUEST, - "legacy digit-leading kid must not be rejected as a bad request on deactivation" - ); - } - - #[test] - fn test_rotate_key_request_deserialization() { - let json = r#"{"kid":"custom-key"}"#; - let req: RotateKeyRequest = - serde_json::from_str(json).expect("should deserialize rotate key request"); - assert_eq!(req.kid, Some("custom-key".to_string())); - } - - #[test] - fn test_deactivate_key_request_deserialization() { - let json = r#"{"kid":"old-key","delete":true}"#; - let req: DeactivateKeyRequest = - serde_json::from_str(json).expect("should deserialize deactivate key request"); - assert_eq!(req.kid, "old-key"); - assert!(req.delete); + fn a_well_formed_kid_need_not_be_one_a_new_key_may_take() { + for kid in ["2026-key", "KidA", "-kid", "ts-2026-01-01"] { + assert!(kid_is_well_formed(kid), "should accept {kid}"); + } + for kid in ["2026-key", "KidA", "-kid"] { + assert!(!kid_is_creatable(kid), "should not create {kid}"); + } + let too_long = "a".repeat(129); + for kid in ["", "kid-a,kid-b", "kid a", "kid/a", &too_long] { + assert!(!kid_is_well_formed(kid), "should refuse {kid:?}"); + } } #[test] diff --git a/crates/trusted-server-core/src/request_signing/rotation.rs b/crates/trusted-server-core/src/request_signing/rotation.rs index 2d7222b705..e971ff6d2f 100644 --- a/crates/trusted-server-core/src/request_signing/rotation.rs +++ b/crates/trusted-server-core/src/request_signing/rotation.rs @@ -167,8 +167,7 @@ impl KeyRotationManager { signing_key: &SigningKey, ) -> Result<(), Report> { // The platform secret-store write interface is string-based, so signing - // keys are persisted as base64 text. The Fastly adapter applies its own - // transport-level base64 encoding when calling the management API. + // keys are persisted as base64 text. let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); services diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 46ff4bf5b4..301b8f799f 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -24,6 +24,17 @@ use crate::settings::Settings; #[derive(Clone, Copy, Debug)] pub struct TerminalPrivateResponse; +/// Request marker meaning the response is personalized to this request and +/// must not be shared through any cache or template. +/// +/// Any integration may set it from a request filter, and core acts on it +/// without knowing which integration asked. Whenever it is present the request +/// keeps to the origin path rather than a shared template. For an HTML +/// document core also buffers the full body, and it enforces private caching +/// on the HTML response that body produces. +#[derive(Debug, Clone, Copy)] +pub struct PersonalizedResponse; + const INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL: &str = "private, max-age=60"; #[derive(Debug, Clone, Copy)] @@ -84,7 +95,10 @@ pub fn enforce_private_no_store(response: &mut Response) { } /// Marks a Trusted Server response as terminal-private and applies its cache policy. -pub(crate) fn enforce_terminal_private_cache_privacy(response: &mut Response) { +/// +/// Public so a response an integration crate synthesizes for one reader is +/// kept out of shared caches the same way. +pub fn enforce_terminal_private_cache_privacy(response: &mut Response) { enforce_private_no_store(response); response.extensions_mut().insert(TerminalPrivateResponse); } @@ -144,6 +158,12 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { .remove::(); } +/// Headers a response must leave with, whatever `settings.response_headers` +/// say. [`apply_response_headers_with_cache_privacy`] sets them again after +/// the operator's headers. +#[derive(Debug, Clone)] +pub struct RequiredHeaders(pub HeaderMap); + /// Applies operator-configured `settings.response_headers` with cookie-privacy /// hardening. /// @@ -188,6 +208,12 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: response.headers_mut().insert(header_name, header_value); } + if let Some(RequiredHeaders(required)) = response.extensions_mut().remove::() { + for (name, value) in &required { + response.headers_mut().insert(name.clone(), value.clone()); + } + } + enforce_uncacheable_cache_privacy(response); // Operator headers can themselves introduce Set-Cookie (alongside public @@ -208,18 +234,19 @@ mod tests { fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" origin_url = "https://origin.test-publisher.example.com" proxy_secret = "unit-test-proxy-secret" + [geo] + assume_single_jurisdiction = true + [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" "#, ) @@ -610,4 +637,30 @@ mod tests { "operator headers should still apply to cacheable responses" ); } + + #[test] + fn a_required_header_outlasts_an_operator_header_of_the_same_name() { + let settings = settings_with_response_headers(&[ + ("Content-Security-Policy", "default-src 'self'"), + ("X-Frame-Options", "SAMEORIGIN"), + ]); + let mut response = response_builder() + .header( + header::CONTENT_SECURITY_POLICY, + "sandbox; default-src 'none'", + ) + .header(header::X_FRAME_OPTIONS, "DENY") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + let required = response.headers().clone(); + response.extensions_mut().insert(RequiredHeaders(required)); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response.headers()[header::CONTENT_SECURITY_POLICY], + "sandbox; default-src 'none'" + ); + assert_eq!(response.headers()[header::X_FRAME_OPTIONS], "DENY"); + } } diff --git a/crates/trusted-server-core/src/robots_txt/contributor.rs b/crates/trusted-server-core/src/robots_txt/contributor.rs new file mode 100644 index 0000000000..c4ee61a4ea --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/contributor.rs @@ -0,0 +1,82 @@ +//! The extension point a module uses to add rules to `/robots.txt`. + +use core::time::Duration; + +use error_stack::Report; + +use super::Contribution; +use crate::error::TrustedServerError; +use crate::module_context::ModuleCall; +use crate::permissions::PermissionSet; + +/// The `[robots-txt] modules` entry that refuses every crawler. +/// +/// Core's own, and it names no vendor. Selecting it makes the file the refusal +/// and nothing else, whatever else is selected, and puts +/// [`REFUSE_ALL_ROBOTS_TAG`](super::REFUSE_ALL_ROBOTS_TAG) on every response. +pub const REFUSE_ALL: &str = "refuse_all"; + +/// The `[robots-txt] modules` entry that allows every crawler everywhere. +/// +/// Core's own, so permission is something the document says rather than +/// something that follows from a lookup coming back empty. +pub const ALLOW_ALL: &str = "allow_all"; + +/// Something that adds rules to `/robots.txt`. +/// +/// A module supplies one through +/// [`IntegrationRegistrationBuilder::with_robots_txt_contributor`](crate::integrations::IntegrationRegistrationBuilder::with_robots_txt_contributor), +/// and `[robots-txt] modules` selects it by the name it was declared under. +/// Core holds what a contributor returns, assembles the file from every +/// selected contributor in the order `modules` names them, and writes the +/// bytes. The contributor never does, so it cannot produce a malformed file. +/// +/// `Send + Sync` so the registry can share it between requests, while the +/// future it returns is pinned to one thread, because the host SDKs produce +/// `!Send` futures on wasm32. +#[async_trait::async_trait(?Send)] +pub trait RobotsTxtContributor: Send + Sync + core::fmt::Debug { + /// The longest one answer may be held before this contributor is asked + /// again. + /// + /// A ceiling, not a schedule. Each contributor's answer is held and + /// refreshed on its own, and the file is assembled from whatever each one + /// holds at the time, so two contributors never have to agree on anything. + fn refresh(&self) -> Duration; + + /// A value that changes whenever this contributor's answer would, such as a + /// hash of its settings. + /// + /// It keys the held answers, so an answer given for different settings is + /// never served for these. Core hashes it into the key an answer is held + /// under, so nothing of it is written to the store. + fn fingerprint(&self) -> String; + + /// The permissions this contributor declares, which decide whether a + /// gated value is passed to it. None by default. + fn required_permissions(&self) -> PermissionSet { + PermissionSet::none() + } + + /// This contributor's rules. + /// + /// Core holds the answer for the publisher and serves it to every + /// crawler. So the call carries the request the answer was asked for, + /// being its method, path, host and scheme without its query, and the + /// request's services, and nothing about who asked. An implementation + /// hands its own function to [`ModuleCall::inject`], naming what it + /// needs. Its answer must be the same for every request the publisher + /// receives, because it is served for all of them. + /// + /// A contribution with no groups is an answer, and is served as one. An + /// error is a failure, and core serves this contributor's held copy + /// instead. With no held copy the whole file fails closed with `503`. + /// + /// # Errors + /// + /// When this contributor cannot give its rules now. + async fn contribute( + &self, + call: ModuleCall<'_>, + ) -> Result>; +} diff --git a/crates/trusted-server-core/src/robots_txt/document.rs b/crates/trusted-server-core/src/robots_txt/document.rs new file mode 100644 index 0000000000..8973e61595 --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/document.rs @@ -0,0 +1,540 @@ +//! The file served at `/robots.txt`, held as a document rather than as text. +//! +//! Contributors hand core groups of records, and core writes every byte. So a +//! contributor cannot produce a malformed file, and the rules that have to hold +//! whatever a contributor says are applied here, in one place. +//! +//! 1. A refusal cannot be loosened. When `refuse_all` is selected the file is +//! the refusal and nothing else, see [`refusal`]. +//! 2. A path in `always_allow` stays open to every crawler a contribution +//! refuses the whole site to, see [`assemble`]. +//! 3. The publisher's own text and the `Sitemap` line go where the publisher +//! put them, around what the contributors gave. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use super::RobotsTxtConfig; + +/// One record in a group, in the order it was given. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Record { + /// A path the group's crawlers may fetch. + Allow(String), + /// A path the group's crawlers may not fetch. + Disallow(String), + /// A line of explanation, without its leading `#`. + Comment(String), + /// Any other `name: value` record, kept where it was given, a Terms + /// Document Locator for example. + Other { + /// The record's name, as it was written. + name: String, + /// Its value. + value: String, + }, +} + +impl Record { + fn line(&self) -> String { + match self { + Self::Allow(path) => format!("Allow: {}", one_line(path)), + Self::Disallow(path) => format!("Disallow: {}", one_line(path)), + Self::Comment(text) => format!("#{}", one_line(text)), + Self::Other { name, value } => format!("{}: {}", one_line(name), one_line(value)), + } + } + + /// `Disallow: /`, or the same rule written with wildcards, such as + /// `Disallow: /*`, which RFC 9309 section 2.2.3 reads as matching every path. + fn refuses_the_whole_site(&self) -> bool { + matches!(self, Self::Disallow(path) + if one_line(path).strip_prefix('/').is_some_and(|rest| rest.bytes().all(|b| b == b'*'))) + } +} + +/// `value` with every CR and LF taken out, so that it cannot end its line and +/// start a record of its own. +fn one_line(value: &str) -> Cow<'_, str> { + if value.contains(['\r', '\n']) { + Cow::Owned(value.replace(['\r', '\n'], "")) + } else { + Cow::Borrowed(value) + } +} + +/// The crawlers a group is for, and the records that apply to them. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Group { + /// Each written on its own `User-Agent` line. An empty name is kept as it + /// was given, because the file is written back as it was read. + pub user_agents: Vec, + /// The group's records, in order. + pub records: Vec, +} + +/// What one contributor adds to the file. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Contribution { + /// Records before the first group, such as a comment introducing the rules. + pub preamble: Vec, + /// The groups, in order. + pub groups: Vec, +} + +impl Contribution { + /// A contribution read from `robots.txt` text, keeping every group, record + /// and comment in the order it came. + /// + /// A `User-Agent` line after a group's rules starts the next group, and + /// consecutive `User-Agent` lines name the crawlers of one group, as RFC + /// 9309 section 2.1 describes. A `#` starts a comment wherever it is, as + /// section 2.2 allows, and the comment is kept as a record of its own after + /// the rule it followed. Blank lines and lines with no `:` carry nothing + /// and are passed over. A record before the first group is kept in the + /// preamble rather than dropped, so nothing given is lost. + #[must_use] + pub fn parse(text: &str) -> Self { + let mut contribution = Self::default(); + let mut current: Option = None; + for raw in text.split('\n') { + let (rule, comment) = match raw.split_once('#') { + Some((rule, comment)) => (rule.trim(), Some(comment.trim_end())), + None => (raw.trim(), None), + }; + if let Some((name, value)) = rule.split_once(':') { + let (name, value) = (name.trim(), value.trim().to_owned()); + match name.to_ascii_lowercase().as_str() { + "user-agent" => { + if current.as_ref().is_some_and(|group| { + group + .records + .iter() + .any(|record| !matches!(record, Record::Comment(_))) + }) && let Some(finished) = current.take() + { + contribution.groups.push(finished); + } + current + .get_or_insert_with(Group::default) + .user_agents + .push(value); + } + "allow" => push(&mut contribution, &mut current, Record::Allow(value)), + "disallow" => push(&mut contribution, &mut current, Record::Disallow(value)), + _ => push( + &mut contribution, + &mut current, + Record::Other { + name: name.to_owned(), + value, + }, + ), + } + } + if let Some(comment) = comment { + push( + &mut contribution, + &mut current, + Record::Comment(comment.to_owned()), + ); + } + } + if let Some(group) = current { + contribution.groups.push(group); + } + contribution + } + + /// Whether this contribution has anything to write. + #[must_use] + pub fn is_empty(&self) -> bool { + self.preamble.is_empty() && self.groups.is_empty() + } + + /// The contribution as text, with a blank line between its parts and none + /// at either end. + fn write(&self) -> String { + let mut parts: Vec = Vec::new(); + if !self.preamble.is_empty() { + parts.push( + self.preamble + .iter() + .map(Record::line) + .collect::>() + .join("\n"), + ); + } + for group in &self.groups { + let mut lines: Vec = group + .user_agents + .iter() + .map(|agent| format!("User-Agent: {}", one_line(agent))) + .collect(); + lines.extend(group.records.iter().map(Record::line)); + parts.push(lines.join("\n")); + } + parts.join("\n\n") + } +} + +fn push(contribution: &mut Contribution, current: &mut Option, record: Record) { + match current { + Some(group) => group.records.push(record), + None => contribution.preamble.push(record), + } +} + +/// What `allow_all` contributes, which is every crawler allowed everywhere. +/// +/// Permission the document says, rather than permission that follows from a +/// lookup coming back empty. +#[must_use] +pub fn allow_all() -> Contribution { + Contribution { + preamble: Vec::new(), + groups: vec![Group { + user_agents: vec!["*".to_owned()], + records: vec![Record::Allow("/".to_owned())], + }], + } +} + +/// The file `refuse_all` serves, which refuses every crawler except for the +/// paths in `always_allow`. +/// +/// Nothing a contributor gives, and none of the publisher's own text, is +/// added, because a refusal cannot be loosened. `Allow` beats `Disallow` on +/// the longest match under RFC 9309 section 2.2.2, so each allowance lets that +/// one path through. +#[must_use] +pub fn refusal(always_allow: &[String]) -> String { + let mut file = String::from("User-agent: *"); + for path in always_allow { + file.push_str("\nAllow: "); + file.push_str(path); + } + file.push_str("\nDisallow: /\n"); + file +} + +/// The whole file, being the publisher's text above, each contribution in the +/// order `modules` names it, the group that keeps `always_allow` open, the +/// `Sitemap` line, and the publisher's text below. +#[must_use] +pub fn assemble(config: &RobotsTxtConfig, contributions: &[Contribution]) -> String { + let mut sections: Vec = Vec::new(); + if let Some(top) = config + .top_text + .as_deref() + .filter(|text| !text.trim().is_empty()) + { + sections.push(top.trim_end().to_owned()); + } + for contribution in contributions.iter().filter(|c| !c.is_empty()) { + sections.push(contribution.write()); + } + if let Some(segment) = kept_open(contributions, &config.always_allow) { + sections.push(segment); + } + if let Some(sitemap) = &config.sitemap { + sections.push(format!("Sitemap: {sitemap}")); + } + if let Some(bottom) = config + .bottom_text + .as_deref() + .filter(|text| !text.trim().is_empty()) + { + sections.push(bottom.trim_end().to_owned()); + } + let mut file = sections.join("\n\n"); + file.push('\n'); + file +} + +/// Every crawler the contributions refuse the whole site to, by the name on +/// its `User-Agent` line, in order and without repeats. `*` is included, so +/// the paths stay open to the crawlers no group names. A group with no name +/// is passed over, because there is nothing to address a rule to. +fn refused_entirely(contributions: &[Contribution]) -> Vec<&str> { + let mut names: Vec<&str> = Vec::new(); + for group in contributions.iter().flat_map(|c| c.groups.iter()) { + if !group.records.iter().any(Record::refuses_the_whole_site) { + continue; + } + for name in &group.user_agents { + if !name.is_empty() && !names.contains(&name.as_str()) { + names.push(name); + } + } + } + names +} + +/// A group of core's own that keeps each path in `always_allow` open to the +/// crawlers the contributions refuse the whole site to, or `None` when there is +/// nothing to keep open. +/// +/// **The contributions are never edited.** RFC 9309 section 2.2.1 has a +/// crawler combine every group that matches it, and section 2.2.2 gives the +/// match with the most octets, so a second group naming the same crawlers with +/// an `Allow` for the path beats `Disallow: /` for that path alone. +/// +/// It goes below the contributions. A crawler that obeys only the first group +/// it finds then still meets `Disallow: /` first, so the worst it can do is +/// fail to read the one page. Written above, it would read the whole site. +fn kept_open(contributions: &[Contribution], paths: &[String]) -> Option { + let names = refused_entirely(contributions); + if names.is_empty() || paths.is_empty() { + return None; + } + let mut lines: Vec = names + .iter() + .map(|name| format!("User-Agent: {}", one_line(name))) + .collect(); + lines.extend(paths.iter().map(|path| format!("Allow: {path}"))); + Some(lines.join("\n")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> RobotsTxtConfig { + RobotsTxtConfig { + modules: vec!["a_contributor".to_owned()], + ..RobotsTxtConfig::refuse_all() + } + } + + /// Text a contributor might read from somewhere else, with the shapes the + /// file has to survive: a record inside a group, crawlers sharing a group, + /// and a group with no name. + const GIVEN: &str = "User-Agent: ExampleBot\nDisallow: /\n\nUser-Agent: \nDisallow: /\n\n\ + User-Agent: OtherBot\nDisallow: /private/\n\nUser-Agent: ThirdBot\n\ + User-Agent: FourthBot\nDisallow: /\n\nUser-Agent: *\n\ + TDL: https://terms.example.com/socw/2.txt\nAllow: /"; + + #[test] + fn a_contribution_is_written_back_exactly_as_it_was_read() { + assert_eq!(Contribution::parse(GIVEN).write(), GIVEN); + } + + #[test] + fn crawlers_named_together_share_one_group_and_a_record_keeps_its_place() { + let parsed = Contribution::parse(GIVEN); + + assert_eq!(parsed.groups.len(), 5); + assert_eq!(parsed.groups[3].user_agents, vec!["ThirdBot", "FourthBot"]); + assert_eq!( + parsed.groups[4].records, + vec![ + Record::Other { + name: "TDL".to_owned(), + value: "https://terms.example.com/socw/2.txt".to_owned(), + }, + Record::Allow("/".to_owned()), + ] + ); + } + + #[test] + fn blank_lines_at_either_end_are_not_part_of_the_file() { + let parsed = Contribution::parse("\nUser-Agent: ExampleBot\nDisallow: /\n\n\n"); + + assert_eq!(parsed.write(), "User-Agent: ExampleBot\nDisallow: /"); + } + + #[test] + fn a_comment_and_a_record_before_any_group_are_kept() { + let parsed = Contribution::parse("# rules for crawlers\nUser-Agent: *\nAllow: /"); + + assert_eq!( + parsed.preamble, + vec![Record::Comment(" rules for crawlers".to_owned())] + ); + assert_eq!( + parsed.write(), + "# rules for crawlers\n\nUser-Agent: *\nAllow: /" + ); + } + + #[test] + fn the_publishers_text_and_the_sitemap_go_around_the_contributions() { + let mut settings = config(); + settings.top_text = Some("# top".to_owned()); + settings.sitemap = Some("https://publisher.example.com/sitemap.xml".to_owned()); + settings.bottom_text = Some("# bottom".to_owned()); + + assert_eq!( + assemble( + &settings, + &[Contribution::parse("User-agent: *\nAllow: /\n")] + ), + "# top\n\nUser-Agent: *\nAllow: /\n\n\ + Sitemap: https://publisher.example.com/sitemap.xml\n\n# bottom\n", + ); + } + + /// The page a refused crawler is sent to stays open, by a group of core's + /// own below the contributions, which are not edited. + #[test] + fn an_always_allowed_path_is_kept_open_to_every_crawler_refused_the_site() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + let file = assemble(&settings, &[Contribution::parse(GIVEN)]); + + assert!( + file.starts_with(GIVEN), + "the contribution is not edited: {file}" + ); + assert!( + file.ends_with( + "\n\nUser-Agent: ExampleBot\nUser-Agent: ThirdBot\nUser-Agent: FourthBot\n\ + Allow: /ai-notice/\n" + ), + "the group naming every crawler refused the whole site goes last: {file}" + ); + } + + /// The path is kept open to `*` as it is to a crawler refused by name, so + /// the crawlers no group names can still read it. + #[test] + fn an_always_allowed_path_is_kept_open_when_every_crawler_is_refused() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + assert_eq!( + assemble( + &settings, + &[Contribution::parse("User-Agent: *\nDisallow: /")] + ), + "User-Agent: *\nDisallow: /\n\nUser-Agent: *\nAllow: /ai-notice/\n", + "a group for `*` below the contribution keeps the path open" + ); + } + + #[test] + fn a_wildcard_that_matches_every_path_refuses_the_whole_site() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + assert_eq!( + assemble( + &settings, + &[Contribution::parse("User-Agent: ExampleBot\nDisallow: /*")] + ), + "User-Agent: ExampleBot\nDisallow: /*\n\n\ + User-Agent: ExampleBot\nAllow: /ai-notice/\n", + "`Disallow: /*` is the same rule as `Disallow: /`" + ); + } + + #[test] + fn a_comment_after_a_rule_does_not_hide_a_whole_site_refusal() { + let mut settings = config(); + settings.always_allow = vec!["/ads.txt".to_owned()]; + let parsed = Contribution::parse("User-Agent: *\nDisallow: / # temporarily closed"); + + assert_eq!( + parsed.groups[0].records, + vec![ + Record::Disallow("/".to_owned()), + Record::Comment(" temporarily closed".to_owned()), + ] + ); + assert!( + assemble(&settings, &[parsed]).ends_with("\n\nUser-Agent: *\nAllow: /ads.txt\n"), + "the path stays open to the crawlers refused the site" + ); + } + + #[test] + fn a_comment_between_user_agent_lines_keeps_their_group() { + let mut settings = config(); + settings.always_allow = vec!["/ads.txt".to_owned()]; + let parsed = Contribution::parse( + "User-Agent: ExampleBot\n# the same rules for its sibling\nUser-Agent: OtherBot\nDisallow: /", + ); + + assert_eq!(parsed.groups.len(), 1); + assert_eq!(parsed.groups[0].user_agents, vec!["ExampleBot", "OtherBot"]); + assert!( + assemble(&settings, &[parsed]) + .ends_with("\n\nUser-Agent: ExampleBot\nUser-Agent: OtherBot\nAllow: /ads.txt\n"), + "every crawler of the group is kept open to the path" + ); + } + + #[test] + fn a_line_break_in_a_value_cannot_start_a_record() { + let contribution = Contribution { + preamble: vec![Record::Comment(" note\nAllow: /".to_owned())], + groups: vec![Group { + user_agents: vec!["ExampleBot\r\nUser-Agent: *".to_owned()], + records: vec![ + Record::Disallow("/x\nUser-Agent: *\nAllow: /".to_owned()), + Record::Other { + name: "TDL\r".to_owned(), + value: "a\rAllow: /".to_owned(), + }, + ], + }], + }; + + assert_eq!( + assemble(&config(), &[contribution]), + "# noteAllow: /\n\n\ + User-Agent: ExampleBotUser-Agent: *\n\ + Disallow: /xUser-Agent: *Allow: /\n\ + TDL: aAllow: /\n", + "every value stays on the line its record starts" + ); + } + + #[test] + fn nothing_is_kept_open_when_nothing_is_refused() { + let mut settings = config(); + settings.always_allow = vec!["/ai-notice/".to_owned()]; + + assert_eq!( + assemble(&settings, &[allow_all()]), + "User-Agent: *\nAllow: /\n" + ); + } + + #[test] + fn contributions_appear_in_the_order_given() { + let first = Contribution::parse("User-Agent: FirstBot\nDisallow: /"); + let second = Contribution::parse("User-Agent: SecondBot\nDisallow: /"); + + assert_eq!( + assemble(&config(), &[first, second]), + "User-Agent: FirstBot\nDisallow: /\n\nUser-Agent: SecondBot\nDisallow: /\n" + ); + } + + #[test] + fn an_empty_contribution_leaves_no_gap() { + assert_eq!( + assemble(&config(), &[Contribution::default(), allow_all()]), + "User-Agent: *\nAllow: /\n" + ); + } + + #[test] + fn refusing_with_no_allowances_refuses_everything() { + assert_eq!(refusal(&[]), "User-agent: *\nDisallow: /\n"); + } + + #[test] + fn each_allowance_is_written_in_order_ahead_of_the_refusal() { + assert_eq!( + refusal(&["/ads.txt".to_owned(), "/ai-notice/".to_owned()]), + "User-agent: *\nAllow: /ads.txt\nAllow: /ai-notice/\nDisallow: /\n" + ); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/holding.rs b/crates/trusted-server-core/src/robots_txt/holding.rs new file mode 100644 index 0000000000..f0ffeddcbf --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/holding.rs @@ -0,0 +1,943 @@ +//! Each contributor's answer, held between requests by the loading cache. +//! +//! A contributor's answer is loaded through +//! [`RuntimeServices::load_cached`], so the contributor is asked once however +//! many requests want its rules at the same moment, and the copy in the host's +//! shared store answers every instance until the contributor's refresh is due. +//! +//! The last answer a contributor gave is also kept apart from the cache, for +//! far longer than any refresh, and read only when the contributor fails. It is +//! then served past its age, because yesterday's rules are a better answer than +//! none. It is kept in the host's value cache when the host supplies one, and +//! in the key-value store otherwise, because a host that starts an instance +//! for each request may give its requests a value cache and no key-value store. +//! A value cache may drop it sooner, so on such a host a copy past its age is +//! likely to be there rather than certain, and a key-value store that cannot +//! give an entry a lifetime keeps it until it is deleted there. + +use core::time::Duration; +use std::sync::Arc; + +use error_stack::Report; +use sha2::{Digest as _, Sha256}; + +use super::{Contribution, RobotsTxtContributor}; +use crate::error::TrustedServerError; +use crate::module_context::{ModuleContext, ResolvedRequest}; +use crate::platform::{ + CacheLifetimes, CacheSpec, KvError, LoadError, LoadedValue, Loader, RuntimeServices, StoreKey, +}; +use crate::settings::Settings; + +/// The cache contributors' answers are held in. +pub(crate) const CACHE: &str = crate::settings::ROBOTS_TXT_CACHE; + +/// Where each contributor's last answer is kept apart from the cache. +const KEPT: &str = "robots-txt-kept"; + +/// How long a contributor's last answer is kept, far past any refresh, so it +/// is there to serve when the contributor cannot answer. +const KEPT_LIFETIME: Duration = Duration::from_secs(30 * 24 * 3600); + +/// The lifetimes `[cache.robots-txt]` leaves unset. A contributor's own +/// refresh ends each copy sooner than the store lifetime. +const DEFAULT_LIFETIMES: CacheLifetimes = CacheLifetimes { + memory: Duration::from_secs(60), + store: Duration::from_secs(7 * 24 * 3600), + store_idle: Duration::from_secs(30 * 24 * 3600), +}; + +/// The cache contributors' answers are held in, with the lifetimes +/// `[cache.robots-txt]` sets. +/// +/// Its loads run in the caller, because a contributor may block the thread it +/// runs on while it asks its service. +pub(crate) fn cache_spec(settings: &Settings) -> CacheSpec { + CacheSpec { + name: CACHE, + lifetimes: settings.cache.robots_txt.lifetimes(DEFAULT_LIFETIMES), + memory_entries: 64, + detachable: false, + } +} + +/// The contribution of `contributor`, selected as `id`, from a copy young +/// enough to serve, and from the contributor otherwise, which is asked for +/// `request`. +/// +/// # Errors +/// +/// When the contributor fails and no answer of its is kept. A kept answer is +/// returned past its age rather than an error. +pub(crate) async fn contribution_for( + id: &'static str, + contributor: &Arc, + request: &ResolvedRequest, + services: &RuntimeServices, + settings: &Settings, +) -> Result> { + let prefix = services.server_key_prefix(); + let held = held_key(id, &contributor.fingerprint()); + let key = StoreKey::new(prefix, CACHE, held.clone()); + let kept = StoreKey::new(prefix, KEPT, held); + let spec = cache_spec(settings); + let ask = || { + load( + Ask { + id, + contributor: Arc::clone(contributor), + request: request.clone(), + services: services.clone(), + }, + kept.clone(), + ) + }; + let loaded = match held_answer(services, &spec, &key, ask()).await { + Err(Held::Unreadable(error)) => { + // A copy that no longer reads would answer every request until + // it ends, so it is removed and the contributor asked again. + log::warn!( + "Asking `{id}` again for robots.txt rules, the held copy is unreadable: {error}" + ); + services + .loading_cache_for(CACHE) + .remove(services, &spec, &key) + .await; + held_answer(services, &spec, &key, ask()).await + } + loaded => loaded, + }; + match loaded { + Ok(contribution) => Ok(contribution), + Err(Held::Failed(error) | Held::Unreadable(error)) => { + match kept_answer(services, &kept).await { + Some(contribution) => { + log::warn!( + "Serving the kept robots.txt rules of `{id}` past their age: {error}" + ); + Ok(contribution) + } + None => Err(Report::new(TrustedServerError::Integration { + integration: id.to_owned(), + message: format!("the robots.txt contributor did not answer: {error}"), + })), + } + } + } +} + +/// Why no held or fresh answer was read. +enum Held { + /// The load failed. + Failed(LoadError), + /// The cache answered with a copy that does not read as a contribution. + Unreadable(LoadError), +} + +/// The answer the cache holds for `key`, loaded with `load` when it holds +/// none. +async fn held_answer( + services: &RuntimeServices, + spec: &CacheSpec, + key: &StoreKey, + load: Loader, +) -> Result { + let bytes = services + .load_cached(spec, key, load) + .await + .map_err(Held::Failed)?; + serde_json::from_slice::(&bytes).map_err(|error| { + Held::Unreadable(LoadError::new(format!( + "the held answer is unreadable: {error}" + ))) + }) +} + +/// What a load owns to ask one contributor, because it may outlive the +/// request that started it. +struct Ask { + /// The name the contributor was declared under. + id: &'static str, + contributor: Arc, + request: ResolvedRequest, + services: RuntimeServices, +} + +/// Asks the contributor, keeps its answer apart for when it next fails, and +/// gives the answer the contributor's refresh as its lifetime. +fn load(ask: Ask, kept: StoreKey) -> Loader { + Box::new(move || { + Box::pin(async move { + let Ask { + id, + contributor, + request, + services, + } = ask; + let context = ModuleContext::new(request.view()).with_services(&services); + let contribution = contributor + .contribute(context.call(id, contributor.required_permissions())) + .await + .map_err(|report| LoadError::new(format!("{report:?}")))?; + let bytes = serde_json::to_vec(&contribution) + .map_err(|error| LoadError::new(format!("the answer cannot be held: {error}")))?; + keep_answer(&services, &kept, &bytes).await; + Ok(LoadedValue::new(bytes).with_lifetime(contributor.refresh())) + }) + }) +} + +/// The key one contributor's answer for one fingerprint is held under, a hash +/// so the fingerprint's contents, whatever they are, never appear in a store. +fn held_key(id: &str, fingerprint: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(id.as_bytes()); + hasher.update(b"\n"); + hasher.update(fingerprint.as_bytes()); + let mut key = String::with_capacity(64); + for byte in hasher.finalize() { + key.push_str(&format!("{byte:02x}")); + } + key +} + +/// The answer kept for when the contributor fails, from the host's value +/// cache when it supplies one and from the key-value store otherwise. +/// +/// Nothing read, for whatever reason, is no kept answer, and the failure that +/// led here is reported instead. +async fn kept_answer(services: &RuntimeServices, kept: &StoreKey) -> Option { + let key = kept.to_string(); + let bytes = match services.value_cache() { + Some(cache) => cache.get(&key).await?, + None => match services.kv_store().get_bytes(&key).await { + Ok(bytes) => bytes?, + Err(error) => { + log::debug!("No kept robots.txt answer could be read: {error}"); + return None; + } + }, + }; + serde_json::from_slice(&bytes) + .inspect_err(|error| log::debug!("The kept robots.txt answer is unreadable: {error}")) + .ok() +} + +/// Keeps an answer for when the contributor next fails, in the host's value +/// cache when it supplies one and in the key-value store otherwise. +async fn keep_answer(services: &RuntimeServices, kept: &StoreKey, bytes: &[u8]) { + let key = kept.to_string(); + let bytes = bytes::Bytes::copy_from_slice(bytes); + if let Some(cache) = services.value_cache() { + cache.put(&key, bytes, KEPT_LIFETIME).await; + return; + } + let store = services.kv_store(); + let written = match store + .put_bytes_with_ttl(&key, bytes.clone(), KEPT_LIFETIME) + .await + { + // A store that cannot give an entry a lifetime keeps it until it is + // replaced, so the answer is still there when the contributor fails. + Err(KvError::Unsupported { .. }) => store.put_bytes(&key, bytes).await, + written => written, + }; + if let Err(error) = written { + log::debug!("The robots.txt answer was not kept for a later failure: {error}"); + } +} + +#[cfg(test)] +pub(crate) mod tests { + use std::sync::Mutex; + + use edgezero_core::body::Body as EdgeBody; + + use super::*; + use crate::evidence::RequestInfo; + use crate::module_context::{ModuleCall, ModuleRequest}; + use crate::platform::test_support::{MemoryLoadingCache, StubHttpClient}; + use crate::prefix::test_support::{assert_keys_follow_the_publisher, trusted_server_for}; + + /// A contributor whose answers a test decides, counting how often it is + /// asked. + #[derive(Debug)] + pub(crate) struct Scripted { + answers: Mutex>>, + pub(crate) asked: Mutex, + pub(crate) refresh: Duration, + pub(crate) fingerprint: Mutex, + } + + impl Scripted { + pub(crate) fn new(answers: Vec>) -> Self { + Self { + answers: Mutex::new(answers), + asked: Mutex::new(0), + refresh: Duration::from_secs(3600), + fingerprint: Mutex::new("first settings".to_owned()), + } + } + + pub(crate) fn times_asked(&self) -> usize { + *self.asked.lock().expect("should lock the count") + } + } + + #[async_trait::async_trait(?Send)] + impl RobotsTxtContributor for Scripted { + fn refresh(&self) -> Duration { + self.refresh + } + + fn fingerprint(&self) -> String { + self.fingerprint.lock().expect("should lock").clone() + } + + async fn contribute( + &self, + _call: ModuleCall<'_>, + ) -> Result> { + *self.asked.lock().expect("should lock the count") += 1; + let next = self + .answers + .lock() + .expect("should lock the answers") + .remove(0); + next.map_err(|message| Report::new(TrustedServerError::Proxy { message })) + } + } + + pub(crate) fn rules(text: &str) -> Contribution { + Contribution::parse(text) + } + + /// A request for `/robots.txt` as a crawler makes it, with a query of its + /// own. + pub(crate) fn robots_request() -> http::Request { + http::Request::builder() + .uri("https://www.publisher.example/robots.txt?crawler=one") + .header(http::header::HOST, "www.publisher.example") + .body(EdgeBody::empty()) + .expect("should build the request") + } + + /// The request the contributors are asked for, as the handler resolves it. + pub(crate) fn request() -> ResolvedRequest { + ResolvedRequest::of(&robots_request(), &crate::platform::ClientInfo::default()) + .without_query() + } + + /// A contributor whose rules name what it was handed, so a test can read + /// back the request, whether the reader's evidence reached it, and + /// whether the services did. + #[derive(Debug)] + pub(crate) struct Echoing; + + impl Echoing { + fn rules( + &self, + request: ModuleRequest<'_>, + evidence: Option<&dyn RequestInfo>, + services: Option<&RuntimeServices>, + ) -> Contribution { + rules(&format!( + "User-Agent: EchoBot\nDisallow: /{}{}?{}/{}/{}", + request.host(), + request.path(), + request.query(), + if evidence.is_some() { + "evidence" + } else { + "no-evidence" + }, + if services.is_some() { + "services" + } else { + "no-services" + }, + )) + } + } + + #[async_trait::async_trait(?Send)] + impl RobotsTxtContributor for Echoing { + fn refresh(&self) -> Duration { + Duration::from_secs(60) + } + + fn fingerprint(&self) -> String { + "echo".to_owned() + } + + async fn contribute( + &self, + call: ModuleCall<'_>, + ) -> Result> { + Ok(call.inject(self, Self::rules)?) + } + } + + /// What [`Echoing`] answers for [`request`]. + pub(crate) const ECHOED: &str = + "User-Agent: EchoBot\nDisallow: /www.publisher.example/robots.txt?/no-evidence/services"; + + /// The answer is shared by every crawler, so the contributor is handed + /// the request it was asked for and its services, and nothing about who + /// asked, not even the query. + #[tokio::test] + async fn the_contributor_is_asked_for_the_request_and_not_who_asked() { + let echoing: Arc = Arc::new(Echoing); + let services = services_with(Arc::default(), Arc::default()); + + let answer = contribution_for("echo", &echoing, &request(), &services, &settings()) + .await + .expect("should ask the contributor"); + + assert_eq!(answer, rules(ECHOED)); + } + + /// A store held in memory for the life of one test, so that what one + /// instance wrote is what the next one reads. + #[derive(Debug, Default)] + pub(crate) struct MemoryKvStore { + pub(crate) entries: Mutex>, + /// The lifetime each entry was last written with. + pub(crate) lifetimes: Mutex>, + /// Whether a write with a lifetime is refused, as it is by a store + /// that has none to give. + pub(crate) refuses_lifetimes: bool, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformKvStore for MemoryKvStore { + async fn get_bytes( + &self, + key: &str, + ) -> Result, crate::platform::KvError> { + Ok(self + .entries + .lock() + .expect("should lock the test store") + .get(key) + .cloned()) + } + + async fn put_bytes( + &self, + key: &str, + value: bytes::Bytes, + ) -> Result<(), crate::platform::KvError> { + self.entries + .lock() + .expect("should lock the test store") + .insert(key.to_owned(), value); + Ok(()) + } + + async fn put_bytes_with_ttl( + &self, + key: &str, + value: bytes::Bytes, + ttl: Duration, + ) -> Result<(), crate::platform::KvError> { + if self.refuses_lifetimes { + return Err(KvError::Unsupported { + operation: "put_bytes_with_ttl".to_owned(), + }); + } + self.lifetimes + .lock() + .expect("should lock the test store") + .insert(key.to_owned(), ttl); + self.put_bytes(key, value).await + } + + async fn delete(&self, key: &str) -> Result<(), crate::platform::KvError> { + self.entries + .lock() + .expect("should lock the test store") + .remove(key); + Ok(()) + } + + async fn list_keys_page( + &self, + _prefix: &str, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + Ok(edgezero_core::key_value_store::KvPage::default()) + } + } + + /// Services keeping answers in `cache` and kept answers in `store`, as one + /// instance of a host with no value cache, for the publisher [`settings`] + /// describes. + pub(crate) fn services_with( + cache: Arc, + store: Arc, + ) -> RuntimeServices { + crate::platform::test_support::build_services_with_http_client(Arc::new( + StubHttpClient::new(), + )) + .with_kv_store(store) + .with_loading_cache(cache) + .with_prefixes(trusted_server_for("www.publisher.example")) + } + + pub(crate) fn settings() -> Settings { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.domain = "www.publisher.example".to_owned(); + settings + } + + fn contributor(scripted: &Arc) -> Arc { + Arc::clone(scripted) as Arc + } + + #[tokio::test] + async fn a_contributor_is_asked_once_and_its_answer_is_held_after_that() { + let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))])); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services_with(Arc::clone(&cache), Arc::default()); + + let first = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let second = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve the held copy"); + + assert_eq!(first, second); + assert_eq!(scripted.times_asked(), 1); + assert!( + cache.calls().iter().all(|(name, _)| *name == CACHE), + "every call should be to the robots.txt cache" + ); + } + + #[tokio::test] + async fn the_held_answer_is_keyed_by_the_publisher() { + let scripted = Arc::new(Scripted::new(vec![ + Ok(rules("User-Agent: OneBot\nDisallow: /")), + Ok(rules("User-Agent: OtherBot\nDisallow: /")), + ])); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services_with(Arc::clone(&cache), Arc::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should answer the first publisher"); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services.with_prefixes(trusted_server_for("other.example")), + &settings(), + ) + .await + .expect("should answer the second publisher"); + + assert_eq!(answer, rules("User-Agent: OtherBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2, "each publisher is asked for"); + let keys = cache.keys(); + assert!( + keys[0].starts_with("ts.other.example/robots-txt/"), + "{keys:?}" + ); + assert!( + keys[1].starts_with("ts.publisher.example/robots-txt/"), + "{keys:?}" + ); + } + + /// Both the held answer and the one kept for a failure are keyed by the + /// publisher the request's services carry. + #[tokio::test] + async fn the_held_and_kept_keys_follow_the_publisher() { + let keys_for = |domain: &str| { + let scripted = Arc::new(Scripted::new(vec![Ok(rules("User-Agent: *\nAllow: /"))])); + let cache = Arc::new(MemoryLoadingCache::default()); + let store = Arc::new(MemoryKvStore::default()); + let services = services_with(Arc::clone(&cache), Arc::clone(&store)) + .with_prefixes(trusted_server_for(domain)); + futures::executor::block_on(contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + )) + .expect("should ask the contributor"); + let kept: Vec = store + .entries + .lock() + .expect("should lock the test store") + .keys() + .cloned() + .collect(); + format!("{:?} {kept:?}", cache.keys()) + }; + + assert_keys_follow_the_publisher(keys_for); + } + + #[tokio::test] + async fn changed_settings_are_not_answered_from_the_old_copy() { + let scripted = Arc::new(Scripted::new(vec![ + Ok(rules("User-Agent: FirstBot\nDisallow: /")), + Ok(rules("User-Agent: SecondBot\nDisallow: /")), + ])); + let services = services_with(Arc::default(), Arc::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask for the first settings"); + *scripted.fingerprint.lock().expect("should lock") = "second settings".to_owned(); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask again for the new settings"); + + assert_eq!(answer, rules("User-Agent: SecondBot\nDisallow: /")); + } + + #[tokio::test] + async fn the_kept_answer_is_served_when_the_contributor_fails() { + let mut scripted = Scripted::new(vec![ + Ok(rules("User-Agent: HeldBot\nDisallow: /")), + Err("down".to_owned()), + ]); + // A refresh of zero gives each answer no time in the cache, so the + // second request asks again on any machine at any moment. + scripted.refresh = Duration::ZERO; + let scripted = Arc::new(scripted); + let services = services_with(Arc::default(), Arc::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve yesterday's rules"); + + assert_eq!(answer, rules("User-Agent: HeldBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2, "should have asked again first"); + } + + #[tokio::test] + async fn the_kept_answer_outlives_the_instance_that_kept_it() { + let scripted = Arc::new(Scripted::new(vec![ + Ok(rules("User-Agent: KeptBot\nDisallow: /")), + Err("down".to_owned()), + ])); + let store = Arc::new(MemoryKvStore::default()); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services_with(Arc::default(), Arc::clone(&store)), + &settings(), + ) + .await + .expect("should ask the contributor"); + // A second instance, with a cache of its own and the same store. + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services_with(Arc::default(), Arc::clone(&store)), + &settings(), + ) + .await + .expect("should serve the kept answer"); + + assert_eq!(answer, rules("User-Agent: KeptBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2); + let keys: Vec = store + .entries + .lock() + .expect("should lock the test store") + .keys() + .cloned() + .collect(); + assert_eq!(keys.len(), 1, "{keys:?}"); + assert!( + keys[0].starts_with("ts.publisher.example/robots-txt-kept/"), + "{keys:?}" + ); + let lifetimes = store.lifetimes.lock().expect("should lock the test store"); + assert_eq!( + lifetimes.values().copied().collect::>(), + vec![KEPT_LIFETIME], + "the answer should be kept far longer than any refresh" + ); + } + + /// A store with no lifetime to give still keeps the answer, so it is + /// there to serve when the contributor fails. + #[tokio::test] + async fn a_store_without_lifetimes_still_keeps_the_answer() { + let mut scripted = Scripted::new(vec![ + Ok(rules("User-Agent: OneBot\nDisallow: /")), + Err("cannot answer".to_owned()), + ]); + scripted.refresh = Duration::ZERO; + let scripted = Arc::new(scripted); + let store = Arc::new(MemoryKvStore { + refuses_lifetimes: true, + ..MemoryKvStore::default() + }); + let services = services_with(Arc::default(), Arc::clone(&store)); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let later = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve the kept answer"); + + assert_eq!(later, rules("User-Agent: OneBot\nDisallow: /")); + assert!( + store + .lifetimes + .lock() + .expect("should lock the test store") + .is_empty(), + "the entry was written without a lifetime" + ); + } + + #[tokio::test] + async fn with_nothing_kept_a_failure_is_an_error_and_is_not_held() { + let scripted = Arc::new(Scripted::new(vec![ + Err("down".to_owned()), + Ok(rules("User-Agent: LateBot\nDisallow: /")), + ])); + let services = services_with(Arc::default(), Arc::default()); + + assert!( + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings() + ) + .await + .is_err() + ); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("the next request should ask again"); + + assert_eq!(answer, rules("User-Agent: LateBot\nDisallow: /")); + assert_eq!(scripted.times_asked(), 2); + } + + /// A held copy that no longer reads is removed and the contributor asked + /// again, rather than every request failing until the copy ends. + #[tokio::test] + async fn an_unreadable_held_copy_is_removed_and_the_contributor_asked_again() { + let scripted = Arc::new(Scripted::new(vec![Ok(rules( + "User-Agent: FreshBot\nDisallow: /", + ))])); + let cache = Arc::new(MemoryLoadingCache::default()); + let services = services_with(Arc::clone(&cache), Arc::default()); + let settings = settings(); + let key = StoreKey::new( + services.server_key_prefix(), + CACHE, + held_key("a", &scripted.fingerprint()), + ); + services + .load_cached( + &cache_spec(&settings), + &key, + Box::new(|| { + Box::pin(async { + Ok(LoadedValue::new(bytes::Bytes::from_static( + b"not a contribution", + ))) + }) + }), + ) + .await + .expect("should hold the unreadable copy"); + + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings, + ) + .await + .expect("should ask the contributor again"); + let again = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings, + ) + .await + .expect("should serve the fresh copy"); + + assert_eq!(answer, rules("User-Agent: FreshBot\nDisallow: /")); + assert_eq!(again, answer); + assert_eq!(scripted.times_asked(), 1, "the fresh answer should be held"); + } + + /// A value cache held in memory for the life of one test. + #[derive(Debug, Default)] + struct MemoryValueCache { + entries: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformValueCache for MemoryValueCache { + async fn get(&self, key: &str) -> Option { + self.entries + .lock() + .expect("should lock the test cache") + .get(key) + .cloned() + } + + async fn put(&self, key: &str, value: bytes::Bytes, _lifetime: Duration) { + self.entries + .lock() + .expect("should lock the test cache") + .insert(key.to_owned(), value); + } + } + + /// A host that gives its requests a value cache and no key-value store + /// keeps the answer in the value cache, and serves it from there when the + /// contributor fails. + #[tokio::test] + async fn the_kept_answer_is_in_the_value_cache_where_the_host_has_one() { + let mut scripted = Scripted::new(vec![ + Ok(rules("User-Agent: CachedBot\nDisallow: /")), + Err("down".to_owned()), + ]); + scripted.refresh = Duration::ZERO; + let scripted = Arc::new(scripted); + let value_cache = Arc::new(MemoryValueCache::default()); + let services = crate::platform::RuntimeServices::builder() + .config_store(Arc::new(crate::platform::test_support::NoopConfigStore)) + .secret_store(Arc::new(crate::platform::test_support::NoopSecretStore)) + .kv_store(Arc::new(crate::platform::UnavailableKvStore)) + .backend(Arc::new(crate::platform::test_support::NoopBackend)) + .http_client(Arc::new(StubHttpClient::new())) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .value_cache(Arc::clone(&value_cache) as Arc) + .loading_caches(Arc::new(crate::platform::LoadingCaches::every(Arc::new( + MemoryLoadingCache::default(), + )))) + .client_info(crate::platform::ClientInfo::default()) + .prefixes(trusted_server_for("www.publisher.example")) + .build(); + + contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should ask the contributor"); + let answer = contribution_for( + "a", + &contributor(&scripted), + &request(), + &services, + &settings(), + ) + .await + .expect("should serve the kept answer from the value cache"); + + assert_eq!(answer, rules("User-Agent: CachedBot\nDisallow: /")); + assert_eq!( + value_cache + .entries + .lock() + .expect("should lock the test cache") + .len(), + 1, + "the answer should be kept in the value cache" + ); + } + + #[test] + fn the_key_carries_nothing_of_the_fingerprint() { + let key = held_key("a", "license=secret-value"); + + assert_eq!(key.len(), 64); + assert!(!key.contains("secret"), "{key}"); + assert_ne!(key, held_key("b", "license=secret-value")); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/middleware.rs b/crates/trusted-server-core/src/robots_txt/middleware.rs new file mode 100644 index 0000000000..a51bc37c5c --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/middleware.rs @@ -0,0 +1,228 @@ +//! `/robots.txt` as a middleware of core's own, named `robots_txt` in a +//! `[[fetch]]` entry on the file's path. Its preparation makes the file from +//! the selected contributors, through their holding, and its action replaces +//! the origin's file with it. Where the origin has no file, or answers with a +//! page, the creating rule runs it on an empty one. + +use std::any::Any; +use std::time::Duration; + +use error_stack::Report; + +use super::{RETRY_AFTER_SECONDS, RobotsTxtConfig, Selected, document, file_for}; +use crate::error::TrustedServerError; +use crate::middleware::{ + Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase, Preparation, Refusal, +}; +use crate::module_context::{ModuleCall, ModuleRequest, ResolvedRequest}; +use crate::platform::RuntimeServices; +use crate::settings::Settings; +use crate::streaming_processor::StreamProcessor; + +/// The name an entry writes, and the integration the registry lists the +/// middleware under. +pub const MODULE: &str = "robots_txt"; + +/// The media type the middleware works on. +pub const MEDIA_TYPE: &str = "text/plain"; + +/// What a request is told when no file can be made. +const UNAVAILABLE: &str = "robots.txt is temporarily unavailable\n"; + +/// The middleware, holding the section it was made for and the contributors +/// the registry resolved for it, in the order `modules` names them. +#[derive(Debug)] +pub struct RobotsTxt { + config: RobotsTxtConfig, + selected: Vec, +} + +impl RobotsTxt { + /// The middleware for `config`, with `selected` the contributors in the + /// order `modules` names them. + #[must_use] + pub(crate) fn new(config: RobotsTxtConfig, selected: Vec) -> Self { + Self { config, selected } + } + + /// The file for the request, which is the refusal alone or every selected + /// contributor's held answer assembled. + async fn file( + &self, + request: ModuleRequest<'_>, + settings: &Settings, + services: &RuntimeServices, + ) -> Result> { + if self.config.refuses_all() { + return Ok(document::refusal(&self.config.always_allow)); + } + // The answers are held and served to every crawler, so the + // contributors are handed the request with nothing one crawler put in + // the address. + let request = ResolvedRequest::from_view(&request).without_query(); + file_for(&self.config, &self.selected, &request, services, settings).await + } +} + +impl Middleware for RobotsTxt { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == MEDIA_TYPE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &MiddlewarePhase::ALL + } + + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + _context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let file = match call.inject(self, Self::file) { + Ok(file) => file.await, + Err(withheld) => Err(Report::new(TrustedServerError::Configuration { + message: format!("robots.txt cannot be made for this request: {withheld:?}"), + })), + }; + match file { + Ok(file) => Ok(Box::new(file) as Box), + Err(report) => { + log::error!("Could not assemble robots.txt: {report:?}"); + Err(Refusal::service_unavailable( + Duration::from_secs(u64::from(RETRY_AFTER_SECONDS)), + UNAVAILABLE, + )) + } + } + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + match context.take_prepared::(MODULE) { + Some(file) => MiddlewareAction { + stream: Some(Box::new(ReplaceBody { file: Some(file) })), + ..MiddlewareAction::pass() + }, + // Nothing prepared means the preparation did not run, which no request can cause. + None => { + log::error!("robots.txt was not prepared, so the origin's file is served as it is"); + MiddlewareAction::pass() + } + } + } +} + +/// Drops the body it is given and writes the file in its place. +struct ReplaceBody { + file: Option, +} + +impl StreamProcessor for ReplaceBody { + fn process_chunk(&mut self, _chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + Ok(if is_last { + self.file.take().unwrap_or_default().into_bytes() + } else { + Vec::new() + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::integrations::IntegrationDocumentState; + use crate::middleware::test_support::context; + use crate::middleware::{MiddlewareChain, Prepared}; + + fn middleware() -> RobotsTxt { + RobotsTxt::new(RobotsTxtConfig::refuse_all(), Vec::new()) + } + + /// What the middleware is told in `phase`, with `prepared` as what its + /// preparation left. + fn context_with<'a>( + phase: MiddlewarePhase, + state: &'a IntegrationDocumentState, + prepared: &'a Prepared, + ) -> MiddlewareContext<'a> { + MiddlewareContext { + prepared, + ..context(phase, state) + } + } + + #[test] + fn the_middleware_works_on_text_in_either_phase() { + let middleware = middleware(); + + assert!(middleware.handles_media_type(MEDIA_TYPE)); + assert!(!middleware.handles_media_type(crate::middleware::HTML_MEDIA_TYPE)); + assert_eq!(middleware.phases(), &MiddlewarePhase::ALL); + assert_eq!(middleware.middleware_id(), MODULE); + } + + #[test] + fn the_prepared_file_replaces_whatever_the_origin_sent() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + prepared.insert( + MiddlewarePhase::Fetch, + MODULE, + Box::new(String::from("User-agent: *\nDisallow: /\n")), + ); + let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + + let mut processor = middleware() + .create(&context) + .stream + .expect("a prepared file should be written as a stream processor"); + let mut output = processor + .process_chunk(b"User-agent: *\nAllow: /\n", false) + .expect("should take the origin's file"); + output.extend( + processor + .process_chunk(b"", true) + .expect("should write the file at the end"), + ); + + assert_eq!(output, b"User-agent: *\nDisallow: /\n"); + } + + #[test] + fn with_nothing_prepared_the_origins_file_is_left_as_it_is() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = context_with(MiddlewarePhase::Fetch, &state, &prepared); + + let action = middleware().create(&context); + + assert!(action.stream.is_none(), "should pass the body through"); + assert!(action.head_inserts.is_empty()); + } + + #[test] + fn a_value_of_another_type_under_the_name_is_not_a_file() { + let state = IntegrationDocumentState::default(); + let prepared = Prepared::default(); + prepared.insert(MiddlewarePhase::Serve, MODULE, Box::new(7_u32)); + let context = context_with(MiddlewarePhase::Serve, &state, &prepared); + + assert!(middleware().create(&context).stream.is_none()); + } + + #[test] + fn a_chain_plans_the_middleware_on_text() { + let chain = MiddlewareChain::new( + MiddlewarePhase::Fetch, + MEDIA_TYPE, + vec![std::sync::Arc::new(middleware()) as std::sync::Arc], + ); + + assert_eq!(chain.ids(), [MODULE]); + } +} diff --git a/crates/trusted-server-core/src/robots_txt/mod.rs b/crates/trusted-server-core/src/robots_txt/mod.rs new file mode 100644 index 0000000000..ea6f030794 --- /dev/null +++ b/crates/trusted-server-core/src/robots_txt/mod.rs @@ -0,0 +1,767 @@ +//! A publisher's `robots.txt`, assembled from the rules the selected +//! contributors give. +//! +//! `[robots-txt] modules` names, in order, what makes the file. Two names are +//! core's own and need nothing else: [`REFUSE_ALL`] refuses every crawler, and +//! [`ALLOW_ALL`] allows every crawler everywhere. Any other name is the name a +//! module declared a [`RobotsTxtContributor`] under. Core never names a +//! module here and knows nothing of what any contributor asks or of whom. +//! +//! # Contributors add rules, core writes the file +//! +//! A contributor returns a [`Contribution`], being groups of records, and core +//! writes every byte in [`document`]. That is what lets these hold whatever a +//! contributor says. +//! +//! 1. A refusal cannot be loosened. With `refuse_all` selected the file is the +//! refusal and nothing else. +//! 2. A path in `always_allow` stays open to every crawler a contribution +//! refuses the whole site to, `*` included. A narrower rule is kept as +//! given, so `Disallow: /ads` still closes off `/ads.txt`. +//! 3. The `Sitemap` line, the publisher's own text, and the `X-Robots-Tag` +//! header on every page agree with the file served. +//! 4. Several contributors combine in the order `modules` gives, and none of +//! them owns the file. +//! 5. A contributor never writes bytes, so it cannot produce a malformed file. +//! 6. A contributor that fails with no held answer fails the whole file closed +//! with `503`, because leaving out a contributor's rules would allow by +//! omission whatever those rules refused. A `404` or an empty file reads to +//! a crawler as permission to crawl everything. +//! +//! # Settings +//! +//! `[robots-txt]` holds `modules` and the document's own settings, being +//! `always_allow`, `sitemap`, `top_text` and `bottom_text`. Settings belonging +//! to one contributor go in its own block, `[robots-txt.]`, which that +//! contributor reads and core does not. + +mod contributor; +pub mod document; +mod holding; +pub mod middleware; + +use std::collections::BTreeMap; +use std::sync::Arc; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::Response; +use http::header::{self, HeaderValue}; +use serde::{Deserialize, Deserializer, Serialize}; +use validator::{Validate, ValidationError}; + +pub use contributor::{ALLOW_ALL, REFUSE_ALL, RobotsTxtContributor}; +pub use document::{Contribution, Group, Record}; + +use crate::error::TrustedServerError; +use crate::module_context::ResolvedRequest; +use crate::platform::RuntimeServices; +use crate::settings::Settings; + +/// The path this module answers. +pub const ROBOTS_TXT_PATH: &str = "/robots.txt"; + +/// How long a crawler is asked to wait when there is no file to give it. +const RETRY_AFTER_SECONDS: u32 = 600; + +/// The `X-Robots-Tag` a response carries when `refuse_all` is selected. +/// +/// A robots.txt asks a crawler not to fetch. This tells one that has already +/// fetched not to index, and it is the only one of the two that works on a page +/// a crawler learned about from somewhere else, or reached before reading +/// robots.txt, or chose to fetch anyway. They do different jobs and a site that +/// is not ready to be seen needs both. +/// +/// Set by the same selection so the two cannot drift apart. +/// +/// `nofollow` as well as `noindex`, so a crawler does not walk on from a page +/// it should not have indexed into the rest of a site that is not ready either. +pub const REFUSE_ALL_ROBOTS_TAG: &str = "noindex, nofollow"; + +/// The type folder of every robots.txt crate, which a name written in +/// `[robots-txt] modules` may leave off. +pub const MODULE_TYPE: &str = "robots-txt"; + +/// Settings for serving `robots.txt`. +/// +/// Mapped from the `[robots-txt]` TOML section. With the section absent the +/// publisher keeps their own file and `/robots.txt` reaches their origin like +/// any other path. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +pub struct RobotsTxtConfig { + /// What makes the file, in order. One name or a list, so + /// `modules = "refuse_all"` and `modules = ["refuse_all"]` are the same. + /// + /// Required, and refused at load when absent or empty. There is no default, + /// because a default would decide for a publisher what crawlers may do. + #[serde(default, deserialize_with = "one_or_many")] + pub modules: Vec, + + /// Paths kept open to every crawler refused the whole site, the page a + /// refused crawler is sent to for example. A narrower rule, such as + /// `Disallow: /ads`, still closes off `/ads.txt`. `/robots.txt` needs no + /// entry, because RFC 9309 section 2.2.2 says it is implicitly allowed. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[validate(custom(function = validate_always_allow))] + pub always_allow: Vec, + + /// The site's sitemap, added as a `Sitemap:` line. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(url)] + pub sitemap: Option, + + /// The publisher's own text, placed above the contributors' rules. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_text: Option, + + /// The publisher's own text, placed below the contributors' rules. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bottom_text: Option, + + /// Each contributor's own settings, `[robots-txt.]`, keyed by its + /// name. + /// + /// Read by that contributor and by nothing in core, which only checks that + /// each is a block and belongs to a contributor `modules` names. + #[serde(flatten)] + pub contributors: BTreeMap, +} + +fn one_or_many<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum OneOrMany { + One(String), + Many(Vec), + } + Ok(match OneOrMany::deserialize(deserializer)? { + OneOrMany::One(name) => vec![name], + OneOrMany::Many(names) => names, + }) +} + +fn validate_always_allow(paths: &[String]) -> Result<(), ValidationError> { + // A path is written into the file as it stands, so it must be a path and + // must not be able to start a second line. + if paths + .iter() + .any(|path| !path.starts_with('/') || path.chars().any(char::is_whitespace)) + { + return Err(ValidationError::new("robots_txt_always_allow_not_a_path")); + } + Ok(()) +} + +impl RobotsTxtConfig { + /// The configuration that refuses every crawler with no exception. + /// + /// The starting point for a publisher who wants the refusal with + /// allowances of their own. Needs no contributor, no key and no network. + #[must_use] + pub fn refuse_all() -> Self { + Self { + modules: vec![REFUSE_ALL.to_owned()], + always_allow: Vec::new(), + sitemap: None, + top_text: None, + bottom_text: None, + contributors: BTreeMap::new(), + } + } + + /// Whether `modules` names `id`, by its full name or by its short form, + /// whichever of the two `id` and the name in `modules` are written in. + #[must_use] + pub fn selects(&self, id: &str) -> bool { + self.modules.iter().any(|selected| { + crate::module_name::resolve(MODULE_TYPE, selected, &[id]).is_some() + || crate::module_name::short_form(MODULE_TYPE, selected) == id + }) + } + + /// Whether the file is the refusal, which it is whenever `refuse_all` is + /// selected, whatever else is. + #[must_use] + pub fn refuses_all(&self) -> bool { + self.selects(REFUSE_ALL) + } + + /// The `X-Robots-Tag` every response should carry, or `None` when the + /// publisher's crawler policy places no blanket rule on pages. + #[must_use] + pub fn response_robots_tag(&self) -> Option<&'static str> { + self.refuses_all().then_some(REFUSE_ALL_ROBOTS_TAG) + } + + /// The settings block of the contributor named `id`, when there is one. + #[must_use] + pub fn contributor_settings(&self, id: &str) -> Option<&serde_json::Value> { + self.contributors.get(id).or_else(|| { + self.contributors + .get(crate::module_name::short_form(MODULE_TYPE, id)) + }) + } + + /// Checks what the type cannot. + /// + /// # Errors + /// + /// When `modules` names nothing or names one thing twice, when a key is + /// not a setting, and when a contributor's block belongs to one `modules` + /// does not name. + pub fn prepare_runtime(&self) -> Result<(), Report> { + if self.modules.is_empty() { + return Err(Report::new(configuration( + "[robots-txt] names no module. Name what makes the file, in order: \ + `refuse_all` to refuse every crawler, `allow_all` to allow every crawler, \ + or the name of each module whose rules make the file. There is no default, \ + because a default would decide for the publisher what crawlers may do." + .to_owned(), + ))); + } + // Compared on the short form, so the full name and the short form of + // one module count as naming it twice. + for (index, name) in self.modules.iter().enumerate() { + let short_name = crate::module_name::short_form(MODULE_TYPE, name); + if self.modules[..index] + .iter() + .any(|earlier| crate::module_name::short_form(MODULE_TYPE, earlier) == short_name) + { + return Err(Report::new(configuration(format!( + "[robots-txt] modules names `{name}` twice." + )))); + } + } + for (key, value) in &self.contributors { + if !value.is_object() { + return Err(Report::new(configuration(format!( + "robots-txt.{key} is not a setting. [robots-txt] holds modules, \ + always_allow, sitemap, top_text, bottom_text, and a block for each \ + contributor that has settings of its own." + )))); + } + if !self.selects(key) { + return Err(Report::new(configuration(format!( + "[robots-txt.{key}] configures a contributor that `modules` does not \ + name, so nothing would read it. Add `{key}` to modules or remove the \ + block." + )))); + } + } + Ok(()) + } +} + +fn configuration(message: String) -> TrustedServerError { + TrustedServerError::Configuration { message } +} + +/// Puts [`REFUSE_ALL_ROBOTS_TAG`] on an outgoing response when `refuse_all` is +/// selected, in place of any `X-Robots-Tag` it carried, and does nothing +/// otherwise. Called once where an adapter finalizes a response. +pub fn apply_response_robots_tag(settings: &Settings, response: &mut Response) { + if settings + .robots_txt + .as_ref() + .and_then(RobotsTxtConfig::response_robots_tag) + .is_none() + { + return; + } + response.headers_mut().insert( + header::HeaderName::from_static("x-robots-tag"), + HeaderValue::from_static(REFUSE_ALL_ROBOTS_TAG), + ); +} + +/// One entry of `modules` as the registry resolved it: the name a +/// contributor was declared under and the contributor, or `allow_all` and +/// none. +pub(crate) type Selected = (&'static str, Option>); + +/// The entries of `config.modules` resolved against `contributors`, the +/// contributors a deployment runs by the names they were declared under. +/// +/// # Errors +/// +/// When an entry names a contributor `contributors` does not carry, which +/// the registry refuses before it asks, so this is a wiring fault. +pub(crate) fn select( + config: &RobotsTxtConfig, + contributors: &[(&'static str, Arc)], +) -> Result, Report> { + let mut selected = Vec::with_capacity(config.modules.len()); + for written in &config.modules { + if written == REFUSE_ALL { + // The refusal needs no contributor, and the middleware answers it + // before asking any. + continue; + } + if written == ALLOW_ALL { + selected.push((ALLOW_ALL, None)); + continue; + } + let names: Vec<&str> = contributors.iter().map(|(name, _)| *name).collect(); + let Some((declared, contributor)) = + crate::module_name::resolve(MODULE_TYPE, written, &names) + .and_then(|declared| contributors.iter().find(|(name, _)| *name == declared)) + else { + return Err(Report::new(configuration(format!( + "[robots-txt] modules names `{written}`, which no module in this deployment supplies" + )))); + }; + selected.push((declared, Some(Arc::clone(contributor)))); + } + Ok(selected) +} + +/// The file for `config`, from the contributions of `selected` in order, +/// each asked for `request`. +pub(crate) async fn file_for( + config: &RobotsTxtConfig, + selected: &[Selected], + request: &ResolvedRequest, + services: &RuntimeServices, + settings: &Settings, +) -> Result> { + let mut contributions = Vec::with_capacity(selected.len()); + for (id, contributor) in selected { + contributions.push(match contributor { + None => document::allow_all(), + Some(contributor) => { + holding::contribution_for(id, contributor, request, services, settings).await? + } + }); + } + Ok(document::assemble(config, &contributions)) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::holding::tests::{ + ECHOED, Echoing, MemoryKvStore, Scripted, request, robots_request, rules, services_with, + settings, + }; + use super::*; + use crate::integrations::{IntegrationBuilder, IntegrationRegistration, IntegrationRegistry}; + use crate::middleware::Middleware; + use crate::module_context::{ModuleContext, ModuleRequest}; + use http::StatusCode; + + fn parsed(document: serde_json::Value) -> RobotsTxtConfig { + serde_json::from_value(document).expect("should read the robots.txt settings") + } + + #[test] + fn a_single_module_reads_as_a_list_of_one() { + let config = parsed(serde_json::json!({ "modules": "refuse_all" })); + + assert_eq!(config.modules, vec!["refuse_all"]); + assert!(config.refuses_all()); + config + .prepare_runtime() + .expect("a single module written as a string must load"); + } + + #[test] + fn a_list_of_modules_keeps_its_order() { + let config = parsed(serde_json::json!({ + "modules": ["a_module", "allow_all"], + "a_module": { "anything": "the module reads" }, + })); + + assert_eq!(config.modules, vec!["a_module", "allow_all"]); + config.prepare_runtime().expect("should load"); + } + + #[test] + fn no_module_is_refused_rather_than_defaulted() { + let error = parsed(serde_json::json!({ "sitemap": "https://example.com/sitemap.xml" })) + .prepare_runtime() + .expect_err("there is no default"); + let message = format!("{error:?}"); + + assert!(message.contains("refuse_all"), "{message}"); + assert!(message.contains("allow_all"), "{message}"); + } + + #[test] + fn an_empty_list_is_refused_the_same_way() { + assert!( + parsed(serde_json::json!({ "modules": [] })) + .prepare_runtime() + .is_err() + ); + } + + #[test] + fn a_name_given_twice_is_refused() { + assert!( + parsed(serde_json::json!({ "modules": ["allow_all", "allow_all"] })) + .prepare_runtime() + .is_err() + ); + } + + /// A section may write a contributor's full name or leave the type folder + /// off, so both spellings select the block, and naming both is naming the + /// contributor twice. + #[test] + fn a_contributor_named_in_full_selects_its_block() { + parsed(serde_json::json!({ + "modules": "robots-txt.a_module", + "a_module": { "anything": "the module reads" }, + })) + .prepare_runtime() + .expect("the full name should select the block written at the short name"); + + let error = parsed(serde_json::json!({ + "modules": ["a_module", "robots-txt.a_module"], + "a_module": { "anything": "the module reads" }, + })) + .prepare_runtime() + .expect_err("one contributor named in two spellings is named twice"); + assert!(format!("{error:?}").contains("twice"), "{error:?}"); + } + + #[test] + fn a_key_that_is_not_a_setting_is_refused() { + let message = format!( + "{:?}", + parsed(serde_json::json!({ "modules": "allow_all", "sitmap": "a typo" })) + .prepare_runtime() + .expect_err("a misspelled setting must not load") + ); + + assert!(message.contains("is not a setting"), "{message}"); + } + + #[test] + fn a_block_for_a_contributor_not_selected_is_refused() { + let message = format!( + "{:?}", + parsed(serde_json::json!({ "modules": "allow_all", "a_module": {} })) + .prepare_runtime() + .expect_err("nothing would read the block") + ); + + assert!(message.contains("does not name"), "{message}"); + } + + #[test] + fn a_path_kept_open_must_be_a_path_on_one_line() { + for path in ["ads.txt", "/ads.txt\nDisallow: /", "/a path"] { + let config = parsed(serde_json::json!({ + "modules": "refuse_all", + "always_allow": [path], + })); + + assert!(config.validate().is_err(), "should refuse {path:?}"); + } + parsed(serde_json::json!({ "modules": "refuse_all", "always_allow": ["/ads.txt"] })) + .validate() + .expect("should accept a path"); + } + + #[test] + fn refusing_sets_the_response_header_and_a_contributor_does_not() { + assert_eq!( + RobotsTxtConfig::refuse_all().response_robots_tag(), + Some(REFUSE_ALL_ROBOTS_TAG) + ); + assert_eq!( + parsed(serde_json::json!({ "modules": "allow_all" })).response_robots_tag(), + None + ); + } + + #[test] + fn the_header_replaces_one_the_origin_set() { + let mut settings = settings(); + settings.robots_txt = Some(RobotsTxtConfig::refuse_all()); + let mut response = Response::builder() + .header("x-robots-tag", "all") + .body(EdgeBody::empty()) + .expect("should build a response"); + + apply_response_robots_tag(&settings, &mut response); + + let tags: Vec<_> = response.headers().get_all("x-robots-tag").iter().collect(); + assert_eq!(tags, vec![REFUSE_ALL_ROBOTS_TAG]); + } + + #[test] + fn a_response_is_left_alone_when_nothing_refuses_every_crawler() { + let mut allowing = settings(); + allowing.robots_txt = Some(parsed(serde_json::json!({ "modules": "allow_all" }))); + let mut without_a_section = settings(); + without_a_section.robots_txt = None; + + for settings in [allowing, without_a_section] { + let mut response = Response::builder() + .header("x-robots-tag", "noarchive") + .body(EdgeBody::empty()) + .expect("should build a response"); + + apply_response_robots_tag(&settings, &mut response); + + let tags: Vec<_> = response.headers().get_all("x-robots-tag").iter().collect(); + assert_eq!(tags, vec!["noarchive"]); + } + } + + #[tokio::test] + async fn contributions_are_assembled_in_the_order_selected() { + let first: Arc = Arc::new(Scripted::new(vec![Ok(rules( + "User-Agent: FirstBot\nDisallow: /", + ))])); + let config = parsed(serde_json::json!({ + "modules": ["first", "allow_all"], + "sitemap": "https://publisher.example.com/sitemap.xml", + })); + + let file = file_for( + &config, + &[("first", Some(first)), (ALLOW_ALL, None)], + &request(), + &services_with(Arc::default(), Arc::default()), + &settings(), + ) + .await + .expect("should assemble the file"); + + assert_eq!( + file, + "User-Agent: FirstBot\nDisallow: /\n\nUser-Agent: *\nAllow: /\n\n\ + Sitemap: https://publisher.example.com/sitemap.xml\n" + ); + } + + /// What the middleware prepares for `settings`, with the services given, + /// through the registry that made it from `[robots-txt]`, as the handler + /// runs it for `/robots.txt` in the fetch phase. + async fn prepared_by( + settings: &Settings, + extra: &[IntegrationBuilder], + services: &RuntimeServices, + ) -> Result, crate::middleware::Refusal> { + use crate::middleware::{MiddlewarePhase, Prepared}; + + let registry = IntegrationRegistry::with_registrations(settings, extra) + .expect("should build the registry"); + let chain = registry.middleware_chain( + &settings.fetch, + MiddlewarePhase::Fetch, + middleware::MEDIA_TYPE, + ROBOTS_TXT_PATH, + ); + assert_eq!( + chain.ids(), + [middleware::MODULE], + "the entry names the middleware" + ); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(settings) + .with_services(services); + let state = crate::integrations::IntegrationDocumentState::default(); + let prepared = Prepared::default(); + let context = crate::middleware::MiddlewareContext { + prepared: &prepared, + ..crate::middleware::test_support::context(MiddlewarePhase::Fetch, &state) + }; + chain.prepare(&module_context, &context).await?; + Ok(prepared.take::(MiddlewarePhase::Fetch, middleware::MODULE)) + } + + /// `settings` with `config` as its `[robots-txt]` section and the entry + /// that places the middleware on the file. + fn settings_placing(config: RobotsTxtConfig) -> Settings { + let mut settings = settings(); + settings.robots_txt = Some(config); + settings.fetch = + crate::middleware::PhaseEntries::new(vec![crate::middleware::PhaseEntry { + media_type: middleware::MEDIA_TYPE.to_owned(), + path: Some(ROBOTS_TXT_PATH.to_owned()), + middleware: vec![middleware::MODULE.to_owned()], + }]); + settings + } + + /// Leaving out the rules of a contributor that failed would allow by + /// omission what those rules refused, so the whole file fails closed. + #[tokio::test] + async fn with_nothing_held_a_failure_refuses_the_request() { + let failing: Arc = + Arc::new(Scripted::new(vec![Err("cannot answer".to_owned())])); + let config = parsed(serde_json::json!({ "modules": ["failing", "allow_all"] })); + let middleware = + middleware::RobotsTxt::new(config, vec![("failing", Some(failing)), (ALLOW_ALL, None)]); + let settings = settings(); + let services = services_with(Arc::default(), Arc::default()); + let request = robots_request(); + let module_context = + ModuleContext::new(ModuleRequest::of(&request, services.client_info())) + .with_settings(&settings) + .with_services(&services); + let state = crate::integrations::IntegrationDocumentState::default(); + let context = crate::middleware::test_support::context( + crate::middleware::MiddlewarePhase::Fetch, + &state, + ); + + let refusal = middleware + .prepare( + module_context.call( + middleware::MODULE, + crate::permissions::PermissionSet::none(), + ), + &context, + ) + .expect("the middleware prepares") + .await + .expect_err("a crawler reads an empty or allow everything file as permission"); + + assert_eq!(refusal.status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + refusal.retry_after, + Some(std::time::Duration::from_secs(u64::from( + RETRY_AFTER_SECONDS + ))) + ); + assert_eq!(refusal.message, "robots.txt is temporarily unavailable\n"); + } + + /// The refusal asks nothing of anything, and dominates whatever else is + /// selected, because a refusal cannot be loosened. + #[tokio::test] + async fn refusing_every_crawler_asks_no_contributor_and_reads_no_store() { + let mut refusing = parsed(serde_json::json!({ "modules": ["allow_all", "refuse_all"] })); + refusing.always_allow = vec!["/ads.txt".to_owned()]; + let settings = settings_placing(refusing); + let store = Arc::new(MemoryKvStore::default()); + + let file = prepared_by( + &settings, + &[], + &services_with(Arc::default(), Arc::clone(&store)), + ) + .await + .expect("the refusal answers without asking") + .expect("the refusal is prepared"); + + assert_eq!( + file, "User-agent: *\nAllow: /ads.txt\nDisallow: /\n", + "the refusal, whatever else is selected" + ); + assert!( + store + .entries + .lock() + .expect("should lock the test store") + .is_empty(), + "nothing is held for a refusal" + ); + } + + /// The registry makes the middleware only for settings that carry the + /// section, so a publisher keeping their own file is left alone. + #[test] + fn the_middleware_exists_only_with_the_section() { + let without = IntegrationRegistry::new(&settings()).expect("should build the registry"); + assert!( + !without.middleware_ids().contains(&middleware::MODULE), + "{:?}", + without.middleware_ids() + ); + + let with = IntegrationRegistry::new(&settings_placing(RobotsTxtConfig::refuse_all())) + .expect("should build the registry"); + assert!( + with.middleware_ids().contains(&middleware::MODULE), + "{:?}", + with.middleware_ids() + ); + } + + /// The id of the builder that registers the echoing contributor. + const ECHO_BUILDER: &str = "robots-txt-echo"; + + /// The module name that builder is selected by. + const ECHO_BUILDER_MODULE: &str = "testing.robots-txt-echo"; + + fn register_echo( + _settings: &Settings, + ) -> Result, Report> { + Ok(Some( + IntegrationRegistration::builder(ECHO_BUILDER) + .with_robots_txt_contributor("robots-txt.echo", Arc::new(Echoing)) + .build(), + )) + } + + fn accept(_settings: &Settings) -> Result> { + Ok(true) + } + + fn settings_selecting_echo() -> Settings { + let mut settings = settings(); + settings.robots_txt = Some(parsed(serde_json::json!({ "modules": ["echo"] }))); + settings + .insert_module_config("testing", ECHO_BUILDER_MODULE, &serde_json::json!({})) + .expect("should select the test builder"); + settings + } + + fn echo_builder() -> IntegrationBuilder { + IntegrationBuilder::new(ECHO_BUILDER, ECHO_BUILDER, register_echo, accept) + .with_module_name(ECHO_BUILDER_MODULE) + } + + /// The middleware asks a contributor for the request it is answering, + /// being its host and path, without the query and without the reader's + /// evidence. + #[tokio::test] + async fn the_middleware_asks_the_contributors_for_the_request_it_answers() { + let mut settings = settings_selecting_echo(); + settings.fetch = settings_placing(RobotsTxtConfig::refuse_all()).fetch; + + let file = prepared_by( + &settings, + &[echo_builder()], + &services_with(Arc::default(), Arc::default()), + ) + .await + .expect("should answer") + .expect("the file is prepared"); + + let config = settings + .robots_txt + .as_ref() + .expect("the settings carry a [robots-txt] section"); + assert_eq!(file, document::assemble(config, &[rules(ECHOED)])); + } + + /// A name nothing supplies is refused where the registry is built, so the + /// file does not fail on the first request for it. + #[test] + fn a_selection_nothing_supplies_is_refused_when_the_registry_is_built() { + let mut settings = settings(); + settings.robots_txt = Some(parsed(serde_json::json!({ "modules": ["nobody"] }))); + + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should refuse a contributor nothing supplies"); + let message = format!("{error:?}"); + + assert!(message.contains("`nobody`"), "{message}"); + assert!(message.contains("refuse_all"), "{message}"); + assert!(message.contains("allow_all"), "{message}"); + } +} diff --git a/crates/trusted-server-core/src/rsc_flight.rs b/crates/trusted-server-core/src/rsc_flight.rs index a9e3a40cec..dac4a4b6ae 100644 --- a/crates/trusted-server-core/src/rsc_flight.rs +++ b/crates/trusted-server-core/src/rsc_flight.rs @@ -29,7 +29,7 @@ enum RowState { /// URLs like `\/\/origin.example.com` (JSON-escaped slashes) will not be rewritten. This is /// acceptable because Flight responses from client-side navigation typically contain plain URLs, /// not doubly-escaped JSON-in-JS content. For inlined `__next_f` data in HTML (which can have -/// escape sequences), the HTML post-processor in `integrations/nextjs/` handles those cases. +/// escape sequences), the HTML post-processor in `crates/framework/nextjs` handles those cases. pub struct RscFlightUrlRewriter { origin_url: String, origin_http_url: Option, diff --git a/crates/trusted-server-core/src/secret_resolution.rs b/crates/trusted-server-core/src/secret_resolution.rs index f362a0b7b2..6bf4afa67f 100644 --- a/crates/trusted-server-core/src/secret_resolution.rs +++ b/crates/trusted-server-core/src/secret_resolution.rs @@ -1,18 +1,91 @@ //! Runtime resolution of `EdgeZero` app-config secret references. //! //! Config blobs carry secret-store key names at rest. This module walks the -//! public `EdgeZero` metadata contract and replaces those names only in the -//! in-memory value used to build runtime [`crate::settings::Settings`]. +//! public `EdgeZero` metadata contract, together with the leaves +//! [`ConfiguredSecretFields`] reads out of the configuration itself, and +//! replaces those names only in the in-memory value used to build runtime +//! [`crate::settings::Settings`]. + +use std::fmt; use edgezero_core::app_config::{AppConfigMeta, SecretField, SecretKind, SecretPathSegment}; use error_stack::Report; use serde_json::Value; use crate::error::TrustedServerError; +use crate::inspect::config::{PathPattern, PathStep}; use crate::platform::{PlatformSecretStore, StoreName}; +/// Where one resolution pass wrote secrets, what it wrote, and every leaf it +/// looked for, which the configuration view masks. +/// +/// Never serialized, and [`fmt::Debug`] prints only how many there are. +#[derive(Clone, Default)] +pub struct ResolvedSecrets { + paths: Vec>, + values: Vec, + patterns: Vec, +} + +impl ResolvedSecrets { + /// The concrete path of every leaf a secret was written into. + pub(crate) fn paths(&self) -> &[Vec] { + &self.paths + } + + /// The secret values written. + pub(crate) fn values(&self) -> &[String] { + &self.values + } + + /// Every secret leaf the pass looked for, found or not. + pub(crate) fn patterns(&self) -> &[PathPattern] { + &self.patterns + } + + /// A record of `paths` and `values`, for a test of what uses one. + #[cfg(test)] + pub(crate) fn recorded(paths: Vec>, values: Vec) -> Self { + Self { + paths, + values, + patterns: Vec::new(), + } + } +} + +impl fmt::Debug for ResolvedSecrets { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ResolvedSecrets") + .field("paths", &self.paths.len()) + .field("values", &self.values.len()) + .field("patterns", &self.patterns.len()) + .finish() + } +} + +/// Secret leaves whose paths come from the configuration rather than the type. +/// +/// [`AppConfigMeta::secret_fields`] is an associated function of the type, so +/// every path it can return is fixed, and `EdgeZero`'s path segments have no +/// way to say "whatever name the operator chose". A configuration can hold +/// secrets under such a name, as an Edge Cookie module block under a label +/// does, and an implementation of this trait finds those by reading the +/// configuration being loaded. +pub trait ConfiguredSecretFields: AppConfigMeta { + /// The secret leaves `data` holds under names the operator chose, each + /// with its full path from the configuration root. + fn configured_secret_fields(data: &Value) -> Vec; +} + /// Resolve all secret references in a serialized Trusted Server app config. /// +/// Both the fixed leaves [`AppConfigMeta::secret_fields`] lists and the ones +/// [`ConfiguredSecretFields::configured_secret_fields`] reads out of `data` +/// are resolved in one pass, so a failure in either leaves the configuration +/// as it was. +/// /// The input is mutated in memory; the verified envelope is never rewritten. /// Secret values are not included in structural or platform errors. /// @@ -21,167 +94,220 @@ use crate::platform::{PlatformSecretStore, StoreName}; /// Returns [`TrustedServerError::Configuration`] when a required path or key /// is malformed, a secret is unavailable, is not valid UTF-8, or resolves to an /// empty value. -pub fn resolve_secret_references( +pub fn resolve_secret_references( data: &mut Value, secret_store: &dyn PlatformSecretStore, default_store_name: &StoreName, ) -> Result<(), Report> { + resolve_secret_references_with::(data, secret_store, default_store_name, Vec::new()) + .map(|_| ()) +} + +/// Resolve all secret references in a serialized Trusted Server app config, +/// together with `extra`, the leaves known only where the configuration is +/// loaded, such as the ones a deployment's modules declare in their own +/// tables. +/// +/// A leaf listed more than once is looked up once, because the second lookup +/// would read the secret itself as the name of a key. What comes back says +/// where the secrets went, which the configuration view masks. +/// +/// # Errors +/// +/// As [`resolve_secret_references`]. +pub fn resolve_secret_references_with( + data: &mut Value, + secret_store: &dyn PlatformSecretStore, + default_store_name: &StoreName, + extra: Vec, +) -> Result> { + let fields = listed_fields::(data, extra); let mut resolved_data = data.clone(); - for field in C::secret_fields() { + let mut resolver = Resolver { + secret_store, + default_store_name, + resolved: ResolvedSecrets::default(), + }; + for field in fields { if matches!(field.kind, SecretKind::StoreRef) { continue; } - resolve_field( - &mut resolved_data, - &field, - &field.path, - "", - secret_store, - default_store_name, - )?; + resolver + .resolved + .patterns + .extend(PathPattern::from_segments(&field.path)); + resolver.resolve_field(&mut resolved_data, &field, &field.path, "", &[])?; } *data = resolved_data; - Ok(()) + Ok(resolver.resolved) } -fn resolve_field( - node: &mut Value, - field: &SecretField, - remaining: &[SecretPathSegment], - rendered_path: &str, - secret_store: &dyn PlatformSecretStore, - default_store_name: &StoreName, -) -> Result<(), Report> { - match remaining.split_first() { - Some((SecretPathSegment::Field(name), [])) => resolve_leaf( - node, - field, - name.as_ref(), - rendered_path, - secret_store, - default_store_name, - ), - Some((SecretPathSegment::OptionalField(name), [])) => { - if matches!(node.get(name.as_ref()), None | Some(Value::Null)) { - return Ok(()); +/// The secret leaves a resolution pass over `data` with `extra` looks for, +/// with nothing resolved, for a check that runs while the secret fields still +/// hold key names. +pub(crate) fn unresolved_secrets( + data: &Value, + extra: Vec, +) -> ResolvedSecrets { + ResolvedSecrets { + patterns: listed_fields::(data, extra) + .iter() + .filter(|field| !matches!(field.kind, SecretKind::StoreRef)) + .filter_map(|field| PathPattern::from_segments(&field.path)) + .collect(), + ..ResolvedSecrets::default() + } +} + +/// Every secret leaf of `C` and `extra` in `data`, each listed once. +fn listed_fields( + data: &Value, + extra: Vec, +) -> Vec { + let mut listed = std::collections::BTreeSet::new(); + C::secret_fields() + .into_iter() + .chain(C::configured_secret_fields(data)) + .chain(extra) + .filter(|field| listed.insert(field.dotted_path())) + .collect() +} + +/// One resolution pass, recording each secret it writes. +struct Resolver<'a> { + secret_store: &'a dyn PlatformSecretStore, + default_store_name: &'a StoreName, + resolved: ResolvedSecrets, +} + +impl Resolver<'_> { + /// `rendered_path` names the node for messages, and `steps` is the same + /// path as the configuration view addresses it. + fn resolve_field( + &mut self, + node: &mut Value, + field: &SecretField, + remaining: &[SecretPathSegment], + rendered_path: &str, + steps: &[PathStep], + ) -> Result<(), Report> { + match remaining.split_first() { + Some((SecretPathSegment::Field(name), [])) => { + self.resolve_leaf(node, field, name.as_ref(), rendered_path, steps) } - resolve_leaf( - node, - field, - name.as_ref(), - rendered_path, - secret_store, - default_store_name, - ) - } - Some((SecretPathSegment::Field(name), rest)) => { - let next_path = join_field(rendered_path, name.as_ref()); - let child = node - .as_object_mut() - .and_then(|object| object.get_mut(name.as_ref())) - .ok_or_else(|| missing_path(&next_path))?; - if child.is_null() { - return Err(missing_path(&next_path)); + Some((SecretPathSegment::OptionalField(name), [])) => { + if matches!(node.get(name.as_ref()), None | Some(Value::Null)) { + return Ok(()); + } + self.resolve_leaf(node, field, name.as_ref(), rendered_path, steps) } - resolve_field( - child, - field, - rest, - &next_path, - secret_store, - default_store_name, - ) - } - Some((SecretPathSegment::OptionalField(name), rest)) => { - let next_path = join_field(rendered_path, name.as_ref()); - let Some(child) = node - .as_object_mut() - .and_then(|object| object.get_mut(name.as_ref())) - else { - return Ok(()); - }; - if child.is_null() { - return Ok(()); + Some((SecretPathSegment::Field(name), rest)) => { + let next_path = join_field(rendered_path, name.as_ref()); + let child = node + .as_object_mut() + .and_then(|object| object.get_mut(name.as_ref())) + .ok_or_else(|| missing_path(&next_path))?; + if child.is_null() { + return Err(missing_path(&next_path)); + } + let next_steps = with_step(steps, PathStep::Key(name.to_string())); + self.resolve_field(child, field, rest, &next_path, &next_steps) } - resolve_field( - child, - field, - rest, - &next_path, - secret_store, - default_store_name, - ) - } - Some((SecretPathSegment::ArrayEach, rest)) => { - let items = node.as_array_mut().ok_or_else(|| { - configuration_error(format!("expected an array at `{rendered_path}`")) - })?; - for (index, item) in items.iter_mut().enumerate() { - let indexed_path = format!("{rendered_path}[{index}]"); - resolve_field( - item, - field, - rest, - &indexed_path, - secret_store, - default_store_name, - )?; + Some((SecretPathSegment::OptionalField(name), rest)) => { + let next_path = join_field(rendered_path, name.as_ref()); + let Some(child) = node + .as_object_mut() + .and_then(|object| object.get_mut(name.as_ref())) + else { + return Ok(()); + }; + if child.is_null() { + return Ok(()); + } + let next_steps = with_step(steps, PathStep::Key(name.to_string())); + self.resolve_field(child, field, rest, &next_path, &next_steps) } - Ok(()) + Some((SecretPathSegment::ArrayEach, rest)) => { + let items = node.as_array_mut().ok_or_else(|| { + configuration_error(format!("expected an array at `{rendered_path}`")) + })?; + for (index, item) in items.iter_mut().enumerate() { + let indexed_path = format!("{rendered_path}[{index}]"); + let next_steps = with_step(steps, PathStep::Index(index)); + self.resolve_field(item, field, rest, &indexed_path, &next_steps)?; + } + Ok(()) + } + Some(_) => Err(configuration_error(format!( + "unsupported secret path segment in `{}`", + field.dotted_path() + ))), + None => Ok(()), } - Some(_) => Err(configuration_error(format!( - "unsupported secret path segment in `{}`", - field.dotted_path() - ))), - None => Ok(()), } -} -fn resolve_leaf( - parent: &mut Value, - field: &SecretField, - key: &str, - rendered_parent: &str, - secret_store: &dyn PlatformSecretStore, - default_store_name: &StoreName, -) -> Result<(), Report> { - let leaf_path = join_field(rendered_parent, key); - let object = parent.as_object_mut().ok_or_else(|| { - configuration_error(format!("expected an object containing `{leaf_path}`")) - })?; - - let key_name = match object.get(key) { - Some(Value::String(value)) if !value.is_empty() => value.clone(), - Some(Value::Null) | None if field.optional => return Ok(()), - Some(Value::Null) | None => return Err(missing_path(&leaf_path)), - Some(Value::String(_)) => { - return Err(configuration_error(format!( - "secret key reference at `{leaf_path}` must not be empty" - ))); - } - _ => { + fn resolve_leaf( + &mut self, + parent: &mut Value, + field: &SecretField, + key: &str, + rendered_parent: &str, + steps: &[PathStep], + ) -> Result<(), Report> { + let leaf_path = join_field(rendered_parent, key); + let object = parent.as_object_mut().ok_or_else(|| { + configuration_error(format!("expected an object containing `{leaf_path}`")) + })?; + + let key_name = match object.get(key) { + Some(Value::String(value)) if !value.is_empty() => value.clone(), + Some(Value::Null) | None if field.optional => return Ok(()), + Some(Value::Null) | None => return Err(missing_path(&leaf_path)), + Some(Value::String(_)) => { + return Err(configuration_error(format!( + "secret key reference at `{leaf_path}` must not be empty" + ))); + } + _ => { + return Err(configuration_error(format!( + "secret key reference at `{leaf_path}` must be a string" + ))); + } + }; + + let resolved = self + .secret_store + .get_string(self.default_store_name, &key_name) + .map_err(|_| { + configuration_error(format!( + "failed to resolve secret reference at `{leaf_path}` from secret store \ + `{}`", + self.default_store_name + )) + })?; + if resolved.is_empty() { return Err(configuration_error(format!( - "secret key reference at `{leaf_path}` must be a string" + "resolved secret at `{leaf_path}` must not be empty" ))); } - }; - let resolved = secret_store - .get_string(default_store_name, &key_name) - .map_err(|_| { - configuration_error(format!( - "failed to resolve secret reference at `{leaf_path}` from secret store \ - `{default_store_name}`" - )) - })?; - if resolved.is_empty() { - return Err(configuration_error(format!( - "resolved secret at `{leaf_path}` must not be empty" - ))); + self.resolved + .paths + .push(with_step(steps, PathStep::Key(key.to_owned()))); + if !self.resolved.values.contains(&resolved) { + self.resolved.values.push(resolved.clone()); + } + object.insert(key.to_owned(), Value::String(resolved)); + Ok(()) } +} - object.insert(key.to_owned(), Value::String(resolved)); - Ok(()) +/// `steps` with `step` added to the end. +fn with_step(steps: &[PathStep], step: PathStep) -> Vec { + let mut next = Vec::with_capacity(steps.len() + 1); + next.extend_from_slice(steps); + next.push(step); + next } fn join_field(prefix: &str, field: &str) -> String { @@ -238,6 +364,28 @@ mod tests { struct Fixture; + impl ConfiguredSecretFields for Fixture { + /// Every key under `labeled` holds its secret at + /// `labeled..secret`, which is the shape a fixed path cannot + /// name. + fn configured_secret_fields(data: &Value) -> Vec { + data.get("labeled") + .and_then(Value::as_object) + .into_iter() + .flatten() + .map(|(name, _)| SecretField { + kind: SecretKind::KeyInDefault, + optional: false, + path: vec![ + SecretPathSegment::Field("labeled".into()), + SecretPathSegment::Field(name.clone().into()), + SecretPathSegment::Field("secret".into()), + ], + }) + .collect() + } + } + impl AppConfigMeta for Fixture { fn secret_fields() -> Vec { vec![ @@ -298,6 +446,61 @@ mod tests { assert!(data["outer"][0]["optional"].is_null()); } + #[test] + fn a_leaf_listed_twice_is_looked_up_once() { + // The second lookup would read the secret itself as the name of a + // key, which this store does not hold. + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "feature": {"credential": "feature-key"} + }); + let listed_again = vec![SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + SecretPathSegment::OptionalField("feature".into()), + SecretPathSegment::Field("credential".into()), + ], + }]; + + resolve_secret_references_with::( + &mut data, + &store(), + &StoreName::from("secrets"), + listed_again, + ) + .expect("should look a leaf up once however often it is listed"); + + assert_eq!(data["feature"]["credential"], "resolved-feature"); + } + + #[test] + fn resolves_a_leaf_only_the_caller_lists() { + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "added": {"key_name": "token-b"} + }); + let added = vec![SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + SecretPathSegment::OptionalField("added".into()), + SecretPathSegment::Field("key_name".into()), + ], + }]; + + resolve_secret_references_with::( + &mut data, + &store(), + &StoreName::from("secrets"), + added, + ) + .expect("should resolve the leaf the caller listed"); + + assert_eq!(data["added"]["key_name"], "resolved-b"); + assert_eq!(data["outer"][0]["token"], "resolved-a"); + } + #[test] fn resolves_present_and_skips_absent_optional_intermediate() { let mut absent = serde_json::json!({ @@ -316,6 +519,38 @@ mod tests { assert_eq!(present["feature"]["credential"], "resolved-feature"); } + #[test] + fn resolves_a_secret_under_a_name_only_the_configuration_holds() { + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "labeled": {"chosen": {"secret": "feature-key"}}, + }); + + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect("should resolve a secret the fixed paths cannot name"); + + assert_eq!(data["outer"][0]["token"], "resolved-a"); + assert_eq!(data["labeled"]["chosen"]["secret"], "resolved-feature"); + } + + #[test] + fn a_failed_configured_field_leaves_the_fixed_ones_unresolved() { + // Fixed and configured leaves resolve in one pass, so a failure in + // either hands back the configuration exactly as it arrived. + let mut data = serde_json::json!({ + "outer": [{"token": "token-a"}], + "labeled": {"chosen": {"secret": "missing"}}, + }); + let original = data.clone(); + + let err = + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect_err("should reject a missing configured secret"); + + assert!(err.to_string().contains("labeled.chosen.secret")); + assert_eq!(data, original, "should preserve unresolved data on failure"); + } + #[test] fn rejects_missing_required_path_without_secret_values() { for mut data in [ diff --git a/crates/trusted-server-core/src/service_version.rs b/crates/trusted-server-core/src/service_version.rs new file mode 100644 index 0000000000..3f6aa84268 --- /dev/null +++ b/crates/trusted-server-core/src/service_version.rs @@ -0,0 +1,170 @@ +//! The deployment's versions as response headers, a middleware of core's +//! own named `service_version` in a `[[serve]]` entry. A deployment that +//! names it writes `X-TS-Version`, the git version the build compiled in, +//! when the build knows one, and `X-TS-Service-Version`, the version the +//! adapter reports for the service, when it reports one, on every page the +//! entry covers. A deployment that does not name it sends nothing about its +//! versions on a page, because no adapter writes either header. What the +//! build was made from and the platform's version are always in the +//! attestation document, under `build` and `platform.serviceVersion`. + +use http::HeaderValue; + +use crate::attestation::UNKNOWN; +use crate::constants::{HEADER_X_TS_SERVICE_VERSION, HEADER_X_TS_VERSION, TS_GIT_VERSION}; +use crate::middleware::{Middleware, MiddlewareAction, MiddlewareContext, MiddlewarePhase}; + +/// The name an entry writes, and the integration the registry lists the +/// middleware under. +pub const MODULE: &str = "service_version"; + +/// The middleware. It holds the git version the build compiled in, and reads +/// the platform's version from the context. +#[derive(Debug)] +pub struct ServiceVersion { + /// The deployed git version, `None` when the build knew none. + pub git_version: Option<&'static str>, +} + +impl Default for ServiceVersion { + fn default() -> Self { + Self { + git_version: TS_GIT_VERSION, + } + } +} + +impl Middleware for ServiceVersion { + fn middleware_id(&self) -> &'static str { + MODULE + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Serve] + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let mut response_headers = Vec::new(); + if let Some(value) = header_value("git version", self.git_version) { + response_headers.push((HEADER_X_TS_VERSION, value)); + } + let service_version = + Some(context.platform.service_version.as_str()).filter(|version| *version != UNKNOWN); + if let Some(value) = header_value("service version", service_version) { + response_headers.push((HEADER_X_TS_SERVICE_VERSION, value)); + } + MiddlewareAction { + response_headers, + ..MiddlewareAction::pass() + } + } +} + +/// `version` as a header value, or `None`, with a warning, when a header +/// cannot carry it. +fn header_value(what: &str, version: Option<&str>) -> Option { + let version = version?; + match HeaderValue::from_str(version) { + Ok(value) => Some(value), + Err(_) => { + log::warn!("the {what} {version:?} is not a header value, so it is not written"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::attestation::PlatformIdentity; + use crate::integrations::IntegrationDocumentState; + use crate::middleware::test_support; + + fn action_for(git_version: Option<&'static str>, service_version: &str) -> MiddlewareAction { + let platform = PlatformIdentity { + service_version: service_version.to_owned(), + ..PlatformIdentity::named("example-platform") + }; + let document_state = IntegrationDocumentState::default(); + ServiceVersion { git_version }.create(&test_support::context_on( + MiddlewarePhase::Serve, + &document_state, + &platform, + )) + } + + fn headers_of(action: &MiddlewareAction) -> Vec<(&str, &[u8])> { + action + .response_headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_bytes())) + .collect() + } + + #[test] + fn the_git_version_the_build_compiled_in_is_written_as_x_ts_version() { + let action = action_for(Some("v1.3.0"), UNKNOWN); + + assert_eq!( + headers_of(&action), + [("x-ts-version", b"v1.3.0".as_slice())], + "should write the git version alone on a platform reporting no version" + ); + } + + #[test] + fn the_version_the_platform_reports_is_written_as_x_ts_service_version() { + let action = action_for(None, "52"); + + assert_eq!( + headers_of(&action), + [("x-ts-service-version", b"52".as_slice())], + "should write the platform's version alone when the build knew no git version" + ); + } + + #[test] + fn both_versions_are_written_when_both_are_known() { + let action = action_for(Some("v1.3.0"), "52"); + + assert_eq!( + headers_of(&action), + [ + ("x-ts-version", b"v1.3.0".as_slice()), + ("x-ts-service-version", b"52".as_slice()), + ], + "should write the git version and then the platform's" + ); + } + + #[test] + fn a_build_knowing_no_git_version_on_a_platform_reporting_none_writes_nothing() { + assert!( + action_for(None, UNKNOWN).is_pass(), + "should write nothing when neither version is known" + ); + } + + #[test] + fn a_version_that_is_not_a_header_value_is_not_written() { + assert!( + action_for(Some("v1\n"), "52\n").is_pass(), + "should write nothing for versions a header cannot carry" + ); + } + + #[test] + fn the_default_carries_the_git_version_the_build_compiled_in() { + assert_eq!( + ServiceVersion::default().git_version, + TS_GIT_VERSION, + "should carry the compiled-in TS_GIT_VERSION" + ); + } + + #[test] + fn it_runs_in_the_serve_phase_under_its_name() { + assert_eq!(ServiceVersion::default().phases(), [MiddlewarePhase::Serve]); + assert_eq!(ServiceVersion::default().middleware_id(), "service_version"); + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 3efc157701..d34944f173 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -6,23 +6,29 @@ use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; -use std::collections::{HashMap, HashSet}; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; use std::time::Duration; use subtle::ConstantTimeEq as _; use url::Url; -use validator::{Validate, ValidationError}; +use validator::{Validate, ValidationError, ValidationErrors, ValidationErrorsKind}; use crate::auction_config_types::AuctionConfig; use crate::cache_policy::{CachePolicy, CacheVisibility}; use crate::consent_config::ConsentConfig; use crate::constants::INTERNAL_HEADERS; use crate::creative_opportunities::CreativeOpportunitiesConfig; +use crate::ec::module::{ + EcModuleSelection, HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY, check_named_module_configuration, +}; use crate::error::TrustedServerError; use crate::host_header::validate_host_header_override_value; +use crate::middleware::{MiddlewarePhase, PhaseEntries}; use crate::platform::PlatformImageOptimizerRegion; +use crate::provider_table::{ProviderChoice, ProviderList, SectionModules}; use crate::redacted::Redacted; #[cfg(test)] @@ -39,10 +45,11 @@ pub struct Publisher { /// (see [`ec_cookie_domain`](Self::ec_cookie_domain)). #[validate(custom(function = validate_cookie_domain))] pub cookie_domain: String, + #[serde(serialize_with = "crate::redacted::sensitive")] #[validate(custom(function = validate_no_trailing_slash))] pub origin_url: String, /// Optional outbound Host header to send while connecting to `origin_url`. - #[serde(default)] + #[serde(default, serialize_with = "crate::redacted::sensitive")] #[validate(custom(function = validate_host_header_override))] pub origin_host_header_override: Option, /// Secret used to encrypt/decrypt proxied URLs in `/first-party/proxy`. @@ -215,165 +222,200 @@ impl Publisher { } } -#[derive(Default, Clone, Deserialize, Serialize)] -pub struct IntegrationSettings { - #[serde(flatten)] - entries: HashMap, +/// Which integrations run, and the settings each one is given. +/// +/// The sections of module types core does not read itself, such as `[cmp]` or +/// `[tag]`, each named for the type of module it selects. +/// +/// Every top-level table that is not one of Trusted Server's own settings is +/// read as one of these. Whether its type is one a module on offer has is +/// only known where the registry is built, so it is checked there. +#[derive(Clone, Default, PartialEq, Serialize)] +#[serde(transparent)] +pub struct TypeSections(BTreeMap); + +impl std::fmt::Debug for TypeSections { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_map().entries(self.0.iter()).finish() + } } -impl std::fmt::Debug for IntegrationSettings { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut integration_ids = self.entries.keys().collect::>(); - integration_ids.sort_unstable(); - formatter - .debug_struct("IntegrationSettings") - .field("integration_ids", &integration_ids) - .finish() +impl<'de> Deserialize<'de> for TypeSections { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let entries = serde_json::Map::::deserialize(deserializer)?; + let mut sections = BTreeMap::new(); + for (name, value) in entries { + let JsonValue::Object(table) = value else { + return Err(serde::de::Error::custom(format!( + "unknown field `{name}`, which is neither a setting Trusted Server reads nor \ + the section of a module type, which is a table" + ))); + }; + if name.contains('.') || !crate::module_name::is_valid(&name) { + return Err(serde::de::Error::custom(format!( + "unknown field `{name}`. The section of a module type is named for \ + the type, in lower case letters, digits, `_` or `-`" + ))); + } + let section = SectionModules::from_entries(table).map_err(|message| { + serde::de::Error::custom(format!( + "[{name}] is not a section Trusted Server reads itself, so it is read as \ + the section of a module type, and it {message}" + )) + })?; + sections.insert(name, section); + } + Ok(Self(sections)) } } -pub trait IntegrationConfig: DeserializeOwned + Validate { - fn is_enabled(&self) -> bool; +impl TypeSections { + /// Whether no type section is present. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } - /// Validate the public field schema for an explicitly disabled config. - /// - /// The default deserializes the integration's normal schema, except it - /// permits omitted enabled-only required fields. Override this only when a - /// disabled integration has a distinct public schema. + /// Each type section, by name. + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(name, section)| (name.as_str(), section)) + } + + /// The section of `type_name`, when present. + #[must_use] + pub fn section(&self, type_name: &str) -> Option<&SectionModules> { + self.0.get(type_name) + } + + /// The section of `type_name`, created empty when absent. + pub fn section_mut(&mut self, type_name: &str) -> &mut SectionModules { + self.0.entry(type_name.to_owned()).or_default() + } + + /// Refuses a section that selects nothing, and checks each selection. /// /// # Errors /// - /// Returns a deserialization error when the disabled public field schema is invalid. - fn validate_disabled_schema(raw: &JsonValue) -> Result<(), serde_json::Error> { - match serde_json::from_value::(raw.clone()) { - Ok(_) => Ok(()), - Err(error) if error.to_string().starts_with("missing field ") => Ok(()), - Err(error) => Err(error), + /// Naming the first section at fault. + pub fn validate(&self) -> Result<(), Report> { + for (name, section) in &self.0 { + if section.selected().is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{name}] selects no module. A module type's section names what runs \ + with `module` or `modules`, so select one or remove the section" + ), + })); + } + section + .validate(name) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; } + Ok(()) } } -impl IntegrationSettings { - /// Inserts a configuration value for an integration. - /// - /// # Errors - /// - /// Returns an error if the configuration cannot be serialized to JSON. - #[cfg_attr(not(test), allow(dead_code))] - pub fn insert_config( - &mut self, - integration_id: impl Into, - value: &T, - ) -> Result<(), Report> - where - T: Serialize, - { - let json = - serde_json::to_value(value).change_context(TrustedServerError::Configuration { - message: "Failed to serialize integration configuration".to_string(), - })?; - self.entries.insert(integration_id.into(), json); - Ok(()) +/// The section that holds what one vendor's modules share. +pub const SHARED_TYPE: &str = "shared"; + +/// The `[shared]` section, one `[shared.]` table per vendor, holding +/// the settings every module of that vendor reads, such as an endpoint and a +/// license key, read through [`Settings::shared_config`]. +#[derive(Clone, Default, Deserialize, Serialize)] +#[serde(transparent)] +pub struct SharedTables(BTreeMap>); + +impl std::fmt::Debug for SharedTables { + // The tables hold secrets, so only the vendors are shown. + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SharedTables") + .field("vendors", &self.vendors().collect::>()) + .finish() } +} - fn is_explicitly_disabled(raw: &JsonValue) -> bool { - raw.as_object() - .and_then(|map| map.get("enabled")) - .and_then(JsonValue::as_bool) - == Some(false) +impl SharedTables { + /// Whether no vendor has a table. + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() } - fn remove_legacy_static_secret_store_selectors(&mut self) { - let Some(datadome) = self - .entries - .get_mut("datadome") - .and_then(JsonValue::as_object_mut) - else { - return; - }; + /// The vendors with a table, in name order. + pub fn vendors(&self) -> impl Iterator { + self.0.keys().map(String::as_str) + } - let mut removed = datadome.remove("server_side_key_secret_store").is_some(); - if let Some(bypass) = datadome - .get_mut("protection_test_bypass") - .and_then(JsonValue::as_object_mut) - { - removed |= bypass.remove("credential_secret_store").is_some(); - } - if removed { - log::warn!( - "DataDome secret-store selectors are deprecated and ignored; static credentials resolve through the default app-config secret store" - ); - } + /// Whether `[shared.]` is present. + #[must_use] + pub fn contains(&self, vendor: &str) -> bool { + self.0.contains_key(vendor) } - /// Retrieves and validates a typed configuration for an integration. + /// Reads and validates `[shared.]` as `T`, or `None` when the table + /// is absent. /// /// # Errors /// - /// Returns an error if the configuration cannot be parsed from JSON or fails validation. - pub fn get_typed( - &self, - integration_id: &str, - ) -> Result, Report> + /// When the table cannot be read as `T` or fails its validation, naming + /// the table and the underlying error. + pub fn get_typed(&self, vendor: &str) -> Result, Report> where - T: IntegrationConfig, + T: DeserializeOwned + Validate, { - let raw = match self.entries.get(integration_id) { - Some(value) => value, - None => return Ok(None), + let Some(table) = self.0.get(vendor) else { + return Ok(None); }; - - if Self::is_explicitly_disabled(raw) { - T::validate_disabled_schema(raw).change_context(TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration could not be parsed" - ), + let config: T = + serde_json::from_value(JsonValue::Object(table.clone())).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{SHARED_TYPE}.{vendor}] could not be read: {error}"), + }) })?; - return Ok(None); - } - - let config: T = serde_json::from_value(raw.clone()).change_context( - TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration could not be parsed" - ), - }, - )?; - - // Field validation runs only for integrations that resolve to enabled. - // An integration whose `enabled` flag is omitted falls back to its - // serde default, which the explicit-`false` fast path above cannot - // observe. Validating before this check would reject documented - // template placeholders in sections that are not actually turned on. - if !config.is_enabled() { - return Ok(None); - } - - config.validate().map_err(|err| { + config.validate().map_err(|error| { Report::new(TrustedServerError::Configuration { - message: format!( - "Integration '{integration_id}' configuration failed validation: {err}" - ), + message: format!("[{SHARED_TYPE}.{vendor}] failed validation: {error}"), }) })?; - Ok(Some(config)) } -} - -impl Deref for IntegrationSettings { - type Target = HashMap; - fn deref(&self) -> &Self::Target { - &self.entries + /// Refuses a table whose name is not a vendor's name, being lower case + /// letters, digits, `_` or `-`. + /// + /// # Errors + /// + /// Naming the first table at fault. + pub fn validate_names(&self) -> Result<(), Report> { + match self + .vendors() + .find(|vendor| vendor.contains('.') || !crate::module_name::is_valid(vendor)) + { + Some(vendor) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{SHARED_TYPE}] has a table named `{vendor}`, which is not a vendor's \ + name. A vendor's shared table is named in lower case letters, \ + digits, `_` or `-`" + ), + })), + None => Ok(()), + } } } -impl DerefMut for IntegrationSettings { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.entries - } -} +/// The settings type a module reads from its table, beneath the section that +/// selects it. +/// +/// The type states which settings the module takes and how they are +/// validated, and nothing else. Whether the module runs is not its business, +/// because the section's selection names what runs. +pub trait IntegrationConfig: DeserializeOwned + Validate {} /// A partner (SSP, DSP, identity vendor) configured in `[[ec.partners]]`. /// @@ -537,15 +579,65 @@ impl EcPartner { /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, /// KV store names, and partner registry. -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] +/// +/// Every key in the section other than the fields below is one module's +/// `[ec.]` settings table, held in +/// [`module_blocks`](Self::module_blocks), so the field names below are +/// reserved and cannot name a module. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] pub struct Ec { - /// Publisher passphrase used as HMAC key for EC generation. - #[validate(custom(function = Ec::validate_passphrase))] - pub passphrase: Redacted, + /// The name of the Edge Cookie identity module to activate. + /// + /// Set it in the `[ec]` TOML section, for example `"hmac"`. The name is + /// the module's implementation unless its `[ec.]` block names a + /// different one, in which case the name is a label of the operator's + /// choosing (see [`module_blocks`](Self::module_blocks)). Deployment + /// tooling can merge a `TRUSTED_SERVER__EC__MODULE` environment value + /// into the published configuration before it is loaded, so the same + /// compiled WebAssembly can switch modules at deployment. The running + /// server reads its settings from the platform config store, not the + /// environment. When absent, no Edge Cookie is generated and Trusted + /// Server runs statelessly, and the explicit `"none"` spells the same + /// choice. A selection whose implementation needs settings it has no block + /// for is rejected at startup by + /// [`validate_module_selection`](Self::validate_module_selection). + /// + /// Typed as [`EcModuleSelection`], which reads and writes the same + /// string, so every check that asks which module is selected matches on + /// one vocabulary rather than comparing string literals. + #[serde(default)] + pub module: Option, - /// Fastly KV store name for the EC identity graph. + /// Deprecated location of the HMAC passphrase, read so a configuration + /// written for the previous release still starts. + /// + /// [`migrate_legacy_ec_layout`](Self::migrate_legacy_ec_layout) maps it + /// to `module = "hmac"` with the passphrase in the `[ec.hmac]` block and + /// logs a deprecation warning, so a fleet can move configuration and + /// binaries independently. A configuration carrying both the old and the + /// new form is rejected rather than guessed at. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option>, + + /// Extra exact origins allowed to POST the client resolve endpoint. + /// + /// The endpoint always accepts `https://{publisher.domain}` and nothing + /// else by default. A publisher whose pages are served from another origin, + /// `www` being the common case, lists those origins here. Each entry is a + /// serialized origin (RFC 6454 §6.1), being `http://` or `https://` and a + /// host with an optional port and nothing after it, and + /// [`validate_resolve_allowed_origins`](Self::validate_resolve_allowed_origins) + /// refuses any other entry when the settings load. Each is compared with + /// the request's `Origin` by the same-origin test of RFC 6454 §5, so the + /// scheme, the host and the port all have to match, with a missing port + /// meaning the scheme's default. A suffix or subdomain match is never + /// performed, because control of a DNS namespace does not make every host + /// under it a trusted identity-setting origin. #[serde(default)] + pub resolve_allowed_origins: Vec, + + /// Fastly KV store name for the EC identity graph. + #[serde(default, serialize_with = "crate::redacted::sensitive")] pub ec_store: Option, /// Maximum number of concurrent pull-sync requests. @@ -568,8 +660,20 @@ pub struct Ec { /// Partners (SSPs, DSPs, identity vendors) for EC identity sync. #[serde(default, deserialize_with = "vec_from_seq_or_map")] - #[validate(nested)] pub partners: Vec, + + /// The settings tables of the Edge Cookie identity modules, keyed by the + /// name each is written under. + /// + /// A module has a block only when it has settings of its own, so + /// `[ec] module = "hmac"` needs an `[ec.hmac]` block for its required + /// passphrase while a module with no settings needs none. The + /// [`module`](Self::module) selector names the active module and + /// [`validate_module_selection`](Self::validate_module_selection) + /// rejects a block it does not name, so at most one block survives + /// startup. + #[serde(flatten)] + pub module_blocks: EcModuleBlocks, } impl Ec { @@ -631,145 +735,914 @@ impl Ec { } Ok(()) } -} - -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct Rewrite { - /// List of domains to exclude from rewriting. Supports wildcards (e.g., "*.example.com"). - /// URLs from these domains will not be proxied through first-party endpoints. - #[serde(default)] - pub exclude_domains: Vec, -} -impl Rewrite { - /// Checks if a URL should be excluded from rewriting based on domain matching - #[allow(dead_code)] - #[must_use] - pub fn is_excluded(&self, url: &str) -> bool { - // Parse URL to extract host - let Ok(parsed) = url::Url::parse(url) else { - return false; - }; - - let host = parsed.host_str().unwrap_or(""); - - // Check exact domain matches (with wildcard support) - for domain in &self.exclude_domains { - if let Some(suffix) = domain.strip_prefix("*.") { - // Wildcard: *.example.com matches both example.com and sub.example.com - if host == suffix || host.ends_with(&format!(".{}", suffix)) { - return true; - } - } else if host == domain { - return true; + /// Refuses a [`resolve_allowed_origins`](Self::resolve_allowed_origins) + /// entry that is not a serialized origin. + /// + /// The resolve endpoint reads each entry with + /// [`parse_serialized_origin`](crate::ec::resolve::parse_serialized_origin), + /// and an entry that parse rejects matches no request, so every resolve + /// from the origin it was meant to allow would answer `403`. Checking with + /// the same parse here means an entry that loads is one a request can + /// match. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] naming the first entry + /// that is not a serialized origin and what is wrong with it. + pub fn validate_resolve_allowed_origins(&self) -> Result<(), Report> { + for entry in &self.resolve_allowed_origins { + if let Err(reason) = crate::ec::resolve::parse_serialized_origin(entry) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[ec] resolve_allowed_origins entry `{entry}` {reason}. \ + Each entry is a bare origin, being `http://` or `https://` and a \ + host with an optional port, such as `https://www.example.com`" + ), + })); } } - - false + Ok(()) } -} -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] -pub struct Handler { - #[validate(length(min = 1), custom(function = validate_path))] - pub path: String, - #[validate(custom(function = validate_redacted_not_empty))] - pub username: Redacted, - #[validate(custom(function = validate_redacted_not_empty))] - pub password: Redacted, - #[serde(skip, default)] - #[validate(skip)] - regex: OnceLock>, -} + /// Validates the module selection against the configured blocks. + /// + /// When [`module`](Self::module) is set, the selection has to resolve + /// to an implementation this deployment can configure, and every block in + /// [`module_blocks`](Self::module_blocks) has to be the one the + /// selector names, so a deployment that selects a module (in TOML or via + /// the environment override) but has not configured it fails fast at + /// startup rather than silently running stateless. When no module is + /// selected, Trusted Server runs statelessly and this check passes. + /// + /// Whether the named implementation exists at all is settled by + /// [`build_module`](crate::ec::module::build_module), which is the + /// one place that knows both the implementations built into core and the + /// one this deployment's adapter injects. + /// + /// Whether this build compiles an implementation in at all is answered by + /// `check_named_module_configuration` in [`crate::ec::module`], beside + /// the resolution it belongs to, rather than here, because the settings + /// cannot know which implementations are compiled out of this build. Only + /// the resolution knows that. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when a module name or + /// implementation is not a module name, when a block is configured with no + /// selector or alongside `"none"`, when the selector names a key `[ec]` + /// reads as its own setting, when the selected implementation is not + /// compiled into this build, when a block the selector does not name is + /// configured, or when the selected module resolves to an implementation + /// that needs settings and has no block. + pub fn validate_module_selection(&self) -> Result<(), Report> { + for (name, block) in self.module_blocks.iter() { + Self::validate_module_name(name)?; + if let Some(implementation) = &block.implementation { + Self::validate_module_name(implementation)?; + } + } -impl Handler { - /// Known handler password placeholders that must not be used in deployments. - pub const PASSWORD_PLACEHOLDERS: &[&str] = &[ - "replace-with-admin-password-32-bytes", - "replace-with-admin-password", - "change-me-admin-password", - ]; + let Some(selection) = self.module.as_ref() else { + if !self.module_blocks.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec.] module blocks are configured but no [ec] module \ + is selected. Set [ec] module = \"\" to activate one, or \ + remove the blocks to run statelessly" + .to_owned(), + })); + } + return Ok(()); + }; - /// Returns `true` if `password` matches a known placeholder value - /// (case-insensitive). - #[must_use] - pub fn is_placeholder_password(password: &str) -> bool { - let password = password.trim(); - Self::PASSWORD_PLACEHOLDERS - .iter() - .any(|placeholder| placeholder.eq_ignore_ascii_case(password)) - } + // `"none"` is explicit statelessness, the same meaning as omitting the + // selector, spelled out. It is subject to the same rule that no + // module blocks may be left configured. + let EcModuleSelection::Named(name) = selection else { + if !self.module_blocks.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec] module = \"none\" selects stateless operation, but \ + [ec.] module blocks are configured. Remove the blocks, \ + or select the module they configure" + .to_owned(), + })); + } + return Ok(()); + }; - fn compiled_regex(&self) -> Result<&Regex, Report> { - match self - .regex - .get_or_init(|| Regex::new(&self.path).map_err(|err| err.to_string())) + let name = name.as_str(); + Self::validate_module_name(name)?; + if EC_SECTION_KEYS.contains(&name) || name == REMOVED_EC_PROVIDERS_TABLE { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[ec] module = \"{name}\" names a key the `[ec]` section reads as its \ + own setting, so no module can be configured under it. Give the \ + module a name of its own" + ), + })); + } + + // Whether this build compiles the implementation in at all is the + // resolution's question, not the settings', so it is asked there. + let implementation = self.module_blocks.implementation(name); + check_named_module_configuration(implementation)?; + + // A module has a block only when it has settings, and core knows + // which of its own implementations need them. Both modules that + // derive an identifier at the edge take a passphrase, so both need one. + // The demonstration module is built from nothing and needs none. A + // module an adapter injects has the contents of its block read by + // that adapter when it builds the module, so core cannot say whether + // it needs one. + if (implementation == HMAC_MODULE_KEY || implementation == HOST_SIGNALS_MODULE_KEY) + && !self.module_blocks.contains_key(name) { - Ok(regex) => Ok(regex), - Err(message) => Err(Report::new(TrustedServerError::Configuration { + return Err(Report::new(TrustedServerError::Configuration { message: format!( - "Handler path regex `{}` failed to compile: {message}", - self.path + "Edge Cookie module `{name}` is selected but has no `[ec.{name}]` configuration" ), - })), + })); + } + + // Every configured block must be the selected one. An unreferenced + // block is almost always a mistake (a mistyped selector or a stale + // block), and accepting it silently invites configuration drift. + let unreferenced: Vec<&str> = self + .module_blocks + .keys() + .map(String::as_str) + .filter(|configured| *configured != name) + .collect(); + if unreferenced.is_empty() { + Ok(()) + } else { + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[ec.{}] is configured but `{name}` is selected. Remove the unselected \ + block, or correct the selector", + unreferenced.join("], [ec.") + ), + })) } } - /// Eagerly compile the handler regex to fail fast during startup. + /// Validates one module name or implementation id. /// /// # Errors /// - /// Returns a configuration error if the handler path regex does not compile. - pub fn prepare_runtime(&self) -> Result<(), Report> { - self.compiled_regex().map(|_| ()) + /// Returns [`TrustedServerError::Configuration`] when `name` is not a + /// module name, see [`crate::module_name::is_valid`]. + fn validate_module_name(name: &str) -> Result<(), Report> { + if crate::module_name::is_valid(name) { + return Ok(()); + } + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Edge Cookie module name `{name}` is not a module name, which is parts \ + joined by `.`, each of lower case letters, digits, `_` or `-`" + ), + })) } - /// Determine whether this handler applies to the request path. + /// Migrates the deprecated `[ec] passphrase` form to the module layout. + /// + /// A configuration still carrying the old key keeps working for one + /// release cycle, mapping to `module = "hmac"` with the passphrase in + /// the `[ec.hmac]` block, and a deprecation warning names the new + /// location. A configuration carrying both forms is rejected so a + /// half-edited file fails loudly instead of one form silently winning. + /// + /// The deprecated key is held to the same passphrase rules as the new + /// `[ec.hmac]` block. Validation runs before this migration and the + /// deprecated field carries no check of its own, so without the check here + /// a short or empty passphrase in the old location would start a + /// deployment that the new location rejects. /// /// # Errors /// - /// Returns a configuration error if the handler path regex does not compile. - pub fn matches_path(&self, path: &str) -> Result> { - self.compiled_regex().map(|regex| regex.is_match(path)) + /// Returns [`TrustedServerError::Configuration`] when both the deprecated + /// key and any part of the module configuration are present, or when the + /// deprecated passphrase fails [`Self::validate_passphrase`]. + pub fn migrate_legacy_ec_layout(&mut self) -> Result<(), Report> { + let Some(passphrase) = self.passphrase.take() else { + return Ok(()); + }; + if self.module.is_some() || !self.module_blocks.is_empty() { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec] passphrase (deprecated) and the [ec] module configuration \ + are both present. Keep exactly one form, moving the passphrase to \ + [ec.hmac] and deleting the old key" + .to_owned(), + })); + } + Self::validate_passphrase(&passphrase).map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!( + "[ec] passphrase (deprecated) is invalid ({err}). Use a random secret \ + of at least {} bytes, placed in [ec.hmac]", + Self::MIN_PASSPHRASE_LENGTH, + ), + }) + })?; + log::warn!( + "[ec] passphrase is deprecated. Move it to [ec.hmac] passphrase and set \ + [ec] module = \"hmac\"" + ); + self.module = Some(EcModuleSelection::from(HMAC_MODULE_KEY)); + self.module_blocks.insert( + HMAC_MODULE_KEY.to_owned(), + EcModuleBlock::from(HmacModuleConfig { passphrase }), + ); + Ok(()) } } -#[derive(Debug, Default, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct RequestSigning { - #[serde(default = "default_request_signing_enabled")] - pub enabled: bool, - pub config_store_id: String, - pub secret_store_id: String, +impl Validate for Ec { + /// Validates the partner entries and the settings of every block that + /// configures a module built into core. + /// + /// Written out rather than derived because each block's errors are keyed + /// by the name the operator wrote the block under, so a passphrase + /// `[ec.primary]` rejects is reported at `ec.primary.passphrase`. A + /// derived nested validation would key them under this struct's own field + /// name, which is a path no configuration has. + fn validate(&self) -> Result<(), ValidationErrors> { + let mut errors = ValidationErrors::new(); + errors.merge_self("partners", self.partners.validate()); + let blocks = self + .module_blocks + .hmac_blocks() + .map(|(name, config)| (name, config.validate())) + .chain( + self.module_blocks + .host_signals_blocks() + .map(|(name, config)| (name, config.validate())), + ); + for (name, result) in blocks { + if let Err(block_errors) = result { + errors.errors_mut().insert( + Cow::Owned(name.to_owned()), + ValidationErrorsKind::Struct(Box::new(block_errors)), + ); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } } -impl RequestSigning { - /// Reserved example store-id values from the config template, plus the - /// empty string, that must not be deployed while request signing is enabled. - pub const STORE_ID_PLACEHOLDERS: &[&str] = &[ - "", - "", - ]; +/// The keys the `[ec]` section reads as its own settings. +/// +/// Every other key in the section is one module's `[ec.]` settings +/// table, so these names are reserved and cannot name a module. They are +/// written out because serde reads the [`Ec`] fields by name and has no way to +/// report the list back for an error message. +const EC_SECTION_KEYS: &[&str] = &[ + "module", + "passphrase", + "resolve_allowed_origins", + "ec_store", + "pull_sync_concurrency", + "cluster_trust_threshold", + "cluster_recheck_secs", + "partners", +]; - /// Returns `true` if `store_id` is empty or a known template placeholder - /// (case-insensitive). - #[must_use] - pub fn is_placeholder_store_id(store_id: &str) -> bool { - let store_id = store_id.trim(); - store_id.is_empty() - || Self::STORE_ID_PLACEHOLDERS - .iter() - .any(|p| p.eq_ignore_ascii_case(store_id)) - } +/// The removed table that used to hold every module's settings. +const REMOVED_EC_PROVIDERS_TABLE: &str = "providers"; - /// Returns `true` if `store_id` cannot be deployed as-is: a placeholder, or - /// a value with surrounding whitespace that the key-management routes would - /// forward to the management API verbatim. +/// The key in a module block that names the implementation it configures. +/// +/// Anything reading a module block out of a serialized configuration rather +/// than out of [`EcModuleBlock`] reads the same key from here, so the two +/// cannot drift apart. +pub(crate) const MODULE_IMPLEMENTATION_KEY: &str = "implementation"; + +/// The `[ec.]` settings tables of the Edge Cookie identity modules, +/// keyed by the name each is written under. +/// +/// A module that has settings is configured in its own table, for example: +/// +/// ```toml +/// [ec] +/// module = "hmac" +/// +/// [ec.hmac] +/// passphrase = "replace-with-32-plus-byte-random-secret" +/// ``` +/// +/// A table may name the implementation it configures, which makes its own name +/// a label of the operator's choosing, so the same configuration can also be +/// written as: +/// +/// ```toml +/// [ec] +/// module = "primary" +/// +/// [ec.primary] +/// implementation = "hmac" +/// passphrase = "replace-with-32-plus-byte-random-secret" +/// ``` +/// +/// The active module is chosen by the [`Ec::module`] selector, and the one +/// table present must be the one it names (see +/// [`Ec::validate_module_selection`]). +#[derive(Debug, Default, Clone, Serialize)] +#[serde(transparent)] +pub struct EcModuleBlocks(BTreeMap); + +impl EcModuleBlocks { + /// The implementation the module `name` resolves to. + /// + /// A block may name the implementation it configures. Without one, and + /// when the module has no block at all, the module's name is its + /// implementation. + #[must_use] + pub fn implementation<'a>(&'a self, name: &'a str) -> &'a str { + self.0 + .get(name) + .and_then(|block| block.implementation.as_deref()) + .unwrap_or(name) + } + + /// Every configured block that sets the built-in HMAC module up, with + /// the name it is written under. + /// + /// The name is the label the operator chose, so a caller reporting one of + /// these settings names the path the operator wrote rather than the + /// implementation's own. + pub fn hmac_blocks(&self) -> impl Iterator { + self.0 + .iter() + .filter_map(|(name, block)| block.hmac_settings().map(|config| (name.as_str(), config))) + } + + /// Every configured block that sets the built-in host-signal module up, + /// with the name it is written under. + /// + /// The same contract as [`hmac_blocks`](Self::hmac_blocks), for the other + /// module core builds itself. + pub fn host_signals_blocks(&self) -> impl Iterator { + self.0.iter().filter_map(|(name, block)| { + block + .host_signals_settings() + .map(|config| (name.as_str(), config)) + }) + } +} + +impl Deref for EcModuleBlocks { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for EcModuleBlocks { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl<'de> Deserialize<'de> for EcModuleBlocks { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct BlocksVisitor; + + impl<'de> serde::de::Visitor<'de> for BlocksVisitor { + type Value = EcModuleBlocks; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("the `[ec.]` module settings tables") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut blocks = BTreeMap::new(); + while let Some(name) = map.next_key::()? { + let table = map.next_value::()?; + blocks.insert(name.clone(), read_module_block::(&name, table)?); + } + Ok(EcModuleBlocks(blocks)) + } + } + + deserializer.deserialize_map(BlocksVisitor) + } +} + +/// Reads one key left over from the `[ec]` section as the module block it +/// names. +/// +/// This is where the `[ec]` section gets the unknown-key check that the fixed +/// fields lose by holding the module blocks alongside them. A key that is +/// not a table cannot be a module, so it is reported as the mistyped setting +/// it almost certainly is. +fn read_module_block(name: &str, table: JsonValue) -> Result +where + E: serde::de::Error, +{ + if name == REMOVED_EC_PROVIDERS_TABLE { + return Err(E::custom( + "[ec.providers] is no longer read. Each module's settings moved to a table of \ + its own under [ec], so a block written as [ec.providers.hmac] is now [ec.hmac]", + )); + } + let JsonValue::Object(mut table) = table else { + let known = EC_SECTION_KEYS + .iter() + .map(|key| format!("`{key}`")) + .collect::>() + .join(", "); + return Err(E::custom(format!( + "unknown field `{name}`, expected one of {known}, or an [ec.] module \ + settings table" + ))); + }; + let implementation = match table.remove(MODULE_IMPLEMENTATION_KEY) { + None => None, + Some(JsonValue::String(implementation)) => Some(implementation), + Some(_) => { + return Err(E::custom(format!( + "`implementation` in [ec.{name}] must be a string naming the module \ + implementation the block configures" + ))); + } + }; + + // The implementation decides how the rest of the table is read. Core reads + // its own modules' settings here, so a mistyped key fails where the + // configuration is read rather than at the request that needed it, and + // keeps the settings of a module an adapter injects as the raw values + // that adapter deserializes for itself. + let invalid = |err| E::custom(format!("[ec.{name}] is invalid ({err})")); + let settings = match implementation.as_deref().unwrap_or(name) { + HMAC_MODULE_KEY => EcModuleSettings::Hmac( + serde_json::from_value(JsonValue::Object(table)).map_err(invalid)?, + ), + HOST_SIGNALS_MODULE_KEY => EcModuleSettings::HostSignals( + serde_json::from_value(JsonValue::Object(table)).map_err(invalid)?, + ), + _ => EcModuleSettings::Injected(table), + }; + Ok(EcModuleBlock { + implementation, + settings, + }) +} + +/// One module's `[ec.]` settings table. +#[derive(Debug, Clone, Serialize)] +pub struct EcModuleBlock { + /// The implementation this block configures, written as + /// `implementation = ""`, when the block's own name is not it. + /// + /// Setting it makes the block name a label of the operator's choosing, so + /// one implementation can be configured under any name. Everything that + /// resolves the selected module reads the implementation rather than the + /// label. + #[serde(skip_serializing_if = "Option::is_none")] + pub implementation: Option, + + /// Everything the block holds other than `implementation`. + #[serde(flatten)] + pub settings: EcModuleSettings, +} + +impl EcModuleBlock { + /// The built-in HMAC module's settings, when this block configures it. + #[must_use] + pub fn hmac_settings(&self) -> Option<&HmacModuleConfig> { + match &self.settings { + EcModuleSettings::Hmac(config) => Some(config), + EcModuleSettings::HostSignals(_) | EcModuleSettings::Injected(_) => None, + } + } + + /// The built-in host-signal module's settings, when this block + /// configures it. + #[must_use] + pub fn host_signals_settings(&self) -> Option<&HostSignalsModuleConfig> { + match &self.settings { + EcModuleSettings::HostSignals(config) => Some(config), + EcModuleSettings::Hmac(_) | EcModuleSettings::Injected(_) => None, + } + } +} + +impl From for EcModuleBlock { + /// Builds the `[ec.hmac]` block, the built-in HMAC module configured + /// under its own name. A block under a label names its implementation + /// instead. + fn from(config: HmacModuleConfig) -> Self { + Self { + implementation: None, + settings: EcModuleSettings::Hmac(config), + } + } +} + +impl From for EcModuleBlock { + /// Builds the `[ec.host_signals]` block, the built-in host-signal module + /// configured under its own name. A block under a label names its + /// implementation instead. + fn from(config: HostSignalsModuleConfig) -> Self { + Self { + implementation: None, + settings: EcModuleSettings::HostSignals(config), + } + } +} + +/// The settings one module block holds, read according to the +/// implementation the block configures. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum EcModuleSettings { + /// The built-in HMAC module's settings. + Hmac(HmacModuleConfig), + + /// The built-in host-signal module's settings. + HostSignals(HostSignalsModuleConfig), + + /// The settings of a module an adapter injects, kept as the raw values + /// the block held. The adapter that builds the module deserializes them + /// into the vendor crate's own config type, so core never names a vendor + /// and a new module adds nothing here. + Injected(serde_json::Map), +} + +/// Configuration for the built-in HMAC Edge Cookie module. +/// +/// Mapped from the `[ec.hmac]` TOML block, or from a block under a label whose +/// `implementation` is `hmac`. Unknown keys are rejected, so a mistyped +/// setting fails at startup instead of being accepted silently and leaving the +/// intended setting at its default. +#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct HmacModuleConfig { + /// Publisher passphrase used as the HMAC key for EC generation. + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, +} + +/// Configuration for the built-in host-signal Edge Cookie module. +/// +/// Mapped from the `[ec.host_signals]` TOML block, or from a block under a +/// label whose `implementation` is `host_signals`. Unknown keys are rejected, +/// so a mistyped setting fails at startup instead of being accepted silently +/// and leaving the intended setting at its default. +#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct HostSignalsModuleConfig { + /// Passphrase used as the HMAC key over the host signals and client IP. + #[validate(custom(function = Ec::validate_passphrase))] + pub passphrase: Redacted, +} + +/// Device-detection configuration. +/// +/// Mapped from the `[device]` TOML section. Selects which device-detection +/// module classifies a request into device signals, mirroring the Edge +/// Cookie module selection in [`Ec`]. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct DeviceConfig { + /// The key of the device-detection module to activate. + /// + /// Defaults to the built-in `builtin` module when absent, which classifies + /// from the User-Agent alone, so device classification itself makes no + /// host-specific call. The opt-in `fastly` module strengthens the + /// browser/bot gate with the host's TLS and HTTP/2 signals, which the Fastly + /// entry point reads on every request regardless of this selector. Override + /// it with the + /// `TRUSTED_SERVER__device__module` environment variable so the same + /// compiled WebAssembly can switch modules at deployment. An unknown key is + /// rejected at startup by + /// [`validate_module_selection`](Self::validate_module_selection). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, +} + +impl DeviceConfig { + /// Returns the active device-detection module key, defaulting to the + /// built-in heuristic. + #[must_use] + pub fn module_key(&self) -> &str { + self.module.as_deref().unwrap_or("builtin") + } + + /// Validates the selected device-detection module. + /// + /// `builtin` and `fastly` are resolved by the core and the Fastly adapter. + /// Any other key names an integration module that declares a device + /// module, and the registry rejects a key no module supplies, so this + /// cannot be a closed list without shutting modules out of device detection + /// entirely. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] never, today. The error + /// type is kept because the caller treats module validation uniformly + /// across the three capabilities. + pub fn validate_module_selection(&self) -> Result<(), Report> { + Ok(()) + } +} + +/// Which permission signal modules run, and in what order. +/// +/// Mapped from the `[permission-signal]` TOML section, where `modules` +/// selects. `[ec]`, `[geo]` and `[device]` each name one module with +/// `module`, whereas signals compose, because a request can carry a TCF string +/// and a Global Privacy Control header at once and both have something to say. +/// So here `modules` names a list, and the order is the policy, because the +/// last module with an opinion decides. +/// +/// A module that gains settings will take them in a +/// `[permission-signal.]` block named for it. None of the modules that +/// ship has settings, so `modules` is the only key accepted, and any other key +/// is refused as an unknown field rather than silently ignored. +/// +/// See `crates/trusted-server-core/src/permission_signal/README.md`. +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Validate)] +pub struct PermissionSignalConfig { + /// The modules to run, in order, named by their crate folders below + /// `crates/permission-signal`, for example `gpc`, `gpp`, `us-privacy`, + /// `tcf` and `mtm` for the five that ship. + /// + /// Absent means every module the adapter offers, in the order it offers + /// them. A publisher who does not want to act on one removes it from the + /// list, and there is no separate switch, because a module that is not + /// listed does not run. An empty list runs none of them, leaving every + /// permission at its country and region baseline. + /// + /// Which names are valid is only known where the module crates are + /// linked, so the check that each name matches an available module and + /// none is repeated happens at the adapter's composition root, through + /// [`build_permission_signal_modules`], and refuses startup rather than + /// silently ignoring a typo. + /// + /// [`build_permission_signal_modules`]: + /// crate::permission_signal::build_permission_signal_modules + #[serde(skip_serializing_if = "Option::is_none")] + pub modules: Option>, +} + +/// Read by hand rather than derived, so that `sources` and `module`, the keys +/// `modules` replaced, are refused with a message saying what to write instead. A derived +/// struct could only refuse it by name by declaring it as a field, and would +/// then list it among the keys it expects whenever it refused any other. +impl<'de> Deserialize<'de> for PermissionSignalConfig { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let mut section = serde_json::Map::::deserialize(deserializer)?; + if section.contains_key("sources") { + return Err(serde::de::Error::custom( + "[permission-signal] sources is no longer accepted. Name the modules \ + to run, in order, in [permission-signal] modules instead", + )); + } + if section.contains_key("module") { + return Err(serde::de::Error::custom( + "[permission-signal] module is `modules` here, a list, because signals \ + compose. Name the modules to run, in order, in [permission-signal] \ + modules instead", + )); + } + if let Some(key) = section.keys().find(|key| key.as_str() != "modules") { + return Err(serde::de::Error::custom(format!( + "unknown field `{key}` in [permission-signal], expected `modules`. No \ + permission signal module takes settings yet, so a \ + [permission-signal.] block is not accepted" + ))); + } + // Read as an option, so an explicit JSON null is the same as leaving + // the key out. + let modules = match section.remove("modules") { + Some(value) => serde_json::from_value(value).map_err(serde::de::Error::custom)?, + None => None, + }; + Ok(Self { modules }) + } +} + +/// Which prefix module gives the start of every name the browser holds and +/// every key in a shared store. +/// +/// Mapped from the `[prefix]` TOML section. See [`crate::prefix`]. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrefixConfig { + /// The prefix module to run, named as any module is, by its crate folder. + /// + /// Core's own module runs when this is absent or names `ts`, giving `ts` + /// for the names the browser holds and `ts.` for shared + /// store keys. A name no module supplies is refused when the registry is + /// built, in + /// [`IntegrationRegistry::with_plan_and_registrations`](crate::integrations::IntegrationRegistry::with_plan_and_registrations). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, +} + +/// Geo / IP intelligence configuration. +/// +/// Mapped from the `[geo]` TOML section. Selects which module resolves a +/// client IP into [`GeoInfo`](crate::platform::GeoInfo), mirroring the Edge +/// Cookie module selection in [`Ec`]. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct GeoConfig { + /// The key of the geo module to activate. + /// + /// No module is the default: Trusted Server resolves no geolocation and + /// makes no host geo call, so a default deployment is not tied to any host + /// geo service, and the permission baseline comes from the top of the + /// `rules` tree in `permissions.yaml`. `module = "none"` spells + /// the same choice explicitly. The host platform's own geo lookup is + /// opt-in via `module = "platform"`. Override it with the + /// `TRUSTED_SERVER__geo__module` environment variable so the same compiled + /// WebAssembly can switch modules at deployment. An unknown key is rejected + /// at startup by + /// [`validate_module_selection`](Self::validate_module_selection). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, + + /// Acknowledges that, with no geo module, every request is treated as + /// coming from the place at the top of the `permissions.yaml` `rules` tree. + /// + /// With geolocation off, a visitor from any other jurisdiction silently + /// receives that top node's permission rules. A deployment that + /// runs an Edge Cookie module without a geo module must set this to + /// `true`, checked at startup by + /// [`validate_jurisdiction_acknowledgment`](Self::validate_jurisdiction_acknowledgment), + /// so serving a single jurisdiction is an explicit operator decision rather + /// than an accident of the default configuration. + #[serde(default)] + pub assume_single_jurisdiction: bool, +} + +impl GeoConfig { + /// Validates that the selected geo module is available in this build. + /// + /// No selector is valid and is the default, running without geolocation, + /// the same way the Edge Cookie module runs statelessly when none is + /// selected. The explicit `"none"` spells the same choice. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when the selected module + /// key is not one this build provides. + pub fn validate_module_selection(&self) -> Result<(), Report> { + // Unset, `none` and `platform` are resolved by the core. Any other key + // names an integration module that declares a geo module, and the + // registry rejects one that no module supplies, so this check cannot be + // a closed list without shutting modules out of geo entirely. + Ok(()) + } + + /// Validates that the compiled `permissions.yaml` parses and declares its + /// top node. + /// + /// The top node is required: its `group` is the permission baseline for a + /// request the geo module leaves unmatched, and its `jurisdiction` is the + /// consent handling for that same request, so there must always be one. + /// Checking it here turns a malformed policy into a configuration error at + /// startup rather than a panic on the first lookup. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when the compiled policy + /// fails to parse, most usefully when its top node omits `group` or + /// `jurisdiction`. + pub fn validate_permission_policy() -> Result<(), Report> { + crate::permissions::validate_default_policy().map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("permissions.yaml is not usable: {error}"), + }) + }) + } + + /// Validates that running jurisdiction consumers without geolocation is + /// explicitly acknowledged. + /// + /// With no geo module, every request resolves to the permission baseline + /// at the top of the `permissions.yaml` `rules` tree, so a visitor from any + /// other jurisdiction silently receives that node's rules. That is + /// acceptable only as an explicit operator + /// decision. When an Edge Cookie module is configured (the permission + /// model gates it by jurisdiction) and no geo module is selected, + /// [`assume_single_jurisdiction`](Self::assume_single_jurisdiction) must be + /// `true`. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when an Edge Cookie + /// module is configured, no geo module is selected, and + /// `assume_single_jurisdiction` is not set. + pub fn validate_jurisdiction_acknowledgment( + &self, + ec: &Ec, + ) -> Result<(), Report> { + // Location is resolved by the host lookup (`platform`) or by any + // integration module that declares a geo module. Only an unset + // selector and the explicit `none` resolve nothing, so only those + // two leave every request on the declared jurisdiction. + let geo_disabled = matches!(self.module.as_deref(), None | Some("none")); + // The selector is a typed enum, so statelessness is the absent + // selector or the explicit `none`, matched rather than compared as a + // string. + let ec_active = !matches!(ec.module, None | Some(EcModuleSelection::None)); + if geo_disabled && ec_active && !self.assume_single_jurisdiction { + return Err(Report::new(TrustedServerError::Configuration { + message: "[ec] module is configured but no [geo] module is selected, so \ + every request would be treated as the top of the permissions.yaml \ + rules tree. Set [geo] assume_single_jurisdiction = true to \ + acknowledge single-jurisdiction operation, or select a geo module" + .to_owned(), + })); + } + Ok(()) + } +} + +#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct Rewrite { + /// List of domains to exclude from rewriting. Supports wildcards (e.g., "*.example.com"). + /// URLs from these domains will not be proxied through first-party endpoints. + #[serde(default)] + pub exclude_domains: Vec, +} + +impl Rewrite { + /// Checks if a URL should be excluded from rewriting based on domain matching + #[allow(dead_code)] + #[must_use] + pub fn is_excluded(&self, url: &str) -> bool { + // Parse URL to extract host + let Ok(parsed) = url::Url::parse(url) else { + return false; + }; + + let host = parsed.host_str().unwrap_or(""); + + // Check exact domain matches (with wildcard support) + for domain in &self.exclude_domains { + if let Some(suffix) = domain.strip_prefix("*.") { + // Wildcard: *.example.com matches both example.com and sub.example.com + if host == suffix || host.ends_with(&format!(".{}", suffix)) { + return true; + } + } else if host == domain { + return true; + } + } + + false + } +} + +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RequestSigning { + #[serde(default = "default_request_signing_enabled")] + pub enabled: bool, + /// The removed `config_store_id`. The field never holds a value, because + /// reading one always fails. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed key fails with directions" + )] + config_store_id: RemovedSigningStoreId, + /// The removed `secret_store_id`, refused as `config_store_id` is. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed key fails with directions" + )] + secret_store_id: RemovedSigningStoreId, +} + +impl RequestSigning { + /// Request signing switched on or off. #[must_use] - pub fn is_unusable_store_id(store_id: &str) -> bool { - Self::is_placeholder_store_id(store_id) || store_id != store_id.trim() + pub fn new(enabled: bool) -> Self { + Self { + enabled, + ..Self::default() + } } } @@ -1528,6 +2401,7 @@ pub struct ProxyAssetRoute { /// Only the scheme, host, and port are used. Any path or query configured on /// this URL is rejected because the incoming request path/query, or the /// configured rewrite result, replaces them at runtime. + #[serde(serialize_with = "crate::redacted::sensitive")] pub origin_url: String, /// Optional regex matched against the incoming request path before proxying. pub path_pattern: Option, @@ -1760,7 +2634,6 @@ impl ProxyAssetRoute { } #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] pub struct Proxy { /// Enable TLS certificate verification when proxying to HTTPS origins. /// Defaults to true for secure production use. @@ -1776,33 +2649,30 @@ pub struct Proxy { /// /// When empty (the default), proxy hosts are not restricted. Configure this /// in production to constrain signed and fetched first-party proxy targets. - /// When `integrations.prebid.external_bundle_url` is configured, this list + /// When `auction.prebid.external_bundle_url` is configured, this list /// must include its host and any HTTPS redirect targets. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub allowed_domains: Vec, /// Path-prefix-based asset proxy routes evaluated before publisher fallback. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub asset_routes: Vec, + /// The modules this section selects, with each one's settings in the + /// table at its name. + #[serde(flatten)] + pub modules: SectionModules, } fn default_certificate_check() -> bool { true } -fn is_admin_placeholder_password(password: &str) -> bool { - Handler::is_placeholder_password(password) - || matches!( - password.trim().to_ascii_lowercase().as_str(), - "changeme" | "password" | "admin" - ) -} - impl Default for Proxy { fn default() -> Self { Self { certificate_check: default_certificate_check(), allowed_domains: Vec::new(), asset_routes: Vec::new(), + modules: SectionModules::default(), } } } @@ -1897,12 +2767,15 @@ impl Proxy { } } -/// Direct Tinybird Events API telemetry configuration. +/// The analytics module's name in full, which is also the table holding its +/// settings and which every message about them names. +const ANALYTICS_MODULE: &str = "analytics.tinybird"; + +/// Direct Tinybird Events API telemetry configuration, the settings of the +/// module `[analytics]` selects. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct TinybirdSettings { - /// Master enablement for auction telemetry ingestion. - #[serde(default)] - pub enabled: bool, /// Regional Tinybird API host, without scheme or path. #[serde(default)] pub api_host: String, @@ -1950,7 +2823,6 @@ fn default_tinybird_max_body_bytes() -> usize { impl Default for TinybirdSettings { fn default() -> Self { Self { - enabled: false, api_host: String::new(), secret_store: None, auction_dataset: default_tinybird_auction_dataset(), @@ -1969,7 +2841,7 @@ impl TinybirdSettings { self.api_host = self.api_host.trim().to_ascii_lowercase(); if self.secret_store.take().is_some() { log::warn!( - "tinybird.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + "{ANALYTICS_MODULE}.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" ); } self.auction_dataset = self.auction_dataset.trim().to_owned(); @@ -1985,32 +2857,39 @@ impl TinybirdSettings { self.normalize(); if !(0.0..=1.0).contains(&self.access_sample_rate) { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.access_sample_rate must be between 0.0 and 1.0".to_owned(), + message: format!( + "{ANALYTICS_MODULE}.access_sample_rate must be between 0.0 and 1.0" + ), })); } if self.max_body_bytes < 1024 { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.max_body_bytes must be at least 1024".to_owned(), + message: format!("{ANALYTICS_MODULE}.max_body_bytes must be at least 1024"), })); } if self.access_enabled { return Err(Report::new(TrustedServerError::Configuration { - message: "tinybird.access_enabled is reserved for future access-log telemetry; no emitter is currently wired".to_owned(), + message: format!( + "{ANALYTICS_MODULE}.access_enabled is reserved for future access-log telemetry; no emitter is currently wired" + ), })); } - if !self.enabled { - return Ok(()); - } validate_tinybird_api_host(&self.api_host)?; - validate_tinybird_dataset(&self.auction_dataset, "tinybird.auction_dataset")?; + validate_tinybird_dataset( + &self.auction_dataset, + &format!("{ANALYTICS_MODULE}.auction_dataset"), + )?; let token = self.auction_token_secret.as_ref().ok_or_else(|| { Report::new(TrustedServerError::Configuration { - message: - "tinybird.auction_token_secret is required when Tinybird telemetry is enabled" - .to_owned(), + message: format!( + "{ANALYTICS_MODULE}.auction_token_secret is required when [analytics] selects the module" + ), }) })?; - validate_tinybird_secret(token.expose(), "tinybird.auction_token_secret") + validate_tinybird_secret( + token.expose(), + &format!("{ANALYTICS_MODULE}.auction_token_secret"), + ) } } @@ -2023,13 +2902,14 @@ fn validate_tinybird_api_host(host: &str) -> Result<(), Report Result<(), Report, + /// The analytics module's settings. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tinybird: Option, +} + +/// Whether `written`, selected in `[analytics]`, names the analytics module, +/// in full or by its name within the section. +fn names_analytics_module(written: &str) -> bool { + written == ANALYTICS_MODULE + || written == crate::module_name::short_form("analytics", ANALYTICS_MODULE) +} + +/// Whether the document `data` selects the analytics module in `[analytics]`, +/// read before the settings are, to tell which secret references are live. +pub(crate) fn document_selects_analytics_module(data: &JsonValue) -> bool { + data.pointer("/analytics/module") + .and_then(JsonValue::as_str) + .is_some_and(names_analytics_module) +} + +impl AnalyticsConfig { + /// Whether `module` selects the analytics module. + fn selects_analytics_module(&self) -> bool { + self.module.as_deref().is_some_and(names_analytics_module) + } + + /// The analytics module's settings, when `module` selects it. + #[must_use] + pub fn analytics_module(&self) -> Option<&TinybirdSettings> { + if self.selects_analytics_module() { + self.tinybird.as_ref() + } else { + None + } + } + + fn normalize(&mut self) { + if let Some(settings) = &mut self.tinybird { + settings.normalize(); + } + } + + /// Refuses a section that selects nothing or a module this build does + /// not have, and checks the selected module's settings. + fn prepare_runtime(&mut self) -> Result<(), Report> { + let short = crate::module_name::short_form("analytics", ANALYTICS_MODULE); + match self.module.as_deref() { + None => { + return Err(Report::new(TrustedServerError::Configuration { + message: "[analytics] selects no module. A module type's section names \ + what runs with `module` or `modules`, so select one or remove \ + the section" + .to_owned(), + })); + } + Some(written) if !self.selects_analytics_module() => { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[analytics] selects `{written}`, which no module this deployment \ + supplies. The analytics modules it supplies are [{short}]" + ), + })); + } + Some(_) => {} + } + match &mut self.tinybird { + Some(settings) => settings.prepare_runtime(), + // Checking the defaults reports the first setting the module + // needs and the table does not have. + None => TinybirdSettings::default().prepare_runtime(), + } + } +} + +/// Cache behavior configuration. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CacheSettings { + /// Ordered static/rehosted asset rules. The first enabled matching rule wins. #[serde(default)] pub asset_rules: Vec, + /// How long the copies of each robots.txt contributor's answer last, + /// `[cache.robots-txt]`. + #[serde( + rename = "robots-txt", + default, + skip_serializing_if = "LoadingCacheConfig::is_default" + )] + pub robots_txt: LoadingCacheConfig, +} + +/// The most seconds a loading cache lifetime setting accepts, a year. +pub const MAX_LOADING_CACHE_SECONDS: u32 = 365 * 24 * 60 * 60; + +/// The name of the cache that holds each robots.txt module's answer, whose +/// settings are `[cache.robots-txt]`. +pub const ROBOTS_TXT_CACHE: &str = "robots-txt"; + +/// How long one loading cache's copies last, each field defaulting to the +/// cache's own default when absent. +/// +/// See [`crate::platform::PlatformLoadingCache`]. The memory copy is the brief +/// one a process keeps in front of the shared store, and the shared store's +/// copy is the one every instance reads. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct LoadingCacheConfig { + /// The module whose loading cache holds this cache's copies, as its short + /// name or in full, for example `51degrees` or `loading-cache.51degrees`. + /// Absent, no copy is kept and every call loads. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub module: Option, + /// How long the brief copy in process memory is used, in seconds. Zero + /// keeps no memory copy, and a missing value is still loaded once. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_seconds: Option, + /// How long the copy in the shared store is used at most, in seconds. An + /// answer with a shorter lifetime of its own ends sooner. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_seconds: Option, + /// How long the copy in the shared store lasts unused, in seconds, after + /// which it leaves the store. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_idle_seconds: Option, +} + +impl LoadingCacheConfig { + /// Whether nothing is set, so every lifetime is the cache's default. + #[must_use] + pub fn is_default(&self) -> bool { + *self == Self::default() + } + + /// The lifetimes these settings give, each from `defaults` where unset. + #[must_use] + pub fn lifetimes( + &self, + defaults: crate::platform::CacheLifetimes, + ) -> crate::platform::CacheLifetimes { + let seconds = |value: Option, default: Duration| { + value.map_or(default, |value| Duration::from_secs(u64::from(value))) + }; + crate::platform::CacheLifetimes { + memory: seconds(self.memory_seconds, defaults.memory), + store: seconds(self.store_seconds, defaults.store), + store_idle: seconds(self.store_idle_seconds, defaults.store_idle), + } + } + + /// Refuses a lifetime out of range, naming the setting by `table`. + /// + /// # Errors + /// + /// When the memory lifetime is over [`MAX_LOADING_CACHE_SECONDS`], or the + /// store lifetime or idle time is zero or over it. + pub fn check(&self, table: &str) -> Result<(), Report> { + check_loading_cache_seconds(table, "memory_seconds", self.memory_seconds, 0)?; + check_loading_cache_seconds(table, "store_seconds", self.store_seconds, 1)?; + check_loading_cache_seconds(table, "store_idle_seconds", self.store_idle_seconds, 1) + } +} + +fn check_loading_cache_seconds( + table: &str, + field: &str, + value: Option, + least: u32, +) -> Result<(), Report> { + match value { + Some(value) if value < least || value > MAX_LOADING_CACHE_SECONDS => { + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "[{table}] {field} is {value}, and must be from {least} to \ + {MAX_LOADING_CACHE_SECONDS}" + ), + })) + } + _ => Ok(()), + } } impl CacheSettings { + /// Each cache a loading cache may hold, by its name, with the settings + /// its `[cache.]` table carries. + #[must_use] + pub fn loading_caches(&self) -> [(&'static str, &LoadingCacheConfig); 1] { + [(ROBOTS_TXT_CACHE, &self.robots_txt)] + } + fn normalize(&mut self) { for rule in &mut self.asset_rules { rule.normalize(); @@ -2077,9 +3143,11 @@ impl CacheSettings { /// /// # Errors /// - /// Returns a configuration error if any rule ID is duplicate, or if an - /// enabled rule has an invalid policy/matcher or cannot compile its regex/glob. + /// Returns a configuration error if a loading cache lifetime is out of + /// range, if any rule ID is duplicate, or if an enabled rule has an + /// invalid policy/matcher or cannot compile its regex/glob. pub fn prepare_runtime(&self) -> Result<(), Report> { + self.robots_txt.check("cache.robots-txt")?; let mut seen_ids = HashSet::new(); for rule in &self.asset_rules { if rule.id.is_empty() { @@ -2672,6 +3740,126 @@ fn is_default_auction_debug_comment_options(value: &AuctionDebugCommentOptions) *value == AuctionDebugCommentOptions::default() } +// The module selectors are new sections, so a serialized blob that carries +// them is rejected by a base-revision binary that has never heard of them. +// Omitting the default table keeps an unchanged `ts config push` readable +// across a rollout or a rollback. +fn is_default_device_config(value: &DeviceConfig) -> bool { + *value == DeviceConfig::default() +} + +fn is_default_geo_config(value: &GeoConfig) -> bool { + *value == GeoConfig::default() +} + +fn is_default_prefix_config(value: &PrefixConfig) -> bool { + *value == PrefixConfig::default() +} + +fn is_default_permission_signal_config(value: &PermissionSignalConfig) -> bool { + *value == PermissionSignalConfig::default() +} + +/// Message a configuration still carrying the removed `[integrations]` table +/// is rejected with. +const REMOVED_INTEGRATIONS_TABLE_MESSAGE: &str = "Configuration table `[integrations]` was removed. Each module is selected in the \ + section of its type, as [] module = \"\" or modules = [\"\", ...], with its \ + settings in the table at its name, [.], as described in the CHANGELOG.md \ + breaking migration"; + +/// The removed `[integrations]` table. +/// +/// Reading one always fails, with [`REMOVED_INTEGRATIONS_TABLE_MESSAGE`], so +/// the value is never held and the type carries no data. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct RemovedIntegrationsTable; + +impl<'de> Deserialize<'de> for RemovedIntegrationsTable { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // The value is read and discarded first so that a table, a string or + // anything else all reach the same message. Reporting a type error + // instead would send an operator looking for a type problem in a table + // that has simply moved. + serde::de::IgnoredAny::deserialize(deserializer)?; + Err(serde::de::Error::custom(REMOVED_INTEGRATIONS_TABLE_MESSAGE)) + } +} + +/// A table that is no longer read under its old name, refused with directions +/// to its new one when a configuration still carries it. +macro_rules! refused_table { + ($(#[$doc:meta])* $name:ident => $message:expr) => { + $(#[$doc])* + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + pub struct $name; + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + // The value is read and discarded first so that a table, a + // string or anything else all reach the same message. A type + // error instead would send an operator looking for a type + // problem in a table that has simply moved. + serde::de::IgnoredAny::deserialize(deserializer)?; + Err(serde::de::Error::custom($message)) + } + } + }; +} + +refused_table! { + /// The `[adserver]` table, renamed `[ad-server]`. + RenamedAdServerTable => "Configuration table `[adserver]` is now `[ad-server]`. Select the \ + ad server with `[ad-server] module` and move its settings to `[ad-server.]` \ + unchanged" +} + +refused_table! { + /// The `[integration]` table, which is no longer read. + RemovedIntegrationTable => "Configuration table `[integration]` is no longer read. Each \ + module is selected in the section of its type, as [] module = \"\" or \ + modules = [\"\", ...], with its settings in the table at its name, \ + [.]" +} + +refused_table! { + /// The `[[handlers]]` list, which gated paths behind HTTP Basic + /// authentication. + RemovedHandlers => "Configuration list `[[handlers]]` is not read, and no request is \ + asked for a password. Remove every `[[handlers]]` entry. Nothing is served beneath \ + /_ts/admin, so rotate signing keys with `ts keys`, purge the cache with \ + `ts cache purge`, and put a password on a path of the publisher's own at its origin \ + or in the hosting platform" +} + +refused_table! { + /// A store id under `[request_signing]`, which the service does not read. + RemovedSigningStoreId => "`[request_signing]` does not take `config_store_id` or \ + `secret_store_id`. The service reads its signing keys from the stores linked as \ + `jwks_store` and `signing_keys` and writes neither. Remove both keys, and give the \ + ids to `ts keys rotate` and `ts keys deactivate` as `--config-store-id` and \ + `--secret-store-id`" +} + +refused_table! { + /// The `[tinybird]` table, now `[analytics.tinybird]`. + MovedTinybirdTable => "Configuration table `[tinybird]` is now `[analytics.tinybird]`. \ + Select it with `[analytics] module = \"tinybird\"`, which replaces its `enabled` \ + line, and move its other settings there unchanged" +} + +refused_table! { + /// The `[permission_signal]` table, renamed `[permission-signal]`. + RenamedPermissionSignalTable => "Configuration table `[permission_signal]` is now \ + `[permission-signal]`, named exactly as its folder crates/permission-signal. Move its \ + settings there unchanged" +} + /// Behavior of the `` auction dump. Only consulted when /// [`DebugConfig::auction_html_comment`] is true. /// @@ -2686,9 +3874,9 @@ pub struct AuctionDebugCommentOptions { #[serde(default = "default_true")] pub include_provider_responses: bool, - /// Include `mediator_response` when a mediator ran. + /// Include `adserver_response` when an ad server ran. #[serde(default = "default_true")] - pub include_mediator_response: bool, + pub include_adserver_response: bool, /// Include each provider's `bids` array (vs. status/metadata only). #[serde(default = "default_true")] @@ -2730,7 +3918,7 @@ impl Default for AuctionDebugCommentOptions { fn default() -> Self { Self { include_provider_responses: true, - include_mediator_response: true, + include_adserver_response: true, include_bids: true, metadata_keys: default_auction_debug_metadata_keys(), verbosity: AuctionDebugCommentVerbosity::Redacted, @@ -2955,7 +4143,6 @@ fn validate_trusted_client_ip_shared_secret( } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] -#[serde(deny_unknown_fields)] pub struct Settings { #[validate(nested)] pub publisher: Publisher, @@ -2973,14 +4160,54 @@ pub struct Settings { #[serde(default, skip_serializing_if = "Option::is_none")] #[validate(nested)] pub trusted_client_ip: Option, - #[serde(default)] + /// Optional `robots.txt`, assembled by core from the contributors + /// `[robots-txt] modules` selects. + /// + /// `None` leaves `/robots.txt` to the publisher's origin, and must stay + /// omitted from serialized config blobs for the reason given on + /// `trusted_client_ip`. + #[serde( + rename = "robots-txt", + default, + skip_serializing_if = "Option::is_none" + )] #[validate(nested)] - pub ec: Ec, + pub robots_txt: Option, #[serde(default)] - pub integrations: IntegrationSettings, - #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] - pub handlers: Vec, + pub ec: Ec, + /// The removed `[integrations]` table. + /// + /// The name is kept so a configuration written for the previous release is + /// told where its blocks moved, rather than being handed a bare + /// unknown-field error listing every table Trusted Server accepts. The + /// field never holds a value, because reading one always fails. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed table fails with directions" + )] + integrations: RemovedIntegrationsTable, + /// The `[integration]` table, kept so a configuration carrying it is + /// told where modules are selected now. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed table fails with directions" + )] + integration: RemovedIntegrationTable, + /// The sections of module types core does not read itself, such as + /// `[cmp]` or `[tag]`, each named for the type of module it selects. + #[serde(flatten)] + pub sections: TypeSections, + /// The `[[handlers]]` list, kept so a configuration carrying one is + /// told that no path is gated by a password. + #[serde(default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the removed list fails with directions" + )] + handlers: RemovedHandlers, #[serde(default, deserialize_with = "map_from_obj_or_str")] pub response_headers: HashMap, pub request_signing: Option, @@ -2990,23 +4217,123 @@ pub struct Settings { #[serde(default)] #[validate(nested)] pub auction: AuctionConfig, + /// The auction's demand sources. `[demand] modules` selects them and each + /// `[demand.]` table holds one source's settings. + #[serde(default, skip_serializing_if = "ProviderList::is_unset")] + pub demand: ProviderList, + /// The ad server that picks the auction winner. `[ad-server] module` + /// selects it and `[ad-server.]` holds its settings. + #[serde( + rename = "ad-server", + default, + skip_serializing_if = "ProviderChoice::is_unset" + )] + pub adserver: ProviderChoice, + /// The `[adserver]` table, kept so a configuration carrying it is told + /// its new name. + #[serde(rename = "adserver", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the renamed table fails with directions" + )] + renamed_adserver: RenamedAdServerTable, #[serde(default)] pub consent: ConsentConfig, #[serde(default)] pub cache: CacheSettings, + /// What one vendor's modules share, a table per vendor, `[shared.]`. + #[serde(default, skip_serializing_if = "SharedTables::is_empty")] + pub shared: SharedTables, #[serde(default)] pub proxy: Proxy, #[serde(default)] pub creative_opportunities: Option, #[serde(default)] pub image_optimizer: ImageOptimizerSettings, - #[serde(default)] - pub tinybird: TinybirdSettings, + /// Analytics. `[analytics] module` selects the module that runs, with its + /// settings in the table at its name. Omitted from a serialized + /// configuration when absent, for the reason given on `trusted_client_ip`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub analytics: Option, + /// The `[tinybird]` table, kept so a configuration carrying it is told + /// where it moved. + #[serde(rename = "tinybird", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the moved table fails with directions" + )] + moved_tinybird: MovedTinybirdTable, #[serde(default)] pub debug: DebugConfig, + #[serde(default, skip_serializing_if = "is_default_device_config")] + #[validate(nested)] + pub device: DeviceConfig, + #[serde(default, skip_serializing_if = "is_default_geo_config")] + #[validate(nested)] + pub geo: GeoConfig, + /// The prefix module, `[prefix] module`, see [`crate::prefix`]. + #[serde(default, skip_serializing_if = "is_default_prefix_config")] + #[validate(nested)] + pub prefix: PrefixConfig, + #[serde( + rename = "permission-signal", + default, + skip_serializing_if = "is_default_permission_signal_config" + )] + #[validate(nested)] + pub permission_signal: PermissionSignalConfig, + /// The `[permission_signal]` table, kept so a configuration carrying it is + /// told its new name. + #[serde(rename = "permission_signal", default, skip_serializing)] + #[allow( + dead_code, + reason = "the field exists so that reading the renamed table fails with directions" + )] + renamed_permission_signal: RenamedPermissionSignalTable, + /// What the configuration page at `/_ts/config` shows. + #[serde( + default, + skip_serializing_if = "crate::inspect::config::InspectConfig::is_default" + )] + pub inspect: crate::inspect::config::InspectConfig, + /// The attestation endpoint, which serves evidence signed with the + /// operator's key at the address its `endpoint` names. See + /// [`crate::attestation`]. + /// + /// `None` leaves the address to the publisher's origin, and stays out of + /// serialized config blobs for the reason given on `trusted_client_ip`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(nested)] + pub attestation: Option, + /// The page changes run on a document fetched from the origin, written + /// as `[[fetch]]` entries. See [`crate::middleware`]. + #[serde(default, skip_serializing_if = "PhaseEntries::is_empty")] + pub fetch: PhaseEntries, + /// The page changes run on each reader's copy of a document, written as + /// `[[serve]]` entries. + #[serde(default, skip_serializing_if = "PhaseEntries::is_empty")] + pub serve: PhaseEntries, + /// Where the loader wrote secrets, which the configuration view masks. + /// Never read from a document and never written to one. + #[serde(skip)] + resolved_secrets: crate::secret_resolution::ResolvedSecrets, } impl Settings { + /// Where the loader wrote secrets, for the configuration view. + #[must_use] + pub(crate) fn resolved_secrets(&self) -> &crate::secret_resolution::ResolvedSecrets { + &self.resolved_secrets + } + + /// Records where the loader wrote secrets. + pub(crate) fn set_resolved_secrets( + &mut self, + secrets: crate::secret_resolution::ResolvedSecrets, + ) { + self.resolved_secrets = secrets; + } + /// Creates a new [`Settings`] instance from a TOML string. /// /// # Errors @@ -3051,9 +4378,15 @@ impl Settings { /// - [`TrustedServerError::Configuration`] if the TOML is invalid or missing required fields #[cfg(test)] pub fn from_toml_and_env(toml_str: &str) -> Result> { + // The build script compiles the git version in through this variable, + // and cargo sets it on a test process too. It is not a setting. + let variables = std::env::vars() + .filter(|(name, _)| name != "TRUSTED_SERVER__GIT_VERSION") + .collect(); let environment = Environment::default() .prefix(ENVIRONMENT_VARIABLE_PREFIX) - .separator(ENVIRONMENT_VARIABLE_SEPARATOR); + .separator(ENVIRONMENT_VARIABLE_SEPARATOR) + .source(Some(variables)); let toml = File::from_str(toml_str, FileFormat::Toml); let config = Config::builder() @@ -3073,6 +4406,14 @@ impl Settings { Self::finalize_deserialized(settings, "Build-time configuration") } + /// The analytics module's settings, when `[analytics]` selects it. + #[must_use] + pub fn analytics_module(&self) -> Option<&TinybirdSettings> { + self.analytics + .as_ref() + .and_then(AnalyticsConfig::analytics_module) + } + /// Normalize settings after deserialization. /// /// # Errors @@ -3083,9 +4424,9 @@ impl Settings { self.proxy.normalize(); self.image_optimizer.normalize()?; self.debug.auction_html_comment_options.normalize(); - self.tinybird.normalize(); - self.integrations - .remove_legacy_static_secret_store_selectors(); + if let Some(analytics) = &mut self.analytics { + analytics.normalize(); + } self.consent.validate(); Ok(()) } @@ -3106,8 +4447,33 @@ impl Settings { }) })?; - settings.validate_admin_coverage()?; - settings.validate_admin_handler_passwords()?; + settings.ec.migrate_legacy_ec_layout()?; + settings.ec.validate_module_selection()?; + settings.ec.validate_resolve_allowed_origins()?; + settings.validate_module_sections()?; + settings.validate_phase_entries()?; + settings.device.validate_module_selection()?; + settings.geo.validate_module_selection()?; + GeoConfig::validate_permission_policy()?; + settings + .geo + .validate_jurisdiction_acknowledgment(&settings.ec)?; + + // Log the policy's declared default once per settings load, so an + // operator can see which permissions an unmatched request is granted + // without a signal, and which jurisdiction its consent gates apply. + let maps = crate::permissions::PermissionMaps::standard(); + let granted: Vec = maps + .baseline(None, None) + .permissions() + .iter() + .map(|permission| permission.to_string()) + .collect(); + log::debug!( + "Permission baseline: permissions.yaml top node, jurisdiction {}; granted without a signal: [{}]", + maps.default_jurisdiction(), + granted.join(", ") + ); Ok(settings) } @@ -3124,14 +4490,17 @@ impl Settings { self.image_optimizer.prepare_runtime()?; self.cache.prepare_runtime()?; self.proxy.prepare_runtime()?; - self.tinybird.prepare_runtime()?; + if let Some(analytics) = &mut self.analytics { + analytics.prepare_runtime()?; + } self.debug .auction_html_comment_options .validate_metadata_keys()?; self.validate_asset_image_optimizer_profile_sets()?; - - for handler in &self.handlers { - handler.prepare_runtime()?; + self.shared.validate_names()?; + if let Some(robots_txt) = &self.robots_txt { + robots_txt.prepare_runtime()?; + self.require_robots_txt_placed()?; } if let Some(co) = &mut self.creative_opportunities { @@ -3169,6 +4538,59 @@ impl Settings { Ok(()) } + /// Refuses a `[robots-txt]` section unless a `[[fetch]]` entry for + /// `text/plain` with `path = "/robots.txt"` names the `robots_txt` + /// middleware, and refuses any entry naming it on another path or media + /// type. Only a `[[fetch]]` entry makes the file where the origin has + /// none, and an entry on another path would replace that file with the + /// robots.txt rules. + fn require_robots_txt_placed(&self) -> Result<(), Report> { + use crate::robots_txt::ROBOTS_TXT_PATH; + use crate::robots_txt::middleware::{MEDIA_TYPE, MODULE}; + + let names = |entry: &crate::middleware::PhaseEntry| { + entry.middleware.iter().any(|name| name == MODULE) + }; + let on_the_file = |entry: &crate::middleware::PhaseEntry| { + entry.media_type == MEDIA_TYPE && entry.path.as_deref() == Some(ROBOTS_TXT_PATH) + }; + for (phase, entries) in [("fetch", &self.fetch), ("serve", &self.serve)] { + for (index, entry) in entries.entries().iter().enumerate() { + if names(entry) && !on_the_file(entry) { + return Err(Report::new(TrustedServerError::Settings { + message: format!( + "[[{phase}]] entry {} names `{MODULE}` for `{}` on {}, where it \ + would replace each file it covers with robots.txt. Name it only \ + in an entry with media_type = `{MEDIA_TYPE}` and path = \ + `{ROBOTS_TXT_PATH}`", + index + 1, + entry.media_type, + entry.path.as_deref().map_or_else( + || "every path".to_owned(), + |path| format!("`{path}`") + ), + ), + })); + } + } + } + if self + .fetch + .entries() + .iter() + .any(|entry| names(entry) && on_the_file(entry)) + { + return Ok(()); + } + Err(Report::new(TrustedServerError::Settings { + message: format!( + "[robots-txt] is set, but no [[fetch]] entry with media_type = `{MEDIA_TYPE}` and \ + path = `{ROBOTS_TXT_PATH}` names `{MODULE}`, so the file would never be made \ + where the origin has none. Add that entry" + ), + })) + } + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( @@ -3190,8 +4612,15 @@ impl Settings { pub fn reject_placeholder_secrets(&self) -> Result<(), Report> { let mut insecure_fields: Vec = Vec::new(); - if Ec::is_placeholder_passphrase(self.ec.passphrase.expose()) { - insecure_fields.push("ec.passphrase".to_owned()); + for (name, hmac) in self.ec.module_blocks.hmac_blocks() { + if Ec::is_placeholder_passphrase(hmac.passphrase.expose()) { + insecure_fields.push(format!("ec.{name}.passphrase")); + } + } + for (name, host_signals) in self.ec.module_blocks.host_signals_blocks() { + if Ec::is_placeholder_passphrase(host_signals.passphrase.expose()) { + insecure_fields.push(format!("ec.{name}.passphrase")); + } } if Publisher::is_placeholder_proxy_secret(self.publisher.proxy_secret.expose()) { insecure_fields.push("publisher.proxy_secret".to_owned()); @@ -3222,11 +4651,6 @@ impl Settings { )); } } - for handler in &self.handlers { - if Handler::is_placeholder_password(handler.password.expose()) { - insecure_fields.push(format!("handlers[{}].password", handler.path)); - } - } if Publisher::is_placeholder_domain(&self.publisher.domain) { insecure_fields.push("publisher.domain".to_owned()); } @@ -3236,22 +4660,6 @@ impl Settings { if Publisher::is_placeholder_origin_url(&self.publisher.origin_url) { insecure_fields.push("publisher.origin_url".to_owned()); } - // Checked whenever the block is present, not just when it is enabled: - // the key rotate/deactivate admin routes are registered unconditionally - // and read these store IDs without consulting `enabled`, so placeholder - // IDs behind a disabled block would still reach key management at - // runtime. Surrounding whitespace is rejected too: the placeholder check - // trims for comparison but the raw value is what `signing_store_ids` - // forwards to `KeyRotationManager`, so a padded id would validate yet - // reach the management API unusable. - if let Some(request_signing) = &self.request_signing { - if RequestSigning::is_unusable_store_id(&request_signing.config_store_id) { - insecure_fields.push("request_signing.config_store_id".to_owned()); - } - if RequestSigning::is_unusable_store_id(&request_signing.secret_store_id) { - insecure_fields.push("request_signing.secret_store_id".to_owned()); - } - } if insecure_fields.is_empty() { return Ok(()); @@ -3306,172 +4714,223 @@ impl Settings { self.proxy.asset_route_for_path(path) } - /// Resolve the first handler whose regex matches the request path. - /// - /// # Errors - /// - /// Returns a configuration error if any handler regex does not compile. - pub fn handler_for_path( - &self, - path: &str, - ) -> Result, Report> { - for handler in &self.handlers { - if handler.matches_path(path)? { - return Ok(Some(handler)); - } - } - - Ok(None) + /// Every section that selects modules, with its name: `[proxy]`, + /// `[auction]` and each module type's own section. + pub fn module_sections(&self) -> impl Iterator { + [ + ("proxy", &self.proxy.modules), + ("auction", &self.auction.modules), + ] + .into_iter() + .chain(self.sections.iter()) } - /// Returns whether `path` is within the reserved Trusted Server admin - /// namespace. + /// The section that selects the module `name`, and the name as written + /// there, whether in full or with the section's type folder left off. #[must_use] - pub(crate) fn is_admin_path(path: &str) -> bool { - path == "/_ts/admin" || path.starts_with("/_ts/admin/") - } - - /// Known admin endpoint paths that must be covered by a handler. - /// - /// [`from_toml`](Self::from_toml) rejects configurations - /// where any of these paths lack a matching handler, ensuring admin - /// endpoints are always protected by authentication. - /// Update [`ADMIN_ENDPOINTS`](Self::ADMIN_ENDPOINTS) when adding new - /// admin routes to `crates/trusted-server-adapter-fastly/src/app.rs`. - /// - /// The `/_ts/admin/ec/{id}` entry is the canonical router pattern. Its - /// coverage is checked via [`admin_auth_probes`](Self::admin_auth_probes), - /// while validation errors continue to report this operator-facing route - /// template. - pub(crate) const ADMIN_ENDPOINTS: &[&str] = &[ - "/_ts/admin/keys/rotate", - "/_ts/admin/keys/deactivate", - "/_ts/admin/ec", - "/_ts/admin/ec/{id}", - "/_ts/admin/eids", - "/_ts/admin/cache/purge", - ]; - - /// Probes that establish handler coverage for the dynamic - /// `/_ts/admin/ec/{id}` route. - /// - /// Coverage cannot be sampled: the router accepts any single segment after - /// `/_ts/admin/ec/` and basic auth runs on the raw path before routing, so - /// a handler that matches only some ID shapes leaves the rest of the route - /// surface — including malformed IDs, which still reach the admin handler — - /// unauthenticated at configuration time and fail-closed at runtime. - /// - /// Both probes must match the same configuration for the route to count as - /// covered. The bare prefix rejects handlers anchored to specific ID - /// shapes; the concrete ID rejects handlers anchored to the prefix itself - /// (`^/_ts/admin/ec/$`). Together they admit only prefix-level matchers - /// such as `^/_ts/admin` or `^/_ts/admin/ec/`. - const ADMIN_EC_ID_AUTH_PROBES: [&str; 2] = [ - "/_ts/admin/ec/", - concat!( - "/_ts/admin/ec/", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ".Ab12Z9", - ), - ]; + pub fn module_selection(&self, name: &str) -> Option<(&str, &str)> { + self.module_sections().find_map(|(section, modules)| { + modules + .selected() + .iter() + .find(|written| crate::module_name::resolve(section, written, &[name]).is_some()) + .map(|written| (section, written.as_str())) + }) + } - fn admin_auth_probes(path: &'static str) -> [&'static str; 2] { - match path { - "/_ts/admin/ec/{id}" => Self::ADMIN_EC_ID_AUTH_PROBES, - path => [path, path], - } + /// Whether a section selects the module `name`. + #[must_use] + pub fn selects_module(&self, name: &str) -> bool { + self.module_selection(name).is_some() } - /// Returns admin endpoint paths that no configured handler covers. + /// The table `section` holds at the name `written`, as it was written, + /// or an empty one when the section holds none. + #[must_use] + pub fn section_table( + &self, + section: &str, + written: &str, + ) -> serde_json::Map { + self.module_sections() + .find(|(candidate, _)| *candidate == section) + .and_then(|(_, modules)| modules.settings_of(written)) + .cloned() + .unwrap_or_default() + } + + /// Reads and validates a selected module's settings, from the table at the + /// name it is written under beneath the section that selects it, or + /// returns `None` when no section selects it. /// - /// Called during settings finalization to enforce that every admin endpoint - /// has a handler. An empty return - /// value means all admin endpoints are properly covered. + /// A selected module with no table reads an empty one, so a module that + /// takes no settings runs on its selection alone and one that requires a + /// setting reports the setting it is missing. The parse and validation + /// messages carry the underlying error, because a report renders only its + /// outermost message in `ts config validate`. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] if any handler has an invalid path regex. - pub(crate) fn uncovered_admin_endpoints( - &self, - ) -> Result, Report> { - let mut uncovered = Vec::new(); - for &path in Self::ADMIN_ENDPOINTS { - let mut covered = true; - for probe in Self::admin_auth_probes(path) { - let mut probe_covered = false; - for handler in &self.handlers { - if handler.matches_path(probe)? { - probe_covered = true; - break; - } - } - covered &= probe_covered; - } - if !covered { - uncovered.push(path); - } - } - Ok(uncovered) + /// When the table cannot be read as `T` or fails its validation, naming the + /// table. + pub fn module_config(&self, name: &str) -> Result, Report> + where + T: IntegrationConfig, + { + let Some((section, written)) = self.module_selection(name) else { + return Ok(None); + }; + let table = self + .module_sections() + .find(|(candidate, _)| *candidate == section) + .and_then(|(_, modules)| modules.settings_of(written)) + .cloned() + .unwrap_or_default(); + let config: T = serde_json::from_value(JsonValue::Object(table)).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{section}.{written}] could not be read: {error}"), + }) + })?; + config.validate().map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("[{section}.{written}] failed validation: {error}"), + }) + })?; + Ok(Some(config)) } - /// Validates that every admin endpoint is covered by at least one handler. + /// Reads `[shared.]`, the settings every module of `vendor` + /// shares, or `None` when the table is absent. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] listing any uncovered - /// admin endpoints. - pub(crate) fn validate_admin_coverage(&self) -> Result<(), Report> { - let uncovered = self.uncovered_admin_endpoints()?; - if uncovered.is_empty() { - return Ok(()); - } - Err(Report::new(TrustedServerError::Configuration { - message: format!( - "No handler covers admin endpoint(s): {}. \ - Add a [[handlers]] entry with a path regex matching /_ts/admin/ \ - to protect admin access.", - uncovered.join(", ") - ), - })) + /// When the table cannot be read as `T` or fails its validation, naming + /// the table. + pub fn shared_config(&self, vendor: &str) -> Result, Report> + where + T: DeserializeOwned + Validate, + { + self.shared.get_typed(vendor) } - /// Rejects placeholder and well-known weak handler passwords. - /// - /// Applies to every handler rather than to handlers inferred to cover an - /// admin endpoint: handler selection is first-match-wins over operator - /// regexes, so a narrow handler can shadow the admin namespace for paths no - /// probe enumerates. Handlers are Trusted Server's own basic-auth gates, so - /// a placeholder password is never valid on any of them. - pub(crate) fn validate_admin_handler_passwords( - &self, - ) -> Result<(), Report> { - for handler in &self.handlers { - if is_admin_placeholder_password(handler.password.expose()) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Handler `{}` uses a placeholder password; configure a strong secret", - handler.path - ), - })); - } + /// The modules `section` selects, to change, with the section created + /// empty when it is a module type's and absent. + fn section_modules_mut(&mut self, section: &str) -> &mut SectionModules { + match section { + "proxy" => &mut self.proxy.modules, + "auction" => &mut self.auction.modules, + _ => self.sections.section_mut(section), } + } - Ok(()) + /// Selects the module `name` in `section`, written with the section's + /// type folder left off, with no table of its own. + pub fn select_module(&mut self, section: &str, name: &str) { + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).select(&written); } - /// Retrieves the integration configuration of a specific type. + /// Stops selecting the module `name` in `section` and drops its table. + pub fn remove_module(&mut self, section: &str, name: &str) { + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).remove(&written); + } + + /// Selects the module `name` in `section`, with `settings` as its table, + /// which is what writing both in a document does. + /// + /// The section is the name's type folder, except for the sections that + /// Trusted Server reads itself and that also select modules, where the + /// name is written in full. /// /// # Errors /// - /// Returns an error if the integration configuration exists but cannot be deserialized as the requested type. - pub fn integration_config( - &self, - integration_id: &str, - ) -> Result, Report> + /// Returns an error if `settings` cannot be serialized to a table. + pub fn insert_module_config( + &mut self, + section: &str, + name: &str, + settings: &T, + ) -> Result<(), Report> where - T: IntegrationConfig, + T: Serialize, { - self.integrations.get_typed(integration_id) + let JsonValue::Object(table) = + serde_json::to_value(settings).change_context(TrustedServerError::Configuration { + message: "Failed to serialize module configuration".to_string(), + })? + else { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("the settings of `{name}` are not a table"), + })); + }; + let written = crate::module_name::short_form(section, name).to_owned(); + self.section_modules_mut(section).insert(&written, table); + Ok(()) + } + + /// The entries of `phase`, in the order written. + #[must_use] + pub fn phase_entries(&self, phase: MiddlewarePhase) -> &PhaseEntries { + match phase { + MiddlewarePhase::Fetch => &self.fetch, + MiddlewarePhase::Serve => &self.serve, + } + } + + /// Checks the shape of each phase's entries, see + /// [`PhaseEntries::validate`]. Whether a name is one a running module + /// supplies is only knowable where the registry is built, so it is + /// checked there. + /// + /// # Errors + /// + /// Naming the entry at fault. + pub fn validate_phase_entries(&self) -> Result<(), Report> { + for phase in MiddlewarePhase::ALL { + self.phase_entries(phase) + .validate(phase) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + } + Ok(()) + } + + /// Checks every section's selection and tables, and that no module is + /// selected in two sections. + /// + /// # Errors + /// + /// Naming the section at fault. + pub fn validate_module_sections(&self) -> Result<(), Report> { + self.sections.validate()?; + for (section, modules) in [ + ("proxy", &self.proxy.modules), + ("auction", &self.auction.modules), + ] { + modules + .validate(section) + .map_err(|message| Report::new(TrustedServerError::Configuration { message }))?; + } + let mut seen: Vec<(&str, String)> = Vec::new(); + for (section, modules) in self.module_sections() { + for written in modules.selected() { + let full = if written.contains('.') { + written.clone() + } else { + format!("{section}.{written}") + }; + if let Some((earlier, _)) = seen.iter().find(|(_, name)| *name == full) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "`{written}` is selected in both [{earlier}] and [{section}]. A \ + module is selected in one section" + ), + })); + } + seen.push((section, full)); + } + } + Ok(()) } } @@ -3559,8 +5018,14 @@ fn validation_error_summary(errors: &validator::ValidationErrors) -> String { }; match kind { validator::ValidationErrorsKind::Field(validations) => { + // A validator's message, where it gives one, says what the + // code cannot, such as which address an endpoint clashed + // with. for validation in validations { - messages.push(format!("{path}: {}", validation.code)); + messages.push(match &validation.message { + Some(message) => format!("{path}: {} ({message})", validation.code), + None => format!("{path}: {}", validation.code), + }); } } validator::ValidationErrorsKind::Struct(inner) => { @@ -3655,14 +5120,6 @@ fn validate_proxy_origin_url(value: &str) -> Result<(), ValidationError> { Ok(()) } -fn validate_path(value: &str) -> Result<(), ValidationError> { - Regex::new(value).map(|_| ()).map_err(|err| { - let mut validation_error = ValidationError::new("invalid_regex"); - validation_error.add_param("value".into(), &value); - validation_error.add_param("message".into(), &err.to_string()); - validation_error - }) -} fn from_value_or_str<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, @@ -3678,7 +5135,7 @@ where // Helper: allow Vec fields to deserialize from either a JSON array or a map of numeric indices. // This lets env vars such as -// TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS__0=example-browser work; +// TRUSTED_SERVER__AUCTION__PREBID__CLIENT_SIDE_BIDDERS__0=example-browser work; // the config env source represents the value as an object rather than a sequence. // String inputs may also be JSON arrays or comma-separated values. /// Deserializes a `HashMap` from either: @@ -3740,7 +5197,18 @@ where } } -pub(crate) fn vec_from_seq_or_map<'de, D, T>(deserializer: D) -> Result, D::Error> +/// Reads a list setting written as a sequence, as a map keyed by position, +/// or as a string holding a JSON array or comma-separated values. +/// +/// The map and string forms are what an environment overlay produces, so a +/// list reads the same from a file and from the environment. Public so a +/// module crate's own list settings read the same way. +/// +/// # Errors +/// +/// When the value is none of those forms, a map key is not a position, or an +/// item cannot be read as `T`. +pub fn vec_from_seq_or_map<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, T: DeserializeOwned, @@ -3816,7 +5284,8 @@ mod tests { use regex::Regex; use serde_json::json; use std::collections::BTreeSet; - use std::sync::Arc; + + use crate::ec::resolve::NotAnOrigin; /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. @@ -3869,13 +5338,12 @@ mod tests { ); } - use crate::auction::build_orchestrator; - use crate::integrations::{ - IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, - prebid::PrebidIntegrationConfig, - }; + use crate::integrations::IntegrationRegistry; use crate::redacted::Redacted; - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; + use crate::test_support::tests::{ + crate_test_settings_str, crate_test_settings_str_with_ec_section, create_test_settings, + hmac_passphrase, select_hmac_module, + }; fn trusted_client_ip_toml(ip_header: &str, auth_header: &str, shared_secret: &str) -> String { format!( @@ -3895,6 +5363,166 @@ mod tests { ); } + fn settings_str_with_robots_txt(section: &str) -> String { + format!( + "{}\n[robots-txt]\n{section}\n\n[[fetch]]\nmedia_type = \"text/plain\"\npath = \"/robots.txt\"\nmiddleware = [\"robots_txt\"]\n", + crate_test_settings_str() + ) + } + + #[test] + fn a_robots_txt_section_with_no_entry_placing_the_middleware_is_refused() { + let toml = format!( + "{}\n[robots-txt]\nmodules = \"refuse_all\"\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("the file would never be made"); + + let message = format!("{error:?}"); + assert!( + message.contains( + "no [[fetch]] entry with media_type = `text/plain` and path = `/robots.txt` names \ + `robots_txt`" + ), + "{message}" + ); + } + + /// The `[[fetch]]` or `[[serve]]` entry placing the middleware on + /// `/robots.txt`. + const ROBOTS_TXT_ENTRY: &str = + "media_type = \"text/plain\"\npath = \"/robots.txt\"\nmiddleware = [\"robots_txt\"]\n"; + + /// Settings with `[robots-txt]` refusing every crawler and `entries`. + fn settings_str_placing_robots_txt(entries: &str) -> String { + format!( + "{}\n[robots-txt]\nmodules = \"refuse_all\"\n\n{entries}\n", + crate_test_settings_str() + ) + } + + #[test] + fn a_fetch_entry_on_the_files_path_places_robots_txt() { + let toml = settings_str_placing_robots_txt(&format!( + "[[fetch]]\n{ROBOTS_TXT_ENTRY}\n[[serve]]\n{ROBOTS_TXT_ENTRY}" + )); + + Settings::from_toml(&toml).expect("an entry on the file's path places the middleware"); + } + + /// An entry with no path would replace every text file, `ads.txt` + /// included, with the robots.txt rules. + #[test] + fn an_entry_naming_robots_txt_on_another_path_is_refused() { + let toml = settings_str_placing_robots_txt(&format!( + "[[fetch]]\n{ROBOTS_TXT_ENTRY}\n[[fetch]]\nmedia_type = \"text/plain\"\nmiddleware = [\"robots_txt\"]\n" + )); + + let error = Settings::from_toml(&toml).expect_err("every text file would be robots.txt"); + + let message = format!("{error:?}"); + assert!( + message.contains("[[fetch]] entry 2 names `robots_txt` for `text/plain` on every path"), + "{message}" + ); + } + + /// Only a `[[fetch]]` entry makes the file where the origin has none, so + /// a `[[serve]]` placement alone would leave an origin's `404` standing, + /// which a crawler reads as permission to crawl everything. + #[test] + fn robots_txt_placed_only_in_serve_is_refused() { + let toml = settings_str_placing_robots_txt(&format!("[[serve]]\n{ROBOTS_TXT_ENTRY}")); + + let error = Settings::from_toml(&toml).expect_err("the file would never be made"); + + assert!( + format!("{error:?}").contains("no [[fetch]] entry with media_type = `text/plain`"), + "{error:?}" + ); + } + + #[test] + fn the_robots_txt_section_is_read_by_its_hyphenated_name() { + let toml = settings_str_with_robots_txt( + "modules = [\"refuse_all\"]\nalways_allow = [\"/ads.txt\"]", + ); + + let settings = Settings::from_toml(&toml).expect("should read the section"); + + let robots_txt = settings + .robots_txt + .as_ref() + .expect("should carry the section"); + assert_eq!(robots_txt.modules, vec!["refuse_all"]); + assert_eq!(robots_txt.always_allow, vec!["/ads.txt"]); + assert!( + settings + .module_sections() + .all(|(name, _)| name != "robots-txt"), + "should not also be read as the section of a module type" + ); + } + + #[test] + fn a_robots_txt_section_naming_no_module_is_refused_when_the_settings_load() { + let toml = settings_str_with_robots_txt("sitemap = \"https://example.com/sitemap.xml\""); + + let error = Settings::from_toml(&toml).expect_err("there is no default"); + + assert!( + format!("{error:?}").contains("[robots-txt] names no module"), + "should say a module must be named: {error:?}" + ); + } + + #[test] + fn a_robots_txt_path_that_could_start_a_second_line_is_refused() { + let toml = settings_str_with_robots_txt( + "modules = \"refuse_all\"\nalways_allow = [\"/ads.txt\\nDisallow:\"]", + ); + + assert!( + Settings::from_toml(&toml).is_err(), + "should refuse a path that holds white space" + ); + } + + #[test] + fn settings_without_a_robots_txt_section_are_written_without_one() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert!( + value.get("robots-txt").is_none(), + "a build that predates the section refuses a key it does not know" + ); + } + + #[test] + fn a_robots_txt_section_survives_the_stored_form() { + let toml = settings_str_with_robots_txt( + "modules = [\"allow_all\"]\nsitemap = \"https://example.com/sitemap.xml\"", + ); + let settings = Settings::from_toml(&toml).expect("should read the section"); + + let stored = serde_json::to_value(&settings).expect("should serialize the settings"); + let reloaded = Settings::from_json_value(stored).expect("should read the section back"); + + let robots_txt = reloaded + .robots_txt + .as_ref() + .expect("should still carry the section"); + assert_eq!(robots_txt.modules, vec!["allow_all"]); + assert_eq!( + robots_txt.sitemap.as_deref(), + Some("https://example.com/sitemap.xml") + ); + } + /// Mirrors the `Settings` schema of the revision that predates /// `trusted_client_ip`: every key that revision knew, and /// `deny_unknown_fields` so an extra key fails deserialization exactly as an @@ -3956,15 +5584,79 @@ mod tests { #[test] fn serialized_default_config_stays_readable_by_the_base_revision_schema() { - let settings = Settings::from_toml(&crate_test_settings_str()) + let mut settings = Settings::from_toml(&crate_test_settings_str()) .expect("should parse settings without trusted client IP configuration"); + // The guarantee covers a config that configures none of the sections + // added since the base revision. A configured section is serialized and, + // like a configured `trusted_client_ip`, needs a compatible blob restored + // before rolling back to a binary that predates it. The shared test + // config sets `[geo]` because the permission model requires a default + // country, so both selector tables are reset to unset here. + settings.geo = GeoConfig::default(); + settings.device = DeviceConfig::default(); + settings.sections = TypeSections::default(); + settings.auction.modules.clear(); + let value = serde_json::to_value(&settings).expect("should serialize settings"); serde_json::from_value::(value) .expect("base revision schema should accept a config blob with no trusted client IP"); } + #[test] + fn geo_selector_is_omitted_from_serialized_config_when_unset() { + // `ts config push` serializes `Settings` verbatim, so a selector nobody + // set must not appear in the blob. A `deny_unknown_fields` binary that + // predates the selector rejects the key during rollout or rollback. + // + // The shared fixture writes a `[geo]` table (it acknowledges running + // with no geo module), so the assertion is about the selector key + // rather than the table, which is what the blob's compatibility + // actually turns on. + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse settings without a geo selector"); + + let value = serde_json::to_value(&settings).expect("should serialize settings"); + + let geo = value + .get("geo") + .expect("the geo table is written because the fixture acknowledges no geo module"); + assert!( + geo.get("module").is_none(), + "an unset geo selector should not be serialized, got {geo}" + ); + assert!( + value + .get("device") + .is_none_or(|device| device.get("module").is_none()), + "an unset device selector should not be serialized, got {value}" + ); + } + + #[test] + fn a_selected_geo_module_stays_in_the_serialized_config() { + // The shared test settings already carry a `[geo]` table, so the + // selector is set inside that table rather than in a second one, which + // TOML rejects as a duplicate key. + let settings = Settings::from_toml(&crate_test_settings_str().replace( + "[geo]", + "[geo] +module = \"none\"", + )) + .expect("should parse settings with a geo selector"); + + let value = serde_json::to_value(&settings).expect("should serialize settings"); + + assert_eq!( + value + .pointer("/geo/module") + .and_then(serde_json::Value::as_str), + Some("none"), + "a selected geo module should survive serialization" + ); + } + #[test] fn trusted_client_ip_parses_and_redacts_shared_secret_in_debug_output() { let settings = Settings::from_toml(&trusted_client_ip_toml( @@ -4001,7 +5693,7 @@ mod tests { // hand-written `Debug` impl prints only integration IDs, never values. // // Do not use `..Struct::default()` anywhere in this function. A default - // spread would let a new secret field be added to `Handler`, + // spread would let a new secret field be added to `EcPartner`, // `TinybirdSettings`, or any other struct built here without forcing // anyone to consider it. The compile break is the prompt; the canary // list below is still maintained by hand. List every field explicitly. @@ -4009,8 +5701,6 @@ mod tests { fn settings_debug_output_redacts_every_secret_field() { const CANARY_PROXY_SECRET: &str = "CANARY-PROXY-SECRET-0123456789"; const CANARY_EC_PASSPHRASE: &str = "CANARY-EC-PASSPHRASE-0123456789"; - const CANARY_HANDLER_USERNAME: &str = "CANARY-HANDLER-USERNAME-0123456789"; - const CANARY_HANDLER_PASSWORD: &str = "CANARY-HANDLER-PASSWORD-0123456789"; const CANARY_EC_PARTNER_API_TOKEN: &str = "CANARY-EC-PARTNER-API-TOKEN-0123456789"; const CANARY_EC_PARTNER_TS_PULL_TOKEN: &str = "CANARY-EC-PARTNER-TS-PULL-TOKEN-0123456789"; const CANARY_TRUSTED_CLIENT_IP_SHARED_SECRET: &str = @@ -4020,19 +5710,12 @@ mod tests { const CANARY_S3_SESSION_TOKEN: &str = "CANARY-S3-SESSION-TOKEN-0123456789"; const CANARY_TINYBIRD_AUCTION_TOKEN: &str = "CANARY-TINYBIRD-AUCTION-TOKEN-0123456789"; const CANARY_TINYBIRD_ACCESS_TOKEN: &str = "CANARY-TINYBIRD-ACCESS-TOKEN-0123456789"; - const CANARY_DATADOME_SERVER_SIDE_KEY: &str = "CANARY-DATADOME-SERVER-SIDE-KEY-0123456789"; + const CANARY_MODULE_KEY: &str = "CANARY-MODULE-TABLE-KEY-0123456789"; let mut settings = create_test_settings(); settings.publisher.proxy_secret = Redacted::new(CANARY_PROXY_SECRET.to_string()); - settings.ec.passphrase = Redacted::new(CANARY_EC_PASSPHRASE.to_string()); - - settings.handlers = vec![Handler { - path: "^/secure".to_string(), - username: Redacted::new(CANARY_HANDLER_USERNAME.to_string()), - password: Redacted::new(CANARY_HANDLER_PASSWORD.to_string()), - regex: OnceLock::new(), - }]; + select_hmac_module(&mut settings.ec, HMAC_MODULE_KEY, CANARY_EC_PASSPHRASE); settings.ec.partners = vec![EcPartner { name: "canary-partner".to_string(), @@ -4066,33 +5749,35 @@ mod tests { })); settings.proxy.asset_routes = vec![asset_route]; - settings.tinybird = TinybirdSettings { - auction_token_secret: Some(Redacted::new(CANARY_TINYBIRD_AUCTION_TOKEN.to_string())), - access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), - enabled: false, - api_host: String::new(), - secret_store: None, - auction_dataset: String::new(), - access_enabled: false, - access_dataset: String::new(), - access_sample_rate: 0.0f64, - max_body_bytes: 0, - }; + settings.analytics = Some(AnalyticsConfig { + module: Some(ANALYTICS_MODULE.to_string()), + tinybird: Some(TinybirdSettings { + auction_token_secret: Some(Redacted::new( + CANARY_TINYBIRD_AUCTION_TOKEN.to_string(), + )), + access_token_secret: Some(Redacted::new(CANARY_TINYBIRD_ACCESS_TOKEN.to_string())), + api_host: String::new(), + secret_store: None, + auction_dataset: String::new(), + access_enabled: false, + access_dataset: String::new(), + access_sample_rate: 0.0f64, + max_body_bytes: 0, + }), + }); - // `IntegrationSettings` stores integration configs as opaque JSON and - // relies on a hand-written `Debug` impl to suppress their values. That - // impl is the only thing keeping resolved DataDome credentials out of - // this output, so pin it here. + // A module's table is opaque JSON, and the section's hand-written + // `Debug` impl is the only thing keeping a secret a module's table + // holds out of this output, so pin it here. settings - .integrations - .insert_config( - "datadome", + .insert_module_config( + "testing", + "testing.example", &json!({ - "enabled": true, - "server_side_key_secret_name": CANARY_DATADOME_SERVER_SIDE_KEY, + "key_name": CANARY_MODULE_KEY, }), ) - .expect("should insert datadome integration config"); + .expect("should insert a module's table"); let debug = format!("{settings:?}"); @@ -4101,15 +5786,13 @@ mod tests { "should redact secret fields in Settings debug output" ); assert!( - debug.contains("^/secure"), - "should leave non-secret handler path visible in debug output" + debug.contains("canary-partner.example"), + "should leave a non-secret value visible in debug output" ); let canaries = [ ("publisher.proxy_secret", CANARY_PROXY_SECRET), - ("ec.passphrase", CANARY_EC_PASSPHRASE), - ("handlers[].username", CANARY_HANDLER_USERNAME), - ("handlers[].password", CANARY_HANDLER_PASSWORD), + ("ec.hmac.passphrase", CANARY_EC_PASSPHRASE), ("ec.partners[].api_token", CANARY_EC_PARTNER_API_TOKEN), ( "ec.partners[].ts_pull_token", @@ -4132,14 +5815,14 @@ mod tests { CANARY_S3_SESSION_TOKEN, ), ( - "tinybird.auction_token_secret", + "analytics.tinybird.auction_token_secret", CANARY_TINYBIRD_AUCTION_TOKEN, ), - ("tinybird.access_token_secret", CANARY_TINYBIRD_ACCESS_TOKEN), ( - "integrations.datadome.server_side_key_secret_name", - CANARY_DATADOME_SERVER_SIDE_KEY, + "analytics.tinybird.access_token_secret", + CANARY_TINYBIRD_ACCESS_TOKEN, ), + ("testing.example.key_name", CANARY_MODULE_KEY), ]; for (field, canary) in canaries { @@ -4550,13 +6233,147 @@ mod tests { } } + /// The shared fixture with one `[[handlers]]` entry ahead of everything + /// else, as a configuration written for a build that asked for passwords + /// carries it. + fn settings_str_with_a_handler() -> String { + format!( + "[[handlers]]\npath = \"^/_ts/admin\"\nusername = \"admin\"\n\ + password = \"admin-pass\"\n{}", + crate_test_settings_str() + ) + } + + #[test] + fn toml_settings_refuse_a_handlers_list_with_directions() { + let error = Settings::from_toml(&settings_str_with_a_handler()) + .expect_err("should refuse a configuration that still gates a path"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("`[[handlers]]` is not read"), + "should say the list is not read: {rendered}" + ); + assert!( + rendered.contains("ts keys") && rendered.contains("ts cache purge"), + "should say where administration went: {rendered}" + ); + } + + #[test] + fn json_settings_refuse_a_handlers_list_with_directions() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + let mut value = serde_json::to_value(settings) + .expect("should serialize the test settings fixture to JSON"); + value["handlers"] = json!([ + { "path": "^/secure", "username": "user", "password": "handler_password" }, + ]); + + let error = Settings::from_json_value(value) + .expect_err("should refuse a stored document that still gates a path"); + + assert!( + format!("{error:?}").contains("`[[handlers]]` is not read"), + "should say the list is not read: {error:?}" + ); + } + + #[test] + fn an_empty_handlers_list_is_refused_too() { + let toml = format!("handlers = []\n{}", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("should refuse the key itself"); + + assert!( + format!("{error:?}").contains("`[[handlers]]` is not read"), + "should say the list is not read: {error:?}" + ); + } + + #[test] + fn settings_are_written_without_a_handlers_list() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert!( + value.get("handlers").is_none(), + "should write nothing a later read would refuse" + ); + } + + #[test] + fn toml_settings_refuse_a_signing_store_id_with_directions() { + for key in ["config_store_id", "secret_store_id"] { + let fixture = crate_test_settings_str(); + assert!( + fixture.contains("[request_signing]\n"), + "the fixture should have the section the key is added to" + ); + let toml = fixture.replace( + "[request_signing]\n", + &format!("[request_signing]\n{key} = \"01GEXAMPLE\"\n"), + ); + + let error = Settings::from_toml(&toml).expect_err("should refuse the removed key"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("does not take `config_store_id` or `secret_store_id`"), + "should say the key is not read, for {key}: {rendered}" + ); + assert!( + rendered.contains("ts keys rotate"), + "should say where the ids go, for {key}: {rendered}" + ); + } + } + + #[test] + fn json_settings_refuse_a_signing_store_id_with_directions() { + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + let mut value = serde_json::to_value(settings) + .expect("should serialize the test settings fixture to JSON"); + value["request_signing"]["secret_store_id"] = json!("01GEXAMPLE"); + + let error = Settings::from_json_value(value).expect_err("should refuse the removed key"); + + assert!( + format!("{error:?}").contains("--secret-store-id"), + "should say where the id goes: {error:?}" + ); + } + + #[test] + fn request_signing_is_written_as_enabled_alone() { + let mut settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + settings.request_signing = Some(RequestSigning::new(true)); + + let value = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert_eq!( + value["request_signing"], + json!({ "enabled": true }), + "should write nothing a later read would refuse" + ); + let read = Settings::from_json_value(value).expect("should read what was written"); + assert!( + read.request_signing.is_some_and(|signing| signing.enabled), + "should keep request signing on across the round trip" + ); + } + #[test] fn json_settings_rejects_legacy_auction_provider_list_with_migration_guidance() { let settings = Settings::from_toml(&crate_test_settings_str()) .expect("should load the test settings fixture"); let mut value = serde_json::to_value(settings) .expect("should serialize the test settings fixture to JSON"); - value["auction"]["providers"] = json!(["prebid"]); + value["auction"]["providers"] = json!(["example"]); let error = Settings::from_json_value(value) .expect_err("should reject the removed auction provider list schema"); @@ -4566,16 +6383,17 @@ mod tests { "error should identify the removed field, got {rendered}" ); assert!( - rendered.contains("CHANGELOG.md"), - "error should direct operators to the migration guidance, got {rendered}" + rendered.contains("[demand] modules"), + "error should name where the setting moved to, got {rendered}" ); } #[test] fn toml_settings_reject_legacy_auction_provider_list_with_migration_guidance() { let toml = format!( - "{}\n[auction]\nproviders = [\"prebid\"]\n", + "{}\n", crate_test_settings_str() + .replace("[auction]\n", "[auction]\nproviders = [\"example\"]\n") ); let error = Settings::from_toml(&toml) @@ -4586,8 +6404,8 @@ mod tests { "error should identify the removed field, got {rendered}" ); assert!( - rendered.contains("CHANGELOG.md"), - "error should direct operators to the migration guidance, got {rendered}" + rendered.contains("[demand] modules"), + "error should name where the setting moved to, got {rendered}" ); } @@ -4595,7 +6413,7 @@ mod tests { fn auction_debug_comment_options_default_matches_serde_defaults() { let opts = AuctionDebugCommentOptions::default(); assert!(opts.include_provider_responses, "should default to true"); - assert!(opts.include_mediator_response, "should default to true"); + assert!(opts.include_adserver_response, "should default to true"); assert!(opts.include_bids, "should default to true"); assert_eq!( opts.metadata_keys, @@ -4758,61 +6576,235 @@ mod tests { ); } + /// `[analytics]` selecting the analytics module, with `table` as the + /// lines of its settings table. + fn settings_str_with_analytics_module(table: &str) -> String { + format!( + "{}\n[analytics]\nmodule = \"tinybird\"\n\n[analytics.tinybird]\n{table}\n", + crate_test_settings_str() + ) + } + + /// The lines of a telemetry table that passes every check. + const ANALYTICS_MODULE_LINES: &str = + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\""; + #[test] - fn tinybird_defaults_to_disabled_placeholders() { + fn settings_without_an_analytics_section_select_no_module() { let settings = Settings::from_toml(&crate_test_settings_str()) - .expect("should parse settings without tinybird block"); + .expect("should parse settings without an analytics section"); + assert!(settings.analytics.is_none(), "should hold no section"); assert!( - !settings.tinybird.enabled, - "Tinybird should default disabled" - ); - assert_eq!(settings.tinybird.secret_store, None); - assert_eq!(settings.tinybird.auction_dataset, "auction_events_raw"); - assert!(settings.tinybird.auction_token_secret.is_none()); - } - - #[test] - fn tinybird_enabled_requires_host_dataset_and_token() { - let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"https://api.example.com/path\"\n", - crate_test_settings_str() + settings.analytics_module().is_none(), + "should select no module" ); - - let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); + let value = serde_json::to_value(&settings).expect("should serialize the settings"); assert!( - format!("{err:?}").contains("tinybird.api_host"), - "should report tinybird.api_host validation error: {err:?}" + value.get("analytics").is_none() && value.get("tinybird").is_none(), + "should write neither the section nor the table it replaced: {value}" ); } #[test] - fn tinybird_accepts_region_host_without_scheme() { - let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\"\n", - crate_test_settings_str() - ); + fn the_analytics_module_is_selected_by_its_short_or_full_name() { + for name in ["tinybird", "analytics.tinybird"] { + let toml = format!( + "{}\n[analytics]\nmodule = \"{name}\"\n\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", + crate_test_settings_str() + ); + + let settings = Settings::from_toml(&toml).expect("should accept the selection"); + + let telemetry = settings + .analytics_module() + .expect("should select the analytics module"); + assert_eq!(telemetry.api_host, "api.us-east.aws.tinybird.co"); + assert_eq!(telemetry.auction_dataset, "auction_events_raw"); + } + } + + #[test] + fn a_selected_analytics_module_survives_the_stored_form() { + let settings = + Settings::from_toml(&settings_str_with_analytics_module(ANALYTICS_MODULE_LINES)) + .expect("should accept the selection"); + + let stored = serde_json::to_value(&settings).expect("should serialize the settings"); + + assert_eq!(stored["analytics"]["module"], "tinybird"); + assert_eq!( + stored["analytics"]["tinybird"]["api_host"], + "api.us-east.aws.tinybird.co" + ); + assert!( + stored.get("tinybird").is_none(), + "should not write the table that moved: {stored}" + ); + } + + #[test] + fn the_analytics_module_needs_a_regional_host() { + let toml = + settings_str_with_analytics_module("api_host = \"https://api.example.com/path\""); + + let err = Settings::from_toml(&toml).expect_err("should reject invalid api host"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the setting by its table: {err:?}" + ); + } + + #[test] + fn the_analytics_module_needs_its_token() { + let toml = settings_str_with_analytics_module("api_host = \"api.us-east.aws.tinybird.co\""); + + let err = Settings::from_toml(&toml).expect_err("should require the token"); - let settings = Settings::from_toml(&toml).expect("should accept Tinybird region host"); - assert!(settings.tinybird.enabled); - assert_eq!(settings.tinybird.api_host, "api.us-east.aws.tinybird.co"); + assert!( + format!("{err:?}").contains( + "analytics.tinybird.auction_token_secret is required when [analytics] selects \ + the module" + ), + "should say what is missing and why: {err:?}" + ); } #[test] - fn tinybird_access_enabled_is_rejected_until_emitter_is_wired() { + fn a_selected_analytics_module_with_no_table_is_told_what_it_needs() { let toml = format!( - "{}\n[tinybird]\naccess_enabled = true\n", + "{}\n[analytics]\nmodule = \"tinybird\"\n", crate_test_settings_str() ); + let err = Settings::from_toml(&toml).expect_err("should need the table's settings"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.api_host"), + "should name the first setting the module needs: {err:?}" + ); + } + + #[test] + fn access_telemetry_is_rejected_until_an_emitter_is_wired() { + let toml = settings_str_with_analytics_module(&format!( + "{ANALYTICS_MODULE_LINES}\naccess_enabled = true" + )); + let err = Settings::from_toml(&toml) - .expect_err("should reject access telemetry before emitter exists"); + .expect_err("should reject access telemetry before an emitter exists"); + + assert!( + format!("{err:?}").contains("analytics.tinybird.access_enabled"), + "should name the unsupported setting: {err:?}" + ); + } + + #[test] + fn an_analytics_section_selecting_nothing_is_refused() { + let toml = format!( + "{}\n[analytics.tinybird]\n{ANALYTICS_MODULE_LINES}\n", + crate_test_settings_str() + ); + + let err = Settings::from_toml(&toml).expect_err("nothing would read the table"); + + assert!( + format!("{err:?}").contains("[analytics] selects no module"), + "should say the section selects nothing: {err:?}" + ); + } + + #[test] + fn an_analytics_module_this_build_does_not_have_is_refused() { + let toml = format!( + "{}\n[analytics]\nmodule = \"another\"\n", + crate_test_settings_str() + ); + + let err = Settings::from_toml(&toml).expect_err("no such module"); + let message = format!("{err:?}"); + + assert!( + message.contains( + "[analytics] selects `another`, which no module this deployment supplies" + ), + "should name what was written: {message}" + ); assert!( - format!("{err:?}").contains("tinybird.access_enabled"), - "should report unsupported tinybird.access_enabled setting: {err:?}" + message.contains("The analytics modules it supplies are [tinybird]"), + "should list what can be selected: {message}" ); } + /// The token is a secret reference, and an empty or absent one is refused + /// as the settings load, before any deploy step sees the document. + #[test] + fn an_analytics_token_reference_that_is_empty_or_absent_is_refused_when_the_settings_load() { + for (case, table) in [ + ( + "an empty token", + "api_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"\"", + ), + ("no token", "api_host = \"api.us-east.aws.tinybird.co\""), + ] { + let err = Settings::from_toml(&settings_str_with_analytics_module(table)) + .err() + .unwrap_or_else(|| panic!("{case}: the settings should not load")); + + assert!( + format!("{err:?}").contains("auction_token_secret"), + "{case}: should name the token: {err:?}" + ); + } + } + + #[test] + fn the_analytics_module_table_refuses_a_key_it_does_not_know() { + let toml = settings_str_with_analytics_module(&format!( + "{ANALYTICS_MODULE_LINES}\nenabled = true" + )); + + let err = Settings::from_toml(&toml).expect_err("the enabled line is gone"); + + assert!( + format!("{err:?}").contains("enabled"), + "should name the key it does not know: {err:?}" + ); + } + + #[test] + fn a_table_the_section_does_not_select_gives_no_analytics_module() { + let analytics = AnalyticsConfig { + module: None, + tinybird: Some(TinybirdSettings::default()), + }; + + assert!( + analytics.analytics_module().is_none(), + "a table nothing selects should give no settings" + ); + } + + #[test] + fn the_tinybird_table_is_refused_with_directions() { + for table in [ + "enabled = true\napi_host = \"api.us-east.aws.tinybird.co\"", + "enabled = false", + ] { + let toml = format!("{}\n[tinybird]\n{table}\n", crate_test_settings_str()); + + let err = Settings::from_toml(&toml).expect_err("the table has moved"); + + assert!( + format!("{err:?}") + .contains("Configuration table `[tinybird]` is now `[analytics.tinybird]`"), + "should say where the table went: {err:?}" + ); + } + } + #[test] fn settings_rejects_removed_consent_store_toml() { let toml = format!( @@ -4853,27 +6845,16 @@ mod tests { assert!(settings.is_ok()); let settings = settings.expect("should parse valid TOML"); - let prebid_cfg = settings - .integration_config::("prebid") - .expect("Prebid config query should succeed") - .expect("Prebid config should load from test settings"); - assert_eq!(prebid_cfg.timeout_ms, 1000); assert!( settings - .integration_config::("nextjs") - .expect("Next.js config query should succeed") + .module_config::(ENDPOINT_MODULE) + .expect("a query for a module no section selects should succeed") .is_none(), - "Next.js integration should default to disabled" + "a module no section selects should not run" ); - let raw_nextjs = settings - .integrations - .get("nextjs") - .expect("test settings should include nextjs block"); - assert_eq!(raw_nextjs["enabled"], json!(false)); - assert_eq!( - raw_nextjs["rewrite_attributes"], - json!(["href", "link", "url"]), - "Next.js rewrite attributes should default to href/link/url" + assert!( + settings.auction.modules.selected().is_empty(), + "the fixture should select no auction module" ); assert_eq!(settings.publisher.domain, "test-publisher.com"); assert_eq!(settings.publisher.cookie_domain, ".test-publisher.com"); @@ -4892,7 +6873,12 @@ mod tests { ); assert_eq!(settings.publisher.origin_host_header_override, None); assert_eq!( - settings.ec.passphrase.expose(), + settings.ec.module.as_ref(), + Some(&EcModuleSelection::from(HMAC_MODULE_KEY)), + "test settings should select the hmac EC module" + ); + assert_eq!( + hmac_passphrase(&settings.ec, HMAC_MODULE_KEY), "test-secret-key-32-bytes-minimum" ); @@ -4977,6 +6963,99 @@ mod tests { ); } + #[test] + fn loading_cache_lifetimes_are_read_with_defaults_for_what_is_unset() { + let toml_str = format!( + r#"{} + + [cache.robots-txt] + store_seconds = 600 + store_idle_seconds = 3600 + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse the lifetimes"); + let defaults = crate::platform::CacheLifetimes { + memory: Duration::from_secs(5), + store: Duration::from_secs(60), + store_idle: Duration::from_secs(70), + }; + + assert_eq!( + settings.cache.robots_txt.lifetimes(defaults), + crate::platform::CacheLifetimes { + memory: Duration::from_secs(5), + store: Duration::from_secs(600), + store_idle: Duration::from_secs(3600), + } + ); + } + + #[test] + fn a_loading_cache_lifetime_out_of_range_is_refused_naming_it() { + for (field, value, least) in [ + ("store_seconds", "0", 1), + ("store_idle_seconds", "0", 1), + ("memory_seconds", "31536001", 0), + ] { + let toml_str = format!( + "{} + +[cache.robots-txt] +{field} = {value} +", + crate_test_settings_str() + ); + let error = Settings::from_toml(&toml_str) + .expect_err("a lifetime out of range should not load"); + let message = format!("{error:?}"); + assert!( + message.contains(&format!( + "[cache.robots-txt] {field} is {value}, and must be from {least} to 31536000" + )), + "{field}: {message}" + ); + } + } + + #[test] + fn unset_loading_cache_lifetimes_are_not_written_back() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse the settings"); + let written = serde_json::to_value(&settings.cache).expect("should serialize"); + + assert_eq!(written, serde_json::json!({ "asset_rules": [] })); + } + + #[test] + fn a_loading_cache_module_is_read_and_written_back() { + let toml_str = format!( + r#"{} + + [cache.robots-txt] + module = "51degrees" + "#, + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml_str).expect("should parse the selection"); + + assert_eq!( + settings.cache.robots_txt.module.as_deref(), + Some("51degrees") + ); + assert!(!settings.cache.robots_txt.is_default()); + assert_eq!( + settings.cache.loading_caches(), + [(super::ROBOTS_TXT_CACHE, &settings.cache.robots_txt)], + "should list the robots.txt cache by its name" + ); + let written = serde_json::to_value(&settings.cache).expect("should serialize"); + assert_eq!( + written, + serde_json::json!({ "asset_rules": [], "robots-txt": { "module": "51degrees" } }) + ); + } + #[test] fn cache_asset_rule_nextjs_preset_is_operator_controlled() { let toml_str = format!( @@ -5120,6 +7199,140 @@ mod tests { ); } + #[test] + fn module_selection_allows_no_module_for_stateless_operation() { + let ec = Ec::default(); + assert!(ec.module.is_none(), "default Ec selects no module"); + ec.validate_module_selection() + .expect("should allow no module selected and run statelessly"); + } + + #[test] + fn selecting_an_implementation_that_needs_settings_without_its_block_is_rejected() { + // The built-in HMAC module has a required passphrase, so selecting + // it with no `[ec.hmac]` block is a deployment that would run + // stateless under a selector saying otherwise. + let toml_str = crate_test_settings_str_with_ec_section("[ec]\nmodule = \"hmac\"\n"); + + let err = Settings::from_toml(&toml_str) + .expect_err("an implementation with no settings block should fail at startup"); + assert!( + format!("{err:?}").contains("`hmac` is selected but has no `[ec.hmac]` configuration"), + "should name the missing block: {err:?}" + ); + } + + #[test] + fn selecting_a_module_with_no_settings_needs_no_block() { + // A block exists only when a module has settings, and only the + // adapter that injects a module knows whether it has any, so a + // selector naming one core does not supply is left to + // `build_module`, which is where the injected modules are known. + let toml_str = crate_test_settings_str_with_ec_section("[ec]\nmodule = \"acme\"\n"); + + let settings = + Settings::from_toml(&toml_str).expect("a module with no settings should need no block"); + assert!( + settings.ec.module_blocks.is_empty(), + "the selection should stand on its own with no block configured" + ); + } + + /// The fixture settings with `line` added to `[ec]`. + fn settings_toml_with_ec_line(line: &str) -> String { + crate_test_settings_str_with_ec_section(&format!( + "[ec]\nmodule = \"hmac\"\n{line}\n\n[ec.hmac]\n\ + passphrase = \"test-secret-key-32-bytes-minimum\"\n" + )) + } + + /// The `resolve_allowed_origins` line listing `entries`, each written as a + /// TOML basic string. + fn resolve_allowed_origins_line(entries: &[&str]) -> String { + let written: Vec = entries + .iter() + .map(|entry| serde_json::to_string(entry).expect("should write a string")) + .collect(); + format!("resolve_allowed_origins = [{}]", written.join(", ")) + } + + #[test] + fn resolve_allowed_origins_load_only_as_bare_origins() { + let accepted: [(&str, Option<&[&str]>); 7] = [ + ("an absent list", None), + ("an empty list", Some(&[])), + ("https with no port", Some(&["https://www.example.com"])), + ("https with :443", Some(&["https://www.example.com:443"])), + ("http with :80", Some(&["http://www.example.com:80"])), + ( + "a non-default port", + Some(&["https://www.example.com:8443"]), + ), + ( + "the generator's https://{domain} shape", + Some(&["https://www.example.com", "https://m.example.com"]), + ), + ]; + for (label, entries) in accepted { + let line = entries + .map(resolve_allowed_origins_line) + .unwrap_or_default(); + + let settings = Settings::from_toml(&settings_toml_with_ec_line(&line)) + .unwrap_or_else(|err| panic!("should load {label}: {err:?}")); + + assert_eq!( + settings.ec.resolve_allowed_origins, + entries.unwrap_or_default(), + "should keep the entries of {label} as written" + ); + } + + let refused = [ + ("https://www.example.com/", NotAnOrigin::Path), + ("https://www.example.com/path", NotAnOrigin::Path), + ("https://www.example.com\\", NotAnOrigin::Path), + ("https://www.example.com?a=1", NotAnOrigin::Query), + ("https://www.example.com#top", NotAnOrigin::Fragment), + ("https://user@www.example.com", NotAnOrigin::Userinfo), + ("https://user:pass@www.example.com", NotAnOrigin::Userinfo), + ("www.example.com", NotAnOrigin::Scheme), + ("www.example.com:443", NotAnOrigin::Scheme), + ("ftp://www.example.com", NotAnOrigin::Scheme), + ("wss://www.example.com", NotAnOrigin::Scheme), + ("null", NotAnOrigin::Scheme), + ("", NotAnOrigin::Empty), + ("https://", NotAnOrigin::Host), + ("https://www.example.com ", NotAnOrigin::Host), + ]; + for (entry, reason) in refused { + let line = resolve_allowed_origins_line(&["https://www.example.com", entry]); + + let Err(err) = Settings::from_toml(&settings_toml_with_ec_line(&line)) else { + panic!("should refuse `{entry}` at load"); + }; + + assert!( + format!("{err:?}") + .contains(&format!("resolve_allowed_origins entry `{entry}` {reason}")), + "should name `{entry}` and say it {reason}: {err:?}" + ); + } + } + + #[test] + fn a_resolve_allowed_origin_with_a_trailing_slash_is_refused_at_load() { + let line = resolve_allowed_origins_line(&["https://www.example.com/"]); + + let err = Settings::from_toml(&settings_toml_with_ec_line(&line)) + .expect_err("should refuse an entry that would never match a request's Origin"); + + assert!( + format!("{err:?}").contains("`https://www.example.com/`"), + "should name the entry: {err:?}" + ); + } + #[test] fn cache_asset_rule_globs_respect_path_separators() { let toml_str = format!( @@ -5226,6 +7439,25 @@ mod tests { ); } + #[test] + fn module_blocks_without_a_selector_are_rejected() { + // A half-migrated configuration that carries an [ec.hmac] block but + // never selects it would silently run stateless, so it is rejected at + // startup instead. + let toml_str = crate_test_settings_str().replace("module = \"hmac\"\n", ""); + + let err = Settings::from_toml(&toml_str) + .expect_err("a module block with no selector should fail at startup"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + } + #[test] fn cache_asset_rule_policy_validation_rejects_unsafe_config() { let missing_ttl = format!( @@ -5339,6 +7571,78 @@ mod tests { ); } + #[test] + fn legacy_passphrase_migrates_to_the_hmac_module() { + let mut ec = Ec { + passphrase: Some(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())), + ..Ec::default() + }; + ec.migrate_legacy_ec_layout() + .expect("should migrate the deprecated form"); + assert_eq!( + ec.module.as_ref(), + Some(&EcModuleSelection::from(HMAC_MODULE_KEY)), + "the deprecated passphrase should select the hmac module" + ); + assert_eq!( + hmac_passphrase(&ec, HMAC_MODULE_KEY), + "test-secret-key-32-bytes-minimum", + "the passphrase should move into the hmac block" + ); + assert!( + ec.passphrase.is_none(), + "the deprecated field should be consumed by the migration" + ); + } + + /// The crate test configuration with its `[ec]` section rewritten to the + /// deprecated single-passphrase form. + fn legacy_ec_settings_str(passphrase: &str) -> String { + let legacy = crate_test_settings_str_with_ec_section(&format!( + "[ec]\npassphrase = \"{passphrase}\"\n" + )); + assert!( + !legacy.contains("[ec.hmac]"), + "the legacy configuration should carry no module block" + ); + legacy + } + + #[test] + fn a_legacy_passphrase_is_held_to_the_passphrase_rules() { + // Validation runs before the migration and the deprecated field + // carries no check of its own, so the migration itself has to apply + // the passphrase rules. Without that, a value the new `[ec.hmac]` + // block rejects would still start a deployment from the old location. + let short = Settings::from_toml(&legacy_ec_settings_str("short")) + .expect_err("a short legacy passphrase should be rejected"); + assert!( + format!("{short:?}").contains("passphrase (deprecated) is invalid"), + "should name the deprecated passphrase as the fault: {short:?}" + ); + + let empty = Settings::from_toml(&legacy_ec_settings_str("")) + .expect_err("an empty legacy passphrase should be rejected"); + assert!( + format!("{empty:?}").contains("passphrase (deprecated) is invalid"), + "should name the deprecated passphrase as the fault: {empty:?}" + ); + + let settings = + Settings::from_toml(&legacy_ec_settings_str("test-secret-key-32-bytes-minimum")) + .expect("a legacy passphrase of adequate length should still start"); + assert_eq!( + settings.ec.module.as_ref(), + Some(&EcModuleSelection::from(HMAC_MODULE_KEY)), + "an adequate legacy passphrase should still select the hmac module" + ); + assert_eq!( + hmac_passphrase(&settings.ec, HMAC_MODULE_KEY), + "test-secret-key-32-bytes-minimum", + "an adequate legacy passphrase should still move into the hmac block" + ); + } + #[test] fn cache_asset_rule_validation_rejects_invalid_config() { let duplicate_ids = format!( @@ -5417,22 +7721,504 @@ mod tests { } #[test] - fn validate_rejects_trailing_slash_in_origin_url() { + fn legacy_passphrase_alongside_module_config_is_rejected() { + let mut ec = Ec { + passphrase: Some(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())), + module: Some(EcModuleSelection::from(HMAC_MODULE_KEY)), + ..Ec::default() + }; + let err = ec + .migrate_legacy_ec_layout() + .expect_err("both forms present should be rejected"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + } + + #[test] + fn an_unknown_key_in_the_hmac_module_block_is_rejected() { + // A mistyped key dropped silently would leave the setting the + // operator meant to change at its default. let toml_str = crate_test_settings_str().replace( - r#"origin_url = "https://origin.test-publisher.com""#, - r#"origin_url = "https://origin.test-publisher.com/""#, + "passphrase = \"test-secret-key-32-bytes-minimum\"", + "passphrase = \"test-secret-key-32-bytes-minimum\"\n typo_key = \"x\"", + ); + assert!( + toml_str.contains("typo_key"), + "the test configuration should carry the unknown key" ); - let result = Settings::from_toml(&toml_str); + let err = Settings::from_toml(&toml_str) + .expect_err("an unknown key in [ec.hmac] should be rejected"); assert!( - result.is_err(), - "origin_url ending with '/' should fail validation" + format!("{err:?}").contains("typo_key"), + "should name the unknown key: {err:?}" ); } #[test] - fn validate_rejects_invalid_publisher_domains() { - for domain in [ + fn module_none_is_explicit_stateless() { + let ec = Ec { + module: Some(EcModuleSelection::None), + ..Ec::default() + }; + ec.validate_module_selection() + .expect("explicit none with no blocks should be valid"); + } + + #[test] + fn module_none_with_configured_blocks_is_rejected() { + let mut ec = Ec::default(); + select_hmac_module(&mut ec, HMAC_MODULE_KEY, "test-secret-key-32-bytes-minimum"); + ec.module = Some(EcModuleSelection::None); + assert!( + ec.validate_module_selection().is_err(), + "none alongside configured blocks should be rejected" + ); + } + + #[test] + fn device_module_defaults_to_builtin_and_rejects_unknown() { + let config = DeviceConfig::default(); + assert_eq!( + config.module_key(), + "builtin", + "no selector should default to the built-in module" + ); + config + .validate_module_selection() + .expect("should validate the built-in default"); + + let fastly = DeviceConfig { + module: Some("fastly".to_owned()), + }; + fastly + .validate_module_selection() + .expect("should validate the fastly opt-in"); + + // As with geo, a key core does not know is not a settings error, + // because a module's name is a legitimate value here too. + let module_key = DeviceConfig { + module: Some("acme".to_owned()), + }; + module_key + .validate_module_selection() + .expect("a module id should be accepted by settings validation"); + + // And as with geo, the rejection happens at registry build, so a + // mistyped selector cannot fall back to the built-in module in + // silence. + let mut settings = crate::test_support::tests::create_test_settings(); + settings.device.module = Some("acme".to_owned()); + let error = match crate::integrations::IntegrationRegistry::new(&settings) { + Ok(_) => { + panic!("a device module no module supplies should be rejected at registry build") + } + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("acme") && message.contains("[device] module"), + "the error should name the selector and the module, got: {message}" + ); + } + + #[test] + fn an_unselected_module_block_is_rejected() { + // A vendor selector with the vendor block present, plus a stray hmac + // block, is almost always a stale or mistyped configuration. + let toml_str = crate_test_settings_str().replace( + "module = \"hmac\"", + "module = \"acme\"\n\n [ec.acme]\n api_key = \"example\"", + ); + let err = Settings::from_toml(&toml_str) + .expect_err("a configured but unselected block should fail at startup"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + } + + #[test] + fn a_host_signals_block_is_read_as_the_built_in_modules_settings() { + // The block is the built-in module's own settings, rather than kept as + // the raw values of a module an adapter injects. + let settings = Settings::from_toml(&crate_test_settings_str_with_ec_section(&format!( + "[ec] +module = \"{HOST_SIGNALS_MODULE_KEY}\" + +\ + [ec.{HOST_SIGNALS_MODULE_KEY}] +\ + passphrase = \"test-secret-key-32-bytes-minimum\" +" + ))) + .expect("a host_signals selection with its block should load"); + assert!( + settings + .ec + .module_blocks + .get(HOST_SIGNALS_MODULE_KEY) + .and_then(EcModuleBlock::host_signals_settings) + .is_some(), + "the block should be read as the host-signal module's settings" + ); + } + #[test] + fn a_labeled_block_the_selector_does_not_name_is_rejected() { + // A block under a label is still a module block, so it is held to + // the rule every block is held to, which is that the selector names + // it. + let toml_str = crate_test_settings_str_with_ec_section( + r#"[ec] +module = "hmac" + +[ec.hmac] +passphrase = "test-secret-key-32-bytes-minimum" + +[ec.primary] +implementation = "hmac" +passphrase = "another-test-secret-key-32-bytes" +"#, + ); + + let err = Settings::from_toml(&toml_str) + .expect_err("a labeled block the selector does not name should fail at startup"); + let message = format!("{err:?}"); + assert!( + message.contains("[ec.primary] is configured but `hmac` is selected"), + "should name the unselected labeled block, got: {message}" + ); + } + + #[test] + fn the_removed_providers_table_is_rejected_with_its_new_location() { + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"hmac\"\n\n[ec.providers.hmac]\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ); + + let err = Settings::from_toml(&toml_str) + .expect_err("the removed [ec.providers] table should be rejected"); + let message = format!("{err:?}"); + assert!( + message.contains("[ec.providers] is no longer read") && message.contains("[ec.hmac]"), + "should send the operator to the new location, got: {message}" + ); + } + + #[test] + fn a_key_under_ec_that_is_not_a_table_is_an_unknown_field() { + // Holding the module blocks alongside the fixed keys costs the + // section serde's own unknown-key check, so a mistyped setting has to + // be caught where the blocks are read. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"hmac\"\nec_stor = \"ec_identity_store\"\n\n[ec.hmac]\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ); + + let err = + Settings::from_toml(&toml_str).expect_err("a mistyped [ec] key should be rejected"); + let message = format!("{err:?}"); + assert!( + message.contains("unknown field `ec_stor`") && message.contains("`ec_store`"), + "should name the mistyped key and the keys it could have been, got: {message}" + ); + } + + #[test] + fn geo_module_accepts_default_platform_and_none_and_rejects_unknown() { + let config = GeoConfig::default(); + assert!( + config.module.is_none(), + "geo should default to no selector, which selects no geo module" + ); + config + .validate_module_selection() + .expect("should validate the default of running without geolocation"); + + let platform = GeoConfig { + module: Some("platform".to_owned()), + assume_single_jurisdiction: false, + }; + platform + .validate_module_selection() + .expect("should validate the explicit platform selection"); + + let none = GeoConfig { + module: Some("none".to_owned()), + assume_single_jurisdiction: false, + }; + none.validate_module_selection() + .expect("should validate the explicit opt-out of geolocation"); + + // A key core does not know is not a settings error, because a + // module's name is a legitimate value and a closed list here would + // shut every module out of geo. Settings accepts it and the registry + // decides. + let module_key = GeoConfig { + module: Some("acme".to_owned()), + assume_single_jurisdiction: false, + }; + module_key + .validate_module_selection() + .expect("a module id should be accepted by settings validation"); + + // The rejection moved to registry build, where it is known whether any + // module supplies the name. With no module supplying `acme`, building + // the registry fails and the error names the selector and the module. + let mut settings = crate::test_support::tests::create_test_settings(); + settings.geo.module = Some("acme".to_owned()); + // `IntegrationRegistry` is not `Debug`, so the error is taken by match + // rather than `expect_err`. + let error = match crate::integrations::IntegrationRegistry::new(&settings) { + Ok(_) => { + panic!("a geo module no module supplies should be rejected at registry build") + } + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("acme") && message.contains("[geo] module"), + "the error should name the selector and the module, got: {message}" + ); + } + + #[test] + fn the_reserved_ec_keys_are_the_ones_the_section_reads() { + // The list is written out for the unknown-field message and the + // reserved-name check, so it has to stay level with the struct. + let ec = Ec { + passphrase: Some(Redacted::new("test-secret-key-32-bytes-minimum".to_owned())), + ..Ec::default() + }; + let value = serde_json::to_value(&ec).expect("should serialize the [ec] section"); + let written = value + .as_object() + .expect("the section should serialize as a table") + .keys() + .map(String::as_str) + .collect::>(); + + assert_eq!( + written, + EC_SECTION_KEYS.iter().copied().collect::>(), + "EC_SECTION_KEYS should name every key the [ec] section reads as its own" + ); + } + + #[test] + fn unknown_keys_in_module_sections_are_rejected() { + // A mistyped key must fail at startup rather than silently selecting + // a default behind the operator's back. + for (section, bad_key) in [ + ("[geo]", "providr = \"platform\""), + ("[device]", "providr = \"builtin\""), + ] { + let toml_str = format!( + "{}\n\n {section}\n {bad_key}\n", + crate_test_settings_str() + ); + assert!( + Settings::from_toml(&toml_str).is_err(), + "an unknown key in {section} should be rejected" + ); + } + + let toml_str = crate_test_settings_str() + .replace("[ec.hmac]", "[ec.hmac]\n unexpected = \"value\""); + assert!( + Settings::from_toml(&toml_str).is_err(), + "an unknown key in [ec.hmac] should be rejected" + ); + } + + #[test] + fn a_reserved_key_cannot_name_a_module() { + let toml_str = crate_test_settings_str_with_ec_section("[ec]\nmodule = \"ec_store\"\n"); + + let err = + Settings::from_toml(&toml_str).expect_err("a reserved key should not name a module"); + assert!( + format!("{err:?}").contains("names a key the `[ec]` section reads as its own setting"), + "should say why the name cannot be used: {err:?}" + ); + } + + #[test] + fn module_names_and_implementations_follow_the_module_name_rule() { + for ec_section in [ + "[ec]\nmodule = \"Primary\"\n", + "[ec]\nmodule = \"primary\"\n\n[ec.primary]\nimplementation = \"Hmac\"\n", + "[ec]\nmodule = \"primary\"\n\n[ec.Primary]\nimplementation = \"hmac\"\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ] { + let err = Settings::from_toml(&crate_test_settings_str_with_ec_section(ec_section)) + .expect_err("a name that is not a module name should be rejected"); + assert!( + format!("{err:?}").contains("is not a module name"), + "should hold `{ec_section}` to the module name rule: {err:?}" + ); + } + } + + #[test] + fn an_hmac_block_without_its_passphrase_names_the_block() { + // The implementation the block names decides how its settings are + // read, so the block the operator wrote is what the error names. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"primary\"\n\n[ec.primary]\nimplementation = \"hmac\"\n", + ); + + let err = Settings::from_toml(&toml_str) + .expect_err("an hmac block without its passphrase should be rejected"); + let message = format!("{err:?}"); + assert!( + message.contains("[ec.primary] is invalid") && message.contains("passphrase"), + "should name the block and the setting it lacks, got: {message}" + ); + } + + #[test] + fn a_labeled_block_round_trips_through_serialization() { + // The pushed configuration is the serialized settings, so a label and + // the implementation it names have to survive being written back. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"primary\"\n\n[ec.primary]\nimplementation = \"hmac\"\npassphrase = \"test-secret-key-32-bytes-minimum\"\n", + ); + let settings = Settings::from_toml(&toml_str).expect("should parse a labeled module block"); + + let value = serde_json::to_value(&settings).expect("should serialize settings"); + assert_eq!( + value["ec"]["primary"]["implementation"], "hmac", + "the label should keep the implementation it names" + ); + + let reparsed = + Settings::from_json_value(value).expect("should reparse the serialized settings"); + assert_eq!( + reparsed.ec.module_blocks.implementation("primary"), + HMAC_MODULE_KEY, + "the implementation should survive the round trip" + ); + assert_eq!( + hmac_passphrase(&reparsed.ec, "primary"), + "test-secret-key-32-bytes-minimum" + ); + + // A block under the implementation's own name is written back as it + // was, without gaining a key the operator never wrote. + let written = serde_json::to_value(create_test_settings()) + .expect("should serialize the test settings"); + assert!( + written["ec"]["hmac"].get("implementation").is_none(), + "an unlabeled block should not gain an implementation key: {}", + written["ec"]["hmac"] + ); + } + + #[test] + fn an_injected_modules_settings_are_kept_as_written() { + // Core never names a vendor, so the block of a module an adapter + // injects is kept as the values it held for that adapter to read. + let toml_str = crate_test_settings_str_with_ec_section( + "[ec]\nmodule = \"acme\"\n\n[ec.acme]\nendpoint = \"https://ec.acme.example.com\"\n", + ); + let settings = Settings::from_toml(&toml_str).expect("should parse a vendor module block"); + + let block = settings + .ec + .module_blocks + .get("acme") + .expect("should configure the acme block"); + assert_eq!( + settings.ec.module_blocks.implementation("acme"), + "acme", + "a block that names no implementation is its own name's" + ); + let EcModuleSettings::Injected(injected) = &block.settings else { + panic!("a vendor block should keep its settings as the raw values it held"); + }; + assert_eq!(injected["endpoint"], "https://ec.acme.example.com"); + + // They survive being written back into the pushed configuration, so + // the adapter reads what the operator wrote. + let written = serde_json::to_value(&settings).expect("should serialize settings"); + assert_eq!( + written["ec"]["acme"]["endpoint"], + "https://ec.acme.example.com" + ); + } + + #[test] + fn the_compiled_permission_policy_validates_at_startup() { + // The compiled-in sample declares its top node, so startup + // accepts it. A policy that omitted `group` or `jurisdiction` would be + // rejected here rather than panicking on the first lookup, which the + // parser tests in `permissions` cover directly. + GeoConfig::validate_permission_policy() + .expect("the compiled-in sample should validate at startup"); + } + + #[test] + fn ec_without_geo_requires_the_single_jurisdiction_acknowledgment() { + // The base test settings acknowledge single-jurisdiction operation. + // Removing the acknowledgment while an EC module is configured and + // no geo module is selected must fail at startup. + let toml_str = crate_test_settings_str().replace("assume_single_jurisdiction = true\n", ""); + let err = Settings::from_toml(&toml_str) + .expect_err("an EC module with no geo module needs the acknowledgment"); + assert!( + matches!( + err.current_context(), + TrustedServerError::Configuration { .. } + ), + "should be a configuration error, got: {:?}", + err.current_context() + ); + + // Selecting a geo module removes the requirement. + let toml_str = crate_test_settings_str() + .replace("assume_single_jurisdiction = true\n", "") + .replace("[geo]", "[geo]\n module = \"platform\""); + Settings::from_toml(&toml_str) + .expect("a geo module resolves jurisdictions, so no acknowledgment is needed"); + + // With no EC module there is no jurisdiction consumer to protect. + let toml_str = crate_test_settings_str() + .replace("assume_single_jurisdiction = true\n", "") + .replace("module = \"hmac\"", "") + .replace( + "[ec.hmac]\n passphrase = \"test-secret-key-32-bytes-minimum\"", + "", + ); + Settings::from_toml(&toml_str) + .expect("stateless operation needs no jurisdiction acknowledgment"); + } + + #[test] + fn validate_rejects_trailing_slash_in_origin_url() { + let toml_str = crate_test_settings_str().replace( + r#"origin_url = "https://origin.test-publisher.com""#, + r#"origin_url = "https://origin.test-publisher.com/""#, + ); + + let result = Settings::from_toml(&toml_str); + assert!( + result.is_err(), + "origin_url ending with '/' should fail validation" + ); + } + + #[test] + fn validate_rejects_invalid_publisher_domains() { + for domain in [ "", ".example.com", "example.com.", @@ -5596,18 +8382,6 @@ origin_host_header_overide = "www.example.com""#, } } - #[test] - fn prepare_runtime_rejects_invalid_handler_regex() { - let toml_str = crate_test_settings_str().replace(r#"path = "^/secure""#, r#"path = "(""#); - - let err = Settings::from_toml(&toml_str).expect_err("should reject invalid handler regex"); - assert!( - err.to_string() - .contains("Handler path regex `(` failed to compile"), - "should describe the invalid handler regex" - ); - } - #[test] fn test_settings_missing_required_fields() { let re = Regex::new(r"origin_url = .*").expect("regex should compile"); @@ -5815,32 +8589,6 @@ source_domain = "partner.example.com" ); } - #[test] - fn is_placeholder_handler_password_rejects_known_template_value() { - assert!( - Handler::is_placeholder_password("replace-with-admin-password-32-bytes"), - "init-template handler password should be rejected" - ); - } - - #[test] - fn reject_placeholder_secrets_includes_handler_passwords() { - let mut settings = - Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); - settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); - settings.handlers[0].password = - Redacted::new("replace-with-admin-password-32-bytes".to_owned()); - - let err = settings - .reject_placeholder_secrets() - .expect_err("should reject placeholder handler password"); - assert!( - format!("{err:?}").contains("handlers"), - "error should mention handler password field" - ); - } - fn test_partner_with_pull_token(ts_pull_token: &str) -> EcPartner { test_partner_with_tokens(None, ts_pull_token) } @@ -5867,7 +8615,6 @@ source_domain = "partner.example.com" let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); settings.ec.partners = vec![test_partner_with_pull_token( "partner-api-token-32-bytes-minimum", )]; @@ -5886,7 +8633,6 @@ source_domain = "partner.example.com" let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); settings.ec.partners = vec![test_partner_with_pull_token( "unit-test-realistic-pull-sync-token-32-bytes-min", )]; @@ -5901,7 +8647,6 @@ source_domain = "partner.example.com" let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); - settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); settings.ec.partners = vec![test_partner_with_tokens( Some("partner-api-token-32-bytes-minimum"), "replace-with-partner-api-token-32-bytes-minimum", @@ -5921,26 +8666,6 @@ source_domain = "partner.example.com" ); } - #[test] - fn is_unusable_store_id_rejects_placeholders_empty_and_padded_values() { - for placeholder in RequestSigning::STORE_ID_PLACEHOLDERS { - assert!( - RequestSigning::is_unusable_store_id(placeholder), - "should reject placeholder store id '{placeholder}'" - ); - } - for bad in ["", " ", " 01GCFG ", "01GCFG "] { - assert!( - RequestSigning::is_unusable_store_id(bad), - "should reject unusable store id '{bad}'" - ); - } - assert!( - !RequestSigning::is_unusable_store_id("01GCFG"), - "should accept a clean store id" - ); - } - #[test] fn test_settings_empty_toml() { let toml_str = ""; @@ -5970,7 +8695,7 @@ source_domain = "partner.example.com" } #[test] - fn test_handlers_override_with_env() { + fn test_ec_partners_override_with_indexed_env() { let toml_str = crate_test_settings_str(); let origin_key = format!( @@ -5979,127 +8704,50 @@ source_domain = "partner.example.com" ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - // Override handler 0 via env vars - let path_key_0 = format!( - "{}{}HANDLERS{}0{}PATH", + let partner_0_name_key = format!( + "{}{}EC{}PARTNERS{}0{}NAME", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let username_key_0 = format!( - "{}{}HANDLERS{}0{}USERNAME", + let partner_0_source_domain_key = format!( + "{}{}EC{}PARTNERS{}0{}SOURCE_DOMAIN", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let password_key_0 = format!( - "{}{}HANDLERS{}0{}PASSWORD", + let partner_0_openrtb_atype_key = format!( + "{}{}EC{}PARTNERS{}0{}OPENRTB_ATYPE", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - // Admin handler at index 1 (required for admin endpoint coverage) - let path_key_1 = format!( - "{}{}HANDLERS{}1{}PATH", + let partner_0_bidstream_enabled_key = format!( + "{}{}EC{}PARTNERS{}0{}BIDSTREAM_ENABLED", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let username_key_1 = format!( - "{}{}HANDLERS{}1{}USERNAME", + let partner_0_api_token_key = format!( + "{}{}EC{}PARTNERS{}0{}API_TOKEN", ENVIRONMENT_VARIABLE_PREFIX, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR ); - let password_key_1 = format!( - "{}{}HANDLERS{}1{}PASSWORD", + let partner_1_name_key = format!( + "{}{}EC{}PARTNERS{}1{}NAME", ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - - temp_env::with_vars( - [ - (origin_key, Some("https://origin.test-publisher.com")), - (path_key_0, Some("^/env-handler")), - (username_key_0, Some("env-user")), - (password_key_0, Some("env-pass")), - (path_key_1, Some("^/_ts/admin")), - (username_key_1, Some("admin")), - (password_key_1, Some("admin-pass")), - ], - || { - let settings = - Settings::from_toml_and_env(&toml_str).expect("Settings should load from env"); - assert_eq!(settings.handlers.len(), 2); - let handler = &settings.handlers[0]; - assert_eq!(handler.path, "^/env-handler"); - assert_eq!(handler.username.expose(), "env-user"); - assert_eq!(handler.password.expose(), "env-pass"); - }, - ); - } - - #[test] - fn test_ec_partners_override_with_indexed_env() { - let toml_str = crate_test_settings_str(); - - let origin_key = format!( - "{}{}PUBLISHER{}ORIGIN_URL", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_name_key = format!( - "{}{}EC{}PARTNERS{}0{}NAME", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_source_domain_key = format!( - "{}{}EC{}PARTNERS{}0{}SOURCE_DOMAIN", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_openrtb_atype_key = format!( - "{}{}EC{}PARTNERS{}0{}OPENRTB_ATYPE", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_bidstream_enabled_key = format!( - "{}{}EC{}PARTNERS{}0{}BIDSTREAM_ENABLED", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_0_api_token_key = format!( - "{}{}EC{}PARTNERS{}0{}API_TOKEN", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let partner_1_name_key = format!( - "{}{}EC{}PARTNERS{}1{}NAME", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, + ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR, ENVIRONMENT_VARIABLE_SEPARATOR @@ -6190,8 +8838,10 @@ source_domain = "partner.example.com" ); } + /// An environment variable can give a handler to a configuration that + /// carries none, and that is refused as the list itself is. #[test] - fn test_invalid_handler_override_fails_during_runtime_preparation() { + fn a_handler_given_by_an_environment_variable_is_refused_too() { let toml_str = crate_test_settings_str(); let origin_key = format!( @@ -6212,9 +8862,14 @@ source_domain = "partner.example.com" origin_key, Some("https://origin.test-publisher.com"), || { - temp_env::with_var(path_key, Some("("), || { - let _ = Settings::from_toml_and_env(&toml_str) - .expect_err("should reject invalid handler regex override"); + temp_env::with_var(path_key, Some("^/secure"), || { + let error = Settings::from_toml_and_env(&toml_str) + .expect_err("should refuse a handler an environment variable gives"); + + assert!( + format!("{error:?}").contains("`[[handlers]]` is not read"), + "should say the list is not read: {error:?}" + ); }); }, ); @@ -6454,11 +9109,6 @@ source_domain = "partner.example.com" let from_toml = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "example.com" cookie_domain = ".example.com" @@ -6466,7 +9116,13 @@ source_domain = "partner.example.com" proxy_secret = "unit-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse settings without max_buffered_body_bytes"); @@ -6484,11 +9140,6 @@ source_domain = "partner.example.com" // silently breaking traffic. let result = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "example.com" cookie_domain = ".example.com" @@ -6496,155 +9147,261 @@ source_domain = "partner.example.com" proxy_secret = "unit-test-proxy-secret" max_buffered_body_bytes = 0 + [geo] + assume_single_jurisdiction = true + [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" "#, ); + let error = result.expect_err("should reject a zero buffered-body cap"); assert!( - result.is_err(), - "publisher.max_buffered_body_bytes = 0 must fail config validation" + error.to_string().contains("max_buffered_body_bytes"), + "the rejection should be for the zero cap, not another validation, got: {error}" ); } - #[test] - fn test_disabled_integration_does_not_register() { - use crate::integrations::testlight::TestlightConfig; - use serde_json::json; + /// The settings of an example module that takes none. + #[derive(Debug, Deserialize, Validate)] + #[serde(deny_unknown_fields)] + struct NoSettings {} - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "testlight", - &json!({ - "enabled": false, - "endpoint": "https://testlight.test/auction", - "rewrite_scripts": true, - }), - ) - .expect("should insert integration config"); + impl IntegrationConfig for NoSettings {} - let config = settings - .integration_config::("testlight") - .expect("integration parsing should succeed"); + /// The settings of an example module that requires one. + #[derive(Debug, Deserialize, Validate)] + #[serde(deny_unknown_fields)] + struct OneRequiredSetting { + #[validate(url)] + endpoint: String, + } + + impl IntegrationConfig for OneRequiredSetting {} + + /// The example modules' names, in a section of their own type. + const EXAMPLE_SECTION: &str = "example"; + const PLAIN_MODULE: &str = "example.plain"; + const ENDPOINT_MODULE: &str = "example.endpoint"; + + /// A module no section selects has no configuration, whatever else the + /// settings hold. + #[test] + fn a_module_that_is_not_selected_has_no_configuration() { + let settings = create_test_settings(); - assert!(config.is_none(), "Disabled integrations should be skipped"); + assert!( + !settings.selects_module(ENDPOINT_MODULE), + "the shared fixture should not select the example module" + ); + assert!( + settings + .module_config::(ENDPOINT_MODULE) + .expect("reading an unselected module should succeed") + .is_none(), + "a module that is not selected should have no configuration" + ); } + /// A selected module with no table of its own is read from an empty one, + /// so one that takes no settings runs on its selection alone. #[test] - fn disabled_integration_can_omit_enabled_required_fields_and_skip_semantic_validation() { + fn a_selected_module_with_no_table_is_read_from_an_empty_one() { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "gpt", - &json!({ - "enabled": false, - }), - ) - .expect("should insert GPT config"); + settings.select_module(EXAMPLE_SECTION, PLAIN_MODULE); - let config = settings - .integration_config::("gpt") - .expect("minimal disabled GPT config should be ignored"); - assert!(config.is_none(), "disabled GPT config should be skipped"); - IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("disabled invalid integration config should not fail registry startup"); + assert!( + settings + .module_config::(PLAIN_MODULE) + .expect("a module that takes no settings should read from an empty table") + .is_some(), + "selecting the module should be the whole configuration" + ); } + /// The same empty table makes a module that requires a setting report the + /// setting it is missing, rather than starting without it. #[test] - fn minimal_disabled_prebid_deserializes_without_enabled_only_validation() { + fn a_selected_module_without_a_required_setting_names_it() { let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &json!({ - "enabled": false, - }), - ) - .expect("should insert prebid config"); + settings.select_module(EXAMPLE_SECTION, ENDPOINT_MODULE); - let config = settings - .integration_config::("prebid") - .expect("disabled prebid config should be ignored"); - assert!(config.is_none(), "disabled prebid config should be skipped"); - IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), + let error = settings + .module_config::(ENDPOINT_MODULE) + .expect_err("should reject a selected module with no endpoint"); + + let rendered = error.to_string(); + assert!( + rendered.contains("[example.endpoint]") && rendered.contains("endpoint"), + "should name the module's table and the missing setting: {rendered}" + ); + } + + /// A table written for a module its section does not select is refused, + /// rather than sitting in the configuration doing nothing. + #[test] + fn a_table_for_a_module_that_is_not_selected_is_refused() { + let toml = format!( + "{}\n[testing]\nmodule = \"example\"\n\n[testing.another]\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("should reject a table nothing selects"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("[testing.another] is configured") + && rendered.contains("does not select"), + "should name the table and the selection it is missing from: {rendered}" + ); + } + + /// A module type's section that selects nothing is refused, which is also + /// what a misspelt section of Trusted Server's own meets. + #[test] + fn a_section_that_selects_no_module_is_refused() { + let toml = format!( + "{}\n[framework.nextjs]\nrewrite_attributes = [\"href\"]\n", + crate_test_settings_str() + ); + + let error = + Settings::from_toml(&toml).expect_err("should reject a section selecting nothing"); + + assert!( + format!("{error:?}").contains("[framework] selects no module"), + "should name the section: {error:?}" + ); + } + + /// A key left in a module's table that its settings do not have, such as + /// the removed `enabled`, is refused when the module reads its table, so a + /// configuration cannot read as switched off while the module runs. + #[test] + fn an_enabled_key_left_in_a_table_is_refused() { + let toml = format!( + "{}\n[example]\nmodules = [\"endpoint\"]\n\n[example.endpoint]\nendpoint = \"https://endpoint.example\"\nenabled = false\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&toml).expect("core reads no module's table itself"); + + let error = settings + .module_config::(ENDPOINT_MODULE) + .expect_err("should reject a leftover enabled key"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("[example.endpoint]") && rendered.contains("enabled"), + "should name the table and the key: {rendered}" + ); + } + + /// Naming one module twice is a mistake rather than a way of running it + /// twice, so it is refused. + #[test] + fn naming_a_module_twice_is_refused() { + let toml = format!( + "{}\n[example]\nmodules = [\"endpoint\", \"endpoint\"]\n", + crate_test_settings_str() + ); + + let error = Settings::from_toml(&toml).expect_err("should reject a repeated name"); + + assert!( + format!("{error:?}").contains("more than once"), + "should report the repeated name: {error:?}" + ); + } + + /// The tables removed from the configuration are refused with the move + /// spelled out, rather than with a bare unknown-field error. + #[test] + fn the_removed_integration_tables_are_refused_with_directions() { + for (table, expected) in [ + ("[integrations.example]", "CHANGELOG.md"), + ( + "[integration]\nmodules = [\"example\"]", + "is no longer read", ), - ) - .expect("disabled default-enabled prebid config should not fail registry startup"); - build_orchestrator(&settings) - .expect("minimal disabled prebid config should not fail orchestrator startup"); + ] { + let toml = format!("{}\n{table}\n", crate_test_settings_str()); + + let error = Settings::from_toml(&toml).expect_err("should reject the removed table"); + let rendered = format!("{error:?}"); + + assert!( + rendered.contains("the section of its type") && rendered.contains(expected), + "should say where modules are selected now: {rendered}" + ); + } } + /// The same refusal reaches a configuration blob, which is the shape the + /// runtime loads rather than TOML. #[test] - fn disabled_removed_prebid_and_aps_fields_are_rejected() { - for (integration_id, removed_field) in [("prebid", "server_url"), ("aps", "account_id")] { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - integration_id, - &json!({ - "enabled": false, - (removed_field): "removed-value", - }), - ) - .expect("should insert removed integration config field"); - - let error = match integration_id { - "prebid" => settings - .integration_config::(integration_id) - .expect_err("should reject removed disabled Prebid field"), - "aps" => settings - .integration_config::(integration_id) - .expect_err("should reject removed disabled APS field"), - _ => unreachable!("test integration ID should be known"), - }; + fn json_settings_refuse_the_removed_integration_tables() { + for table in ["integrations", "integration"] { + let mut value = serde_json::to_value(create_test_settings()) + .expect("should serialize the test settings fixture to JSON"); + value + .as_object_mut() + .expect("settings should serialize as an object") + .insert(table.to_owned(), json!({ "example": {} })); + + let error = Settings::from_json_value(value) + .expect_err("should reject the removed table in JSON"); + assert!( - format!("{error:?}").contains(removed_field), - "should identify removed field `{removed_field}`: {error:?}" + format!("{error:?}").contains("the section of its type"), + "should say where modules are selected now: {error:?}" ); } } #[test] - fn enabled_invalid_integration_fails_registry_startup() { + fn invalid_settings_for_a_selected_module_fail_registry_startup() { + fn build( + settings: &Settings, + ) -> Result, Report> + { + Ok(settings + .module_config::(ENDPOINT_MODULE)? + .map(|_| crate::integrations::IntegrationRegistration::builder("endpoint").build())) + } + + fn validate(settings: &Settings) -> Result> { + settings + .module_config::(ENDPOINT_MODULE) + .map(|config| config.is_some()) + } + let mut settings = create_test_settings(); settings - .integrations - .insert_config( - "gpt", + .insert_module_config( + EXAMPLE_SECTION, + ENDPOINT_MODULE, &json!({ - "enabled": true, - "script_url": "not a url", + "endpoint": "not a url", }), ) - .expect("should insert GPT config"); + .expect("should insert the example module's table"); + let extra = [crate::integrations::IntegrationBuilder::new( + "endpoint", + "settings-tests", + build, + validate, + ) + .with_module_name(ENDPOINT_MODULE)]; - let err = match IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) { - Ok(_) => panic!("enabled invalid integration should fail registry startup"), + let err = match IntegrationRegistry::with_registrations(&settings, &extra) { + Ok(_) => panic!("a selected module with invalid settings should fail startup"), Err(err) => err, }; assert!( - err.to_string().contains("Integration 'gpt'"), - "should identify the invalid integration config" + format!("{err:?}").contains("[example.endpoint]"), + "should identify the invalid module table: {err:?}" ); } @@ -6700,11 +9457,8 @@ source_domain = "partner.example.com" #[test] fn test_auction_creative_processing_defaults_when_omitted() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - "#; + let toml_str = + crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\n"); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6720,12 +9474,10 @@ source_domain = "partner.example.com" #[test] fn test_auction_rewrite_creatives_accepts_explicit_false() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - rewrite_creatives = false - "#; + let toml_str = crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nenabled = true\nrewrite_creatives = false\n", + ); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6746,12 +9498,7 @@ source_domain = "partner.example.com" #[test] fn test_auction_allowed_context_keys_from_toml() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - allowed_context_keys = ["permutive_segments", "lockr_ids"] - "#; + let toml_str = crate_test_settings_str().replace("[auction]\n", "[auction]\nenabled = true\nallowed_context_keys = [\"permutive_segments\", \"lockr_ids\"]\n"); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert_eq!( settings.auction.allowed_context_keys, @@ -6761,12 +9508,10 @@ source_domain = "partner.example.com" #[test] fn test_auction_empty_allowed_context_keys_blocks_all() { - let toml_str = crate_test_settings_str() - + r#" - [auction] - enabled = true - allowed_context_keys = [] - "#; + let toml_str = crate_test_settings_str().replace( + "[auction]\n", + "[auction]\nenabled = true\nallowed_context_keys = []\n", + ); let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert!( settings.auction.allowed_context_keys.is_empty(), @@ -6785,6 +9530,7 @@ source_domain = "partner.example.com" "*.Example.Org".to_string(), ], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6805,6 +9551,7 @@ source_domain = "partner.example.com" "cdn.example.com".to_string(), ], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6820,6 +9567,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec!["*".to_string(), "tracker.com".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6835,6 +9583,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec!["*".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert!( @@ -6849,6 +9598,7 @@ source_domain = "partner.example.com" certificate_check: true, allowed_domains: vec![" ".to_string(), "\t".to_string()], asset_routes: vec![], + modules: SectionModules::default(), }; proxy.normalize(); assert!( @@ -6867,6 +9617,7 @@ source_domain = "partner.example.com" origin_url: " https://assets.example.com ".to_string(), ..Default::default() }], + modules: SectionModules::default(), }; proxy.normalize(); assert_eq!( @@ -6891,6 +9642,7 @@ source_domain = "partner.example.com" target_path: Some(" /rewritten/$1 ".to_string()), ..Default::default() }], + modules: SectionModules::default(), }; proxy.normalize(); @@ -7615,6 +10367,7 @@ source_domain = "partner.example.com" ..Default::default() }, ], + modules: SectionModules::default(), }; let route = proxy @@ -7643,6 +10396,7 @@ source_domain = "partner.example.com" ..Default::default() }, ], + modules: SectionModules::default(), }; let route = proxy @@ -7829,8 +10583,6 @@ source_domain = "partner.example.com" ); } - // --- admin endpoint coverage --- - #[test] fn test_publisher_rejects_cookie_domain_with_metacharacters() { for bad_domain in [ @@ -7857,357 +10609,22 @@ source_domain = "partner.example.com" ); } - /// Helper that returns a settings TOML string WITHOUT any admin handler, - /// for tests that need to verify uncovered-admin-endpoint behaviour. - fn settings_str_without_admin_handler() -> String { - r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [publisher] - domain = "test-publisher.com" - cookie_domain = ".test-publisher.com" - origin_url = "https://origin.test-publisher.com" - proxy_secret = "unit-test-proxy-secret" - - [ec] - passphrase = "test-secret-key-32-bytes-minimum" - - [request_signing] - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" - "# - .to_string() - } - - #[test] - fn uncovered_admin_endpoints_returns_all_when_no_handler_covers_admin() { - // Deserialize directly to bypass from_toml's admin validation, - // since this test exercises uncovered_admin_endpoints itself. - let settings: Settings = - toml::from_str(&settings_str_without_admin_handler()).expect("should deserialize TOML"); - let uncovered = settings - .uncovered_admin_endpoints() - .expect("should check admin coverage"); - assert_eq!( - uncovered, - vec![ - "/_ts/admin/keys/rotate", - "/_ts/admin/keys/deactivate", - "/_ts/admin/ec", - "/_ts/admin/ec/{id}", - "/_ts/admin/eids", - "/_ts/admin/cache/purge", - ], - "should report every admin endpoint as uncovered" - ); - } - - #[test] - fn uncovered_admin_endpoints_returns_empty_when_handler_covers_admin() { - let settings = create_test_settings(); - let uncovered = settings - .uncovered_admin_endpoints() - .expect("should check admin coverage"); - assert!( - uncovered.is_empty(), - "should report no uncovered admin endpoints when handler covers /_ts/admin" - ); - } - - #[test] - fn uncovered_admin_endpoints_detects_partial_coverage() { - let toml_str = settings_str_without_admin_handler() - + r#" - [[handlers]] - path = "^/_ts/admin/keys/rotate$" - username = "admin" - password = "secret" - "#; - // Deserialize directly to bypass from_toml's admin validation, - // since this test exercises uncovered_admin_endpoints itself. - let settings: Settings = toml::from_str(&toml_str).expect("should deserialize TOML"); - let uncovered = settings - .uncovered_admin_endpoints() - .expect("should check admin coverage"); - assert_eq!( - uncovered, - vec![ - "/_ts/admin/keys/deactivate", - "/_ts/admin/ec", - "/_ts/admin/ec/{id}", - "/_ts/admin/eids", - "/_ts/admin/cache/purge", - ], - "should detect the admin endpoints not covered by the narrow handler" - ); - } - - #[test] - fn from_toml_rejects_literal_parameter_template_auth_coverage() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[{]id[}]$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject literal parameter-template auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the concrete EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_lowercase_only_dynamic_admin_ec_auth_coverage() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[a-f0-9]{64}[.][a-z0-9]{6}$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject lowercase-only dynamic EC auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the mixed-case EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_placeholder_password_on_shadowing_admin_handler() { - // Handler selection is first-match-wins, so a narrow handler placed - // ahead of the admin matcher governs the EC IDs it matches. No probe - // enumerates those IDs, so the placeholder check cannot be limited to - // handlers inferred to cover an admin endpoint. - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/ec/[a-f0-9]{64}[.]zzzzzz$" - username = "admin" - password = "change-me-admin-password" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject placeholder password on shadowing admin handler"); - let message = format!("{error:?}"); - assert!( - message.contains("placeholder password"), - "should identify the placeholder handler password, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_weak_password_on_non_admin_handler() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/private" - username = "admin" - password = "changeme""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject a weak password on any handler"); - let message = format!("{error:?}"); - assert!( - message.contains("placeholder password"), - "should identify the weak handler password, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_sampled_id_only_dynamic_admin_ec_auth_coverage() { - // A handler anchored to the full EC ID grammar still leaves the rest of - // the route surface (malformed IDs, which the router accepts and the - // admin handler rejects with 400) unauthenticated, so coverage must not - // be inferred from ID-shaped samples. - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject ID-sampled dynamic EC auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the dynamic EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_rejects_prefix_anchored_admin_ec_auth_coverage() { - // `^/_ts/admin/ec/$` matches the prefix probe but no actual lookup. - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/$" - username = "admin" - password = "strong-test-password""#, - ); - - let error = Settings::from_toml(&toml_str) - .expect_err("should reject prefix-anchored dynamic EC auth coverage"); - let message = format!("{error:?}"); - assert!( - message.contains("/_ts/admin/ec/{id}"), - "should identify the dynamic EC route as uncovered, got: {message}" - ); - } - - #[test] - fn from_toml_accepts_prefix_matcher_admin_ec_auth_coverage() { - let toml_str = crate_test_settings_str().replace( - r#"path = "^/_ts/admin" - username = "admin" - password = "admin-pass""#, - r#"path = "^/_ts/admin/(keys/rotate|keys/deactivate|ec|eids|cache/purge)$" - username = "admin" - password = "strong-test-password" - - [[handlers]] - path = "^/_ts/admin/ec/" - username = "admin" - password = "strong-test-password""#, - ); - - Settings::from_toml(&toml_str) - .expect("should accept a prefix-level matcher for the dynamic EC route"); - } - - #[test] - fn from_toml_and_env_rejects_config_without_admin_handler() { - let origin_key = format!( - "{}{}PUBLISHER{}ORIGIN_URL", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - temp_env::with_var( - origin_key, - Some("https://origin.test-publisher.com"), - || { - let result = Settings::from_toml_and_env(&settings_str_without_admin_handler()); - assert!( - result.is_err(), - "should reject configuration when admin endpoints are not covered" - ); - let err = format!("{:?}", result.unwrap_err()); - assert!( - err.contains("No handler covers admin endpoint"), - "error should mention uncovered admin endpoints, got: {err}" - ); - }, - ); - } - - #[test] - fn from_toml_rejects_admin_handler_placeholder_password() { - let toml_str = crate_test_settings_str() - .replace(r#"password = "admin-pass""#, r#"password = "changeme""#); - - let result = Settings::from_toml(&toml_str); - assert!( - result.is_err(), - "should reject placeholder password on admin handler" - ); - let err = format!("{:?}", result.expect_err("should reject placeholder")); - assert!( - err.contains("placeholder password"), - "error should mention placeholder admin password, got: {err}" - ); - } - - #[test] - fn from_toml_accepts_non_placeholder_admin_password() { - let settings = Settings::from_toml(&crate_test_settings_str()) - .expect("should accept non-placeholder admin password"); - assert_eq!(settings.handlers.len(), 2, "should parse handlers"); - } - - #[test] - fn from_toml_rejects_config_without_admin_handler() { - let result = Settings::from_toml(&settings_str_without_admin_handler()); - assert!( - result.is_err(), - "should reject configuration when admin endpoints are not covered" - ); - let err = format!("{:?}", result.expect_err("should be an error")); - assert!( - err.contains("No handler covers admin endpoint"), - "error should mention uncovered admin endpoints, got: {err}" - ); - } - - /// Verifies that [`Settings::ADMIN_ENDPOINTS`] stays in sync with the - /// admin route table in `crates/trusted-server-adapter-fastly/src/app.rs`. - /// - /// If this test fails, a route was added or removed in the Fastly - /// router without updating `ADMIN_ENDPOINTS` (or vice versa). #[test] fn settings_parses_creative_opportunities_section() { let toml = r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8280,18 +10697,19 @@ formats = [{ width = 300, height = 250 }] #[test] fn settings_rejects_invalid_creative_opportunity_slot_id() { let toml = r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8316,18 +10734,19 @@ formats = [{ width = 300, height = 250 }] // id injected via env is rejected by from_toml_and_env (the build-time // path uses the same validation against the merged config). let toml = r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8358,18 +10777,19 @@ gam_network_id = "21765378893" fn creative_opportunity_settings_toml(slot_body: &str) -> String { format!( r#" -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "unit-test-admin-secret" - [publisher] domain = "example.com" cookie_domain = ".example.com" origin_url = "https://origin.example.com" proxy_secret = "secret" +[geo] +assume_single_jurisdiction = true + [ec] +module = "hmac" + +[ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" [creative_opportunities] @@ -8470,45 +10890,351 @@ formats = [{{ width = 300, height = 250 }}] ); } + /// An unset selector must not serialize as `"module": null`. + /// + /// The section as a whole is skipped while every field is default, so this + /// serializes the struct directly. Once a later change makes another field + /// required, the section is always emitted and a null selector would then + /// reach a config blob, where a binary that predates the field rejects it. + #[test] + fn an_unset_module_selector_is_omitted_from_the_serialized_section() { + let geo = GeoConfig::default(); + let json = serde_json::to_string(&geo).expect("should serialize the geo section"); + assert!( + !json.contains("module"), + "an unset geo selector should be omitted rather than serialized as null, got {json}" + ); + + let device = DeviceConfig::default(); + let json = serde_json::to_string(&device).expect("should serialize the device section"); + assert!( + !json.contains("module"), + "an unset device selector should be omitted rather than serialized as null, got {json}" + ); + } +} + +#[cfg(test)] +mod permission_signal_config_tests { + use super::*; + use serde_json::json; + + use crate::config::TrustedServerAppConfig; + use crate::test_support::tests::crate_test_settings_str; + + // Which names are valid is only known where the scheme crates are linked, + // so the checks that a name matches an available module, and that none + // is repeated, live with the seam in `permission_signal::select`. What is + // tested here is the shape of the section itself. + + /// The test fixture's configuration with `section` written as its + /// `[permission-signal]` section. + fn settings_toml_with(section: &str) -> String { + format!( + "{}\n[permission-signal]\n{section}\n", + crate_test_settings_str() + ) + } + + #[test] + fn no_section_is_allowed_and_means_every_module() { + let config = PermissionSignalConfig::default(); + assert!( + config.modules.is_none(), + "absent rather than empty, because the two mean opposite things" + ); + } + + #[test] + fn the_section_round_trips_through_toml() { + let parsed: PermissionSignalConfig = + toml::from_str(r#"modules = ["gpc", "tcf"]"#).expect("should parse the section"); + assert_eq!( + parsed.modules.as_deref(), + Some(["gpc".to_owned(), "tcf".to_owned()].as_slice()), + "the order written is the order read, because the order is the policy" + ); + } + #[test] - fn admin_endpoints_match_fastly_router() { - let router_source = include_str!("../../trusted-server-adapter-fastly/src/app.rs"); + fn the_section_round_trips_through_a_config_blob() { + // The section is written by derive and read by hand, so what a push + // writes into a blob must be what a deployment reads back from it. + let written = PermissionSignalConfig { + modules: Some(vec!["tcf".to_owned(), "gpc".to_owned()]), + }; + let blob = serde_json::to_value(&written).expect("should write the section"); + let read: PermissionSignalConfig = + serde_json::from_value(blob).expect("should read back what was written"); + assert_eq!(read, written, "the list and its order survive the blob"); + + let null: PermissionSignalConfig = serde_json::from_value(json!({ "modules": null })) + .expect("should read an explicit null"); + assert_eq!( + null, + PermissionSignalConfig::default(), + "an explicit null is the same as leaving the key out" + ); + } + + #[test] + fn an_empty_list_is_kept_apart_from_no_list() { + let parsed: PermissionSignalConfig = + toml::from_str("modules = []").expect("should parse an empty list"); + assert_eq!( + parsed.modules.as_deref(), + Some(&[][..]), + "a publisher acting on no signal at all writes an empty list, and it must \ + not read back as having written nothing" + ); + } + + #[test] + fn an_unknown_key_is_refused() { + let error = toml::from_str::(r#"module_list = ["gpc"]"#) + .expect_err("should refuse a misspelled key rather than silently ignore it"); + assert!( + error + .to_string() + .contains("unknown field `module_list` in [permission-signal], expected `modules`"), + "the refusal names the key it did not recognize and the one it accepts: {error}" + ); + } + + #[test] + fn the_singular_key_is_refused_naming_modules() { + let error = toml::from_str::(r#"module = ["gpc"]"#) + .expect_err("should refuse the singular key the other sections use"); + assert!( + error + .to_string() + .contains("[permission-signal] module is `modules` here"), + "the refusal says which key to write instead: {error}" + ); + } - for endpoint in Settings::ADMIN_ENDPOINTS { + #[test] + fn the_removed_sources_key_is_refused_naming_modules() { + for written in [ + r#"sources = ["gpc", "tcf"]"#, + "modules = [\"gpc\", \"tcf\"]\nsources = [\"gpc\", \"tcf\"]", + ] { + let error = toml::from_str::(written) + .expect_err("should refuse the removed key, alone or beside its replacement"); assert!( - router_source.contains(endpoint), - "ADMIN_ENDPOINTS lists \"{endpoint}\" but it was not found in \ - crates/trusted-server-adapter-fastly/src/app.rs — remove it from ADMIN_ENDPOINTS or \ - add the route back to the router" + error.to_string().contains( + "[permission-signal] sources is no longer accepted. Name the modules \ + to run, in order, in [permission-signal] modules instead" + ), + "the refusal says which key to write instead: {error}" ); } + } - // Also verify we haven't missed any admin routes in the router. - // Best-effort: only detects string-literal routes in the NamedRoute - // table. If you define admin routes differently (e.g. via constants), - // add them to ADMIN_ENDPOINTS manually. - let admin_routes_in_router: Vec<&str> = router_source - .lines() - .filter_map(|line| { - let trimmed = line.trim(); - // Route entries look like: path: "/_ts/admin/...", - if trimmed.starts_with("path: ") && trimmed.contains("\"/_ts/admin/") { - let start = trimmed.find("\"/_ts/admin/")?; - let rest = &trimmed[start + 1..]; - let end = rest.find('"')?; - Some(&rest[..end]) - } else { - None - } - }) - .collect(); + #[test] + fn every_way_settings_are_read_refuses_the_removed_sources_key() { + let written = settings_toml_with(r#"sources = ["gpc", "tcf"]"#); + + let error = Settings::from_toml(&written).expect_err("should refuse the removed key"); + assert!( + format!("{error:?}").contains("[permission-signal] modules"), + "reading a TOML file names the key that replaced it: {error:?}" + ); + + // `ts config push` parses the file into a TOML value before reading the + // settings from it. + let value: toml::Value = toml::from_str(&written).expect("should parse as TOML"); + let error = value + .try_into::() + .expect_err("should refuse the removed key before a push"); + assert!( + error.to_string().contains("[permission-signal] modules"), + "a push names the key that replaced it: {error}" + ); + + // A deployment reads its settings from a JSON config blob. + let settings = Settings::from_toml(&crate_test_settings_str()) + .expect("should load the test settings fixture"); + let mut blob = serde_json::to_value(settings).expect("should serialize the fixture"); + blob["permission-signal"] = json!({ "sources": ["gpc", "tcf"] }); + let error = + Settings::from_json_value(blob).expect_err("should refuse the removed key at startup"); + assert!( + format!("{error:?}").contains("[permission-signal] modules"), + "startup names the key that replaced it: {error:?}" + ); + } + + #[test] + fn the_old_table_name_is_refused_with_its_new_name() { + let written = format!( + "{}\n[permission_signal]\nmodules = [\"gpc\"]\n", + crate_test_settings_str() + ); + let error = Settings::from_toml(&written).expect_err("should refuse the old table name"); + let message = format!("{error:?}"); + assert!( + message.contains("[permission_signal]") && message.contains("[permission-signal]"), + "the refusal names the old table and the new one: {message}" + ); + } - for route in &admin_routes_in_router { + #[test] + fn the_old_ad_server_table_name_is_refused_with_its_new_name() { + let written = format!( + "{}\n[adserver]\nmodule = \"adserver_mock\"\n", + crate_test_settings_str() + ); + let error = Settings::from_toml(&written).expect_err("should refuse the old table name"); + let message = format!("{error:?}"); + assert!( + message.contains("[adserver]") && message.contains("[ad-server] module"), + "the refusal names the old table and the new selector: {message}" + ); + } + + #[test] + fn a_block_of_module_settings_is_refused_as_an_unknown_field() { + // No module takes settings yet, so a block for one is refused rather + // than read and then ignored. + let written = + settings_toml_with("modules = [\"gpc\"]\n\n[permission-signal.gpc]\nenabled = true"); + let error = + Settings::from_toml(&written).expect_err("should refuse settings no module takes"); + let message = format!("{error:?}"); + assert!( + message.contains("unknown field `gpc` in [permission-signal]") + && message.contains("[permission-signal.] block is not accepted"), + "the refusal names the block and says why it is refused: {message}" + ); + } + + mod shared_tables { + use super::*; + + #[derive(Debug, Deserialize, Validate)] + #[serde(deny_unknown_fields)] + struct ExampleShared { + #[validate(url)] + endpoint: String, + } + + fn with(toml: &str) -> Result> { + Settings::from_toml(&format!("{}\n{toml}", crate_test_settings_str())) + } + + #[test] + fn a_vendor_reads_its_own_shared_table_and_nothing_else() { + let settings = with( + r#" + [shared.example] + endpoint = "https://api.example.com/v1" + "#, + ) + .expect("should load"); + + let read: ExampleShared = settings + .shared_config("example") + .expect("should read") + .expect("the table is present"); + assert_eq!(read.endpoint, "https://api.example.com/v1"); + assert!( + settings + .shared_config::("another") + .expect("an absent table reads as nothing") + .is_none() + ); + } + + #[test] + fn a_shared_table_its_vendor_cannot_read_is_refused_naming_the_table() { + let settings = with( + r#" + [shared.example] + endpoint = "https://api.example.com/v1" + unknown = true + "#, + ) + .expect("core reads none of the table, so it loads"); + + let refused = format!( + "{:?}", + settings + .shared_config::("example") + .expect_err("the vendor does not know the key") + ); assert!( - Settings::ADMIN_ENDPOINTS.contains(route), - "Router has admin route \"{route}\" that is missing from \ - Settings::ADMIN_ENDPOINTS — add it to ensure auth coverage" + refused.contains("[shared.example] could not be read") + && refused.contains("unknown"), + "{refused}" + ); + } + + #[test] + fn a_shared_table_failing_its_validation_is_refused_naming_the_table() { + let settings = with( + r#" + [shared.example] + endpoint = "not-a-url" + "#, + ) + .expect("core reads none of the table, so it loads"); + + let refused = format!( + "{:?}", + settings + .shared_config::("example") + .expect_err("the endpoint is not a URL") + ); + assert!( + refused.contains("[shared.example] failed validation"), + "{refused}" + ); + } + + #[test] + fn a_shared_table_not_named_as_a_vendor_is_refused() { + for table in ["[shared.Example]", "[shared.\"an.example\"]"] { + let refused = with(&format!("{table}\nendpoint = \"https://api.example.com\"")) + .expect_err("a vendor's table is named by its folder"); + + assert!( + format!("{refused:?}").contains("which is not a vendor's name"), + "{table}: {refused:?}" + ); + } + } + + /// The tables hold secrets, so a settings dump shows the vendors alone, + /// and the serialized settings carry the tables whole. + #[test] + fn a_shared_table_is_shown_by_its_vendor_and_serialized_whole() { + let settings = with( + r#" + [shared.example] + endpoint = "https://api.example.com/v1" + license_key = "a-secret-value" + "#, + ) + .expect("should load"); + + let shown = format!("{:?}", settings.shared); + assert!(shown.contains("example"), "{shown}"); + assert!(!shown.contains("a-secret-value"), "{shown}"); + + let serialized = serde_json::to_value(&settings).expect("should serialize"); + assert_eq!( + serialized.pointer("/shared/example/endpoint"), + Some(&serde_json::json!("https://api.example.com/v1")) ); } + + #[test] + fn settings_without_a_shared_section_are_written_without_one() { + let settings = with("").expect("should load"); + + let serialized = serde_json::to_value(&settings).expect("should serialize"); + assert!(serialized.get("shared").is_none()); + } } } diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs index 07108fac64..b753f658f7 100644 --- a/crates/trusted-server-core/src/settings_data.rs +++ b/crates/trusted-server-core/src/settings_data.rs @@ -4,8 +4,9 @@ use serde::Deserialize; use sha2::{Digest as _, Sha256}; pub use crate::config_payload::DEFAULT_CONFIG_STORE_ID; -use crate::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob}; +use crate::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob_with}; use crate::error::TrustedServerError; +use crate::integrations::IntegrationBuilder; use crate::platform::{PlatformConfigStore, PlatformSecretStore, StoreName}; use crate::settings::Settings; @@ -68,7 +69,13 @@ pub fn default_secret_store_name() -> StoreName { StoreName::from(EnvConfig::from_env().store_name("secrets", DEFAULT_SECRET_STORE_ID)) } -/// Loads [`Settings`] from a platform config store and key. +/// Loads [`Settings`] from a platform config store and key, validating +/// against the built-in integrations only. +/// +/// A deployment that composes builders of its own calls +/// [`get_settings_from_config_store_with`] instead, because validation runs +/// during the load and refuses a `[demand]` or `[ad-server]` name it cannot +/// see a builder for. /// /// # Errors /// @@ -81,10 +88,41 @@ pub fn get_settings_from_config_store( store_name: &StoreName, key: &str, default_secret_store_name: &StoreName, +) -> Result> { + get_settings_from_config_store_with( + config_store, + secret_store, + store_name, + key, + default_secret_store_name, + &[], + ) +} + +/// Loads [`Settings`] from a platform config store and key, validating +/// against the built-in integrations followed by `extra_integrations`. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the config blob is +/// missing, cannot be read, fails envelope verification, secret resolution, +/// or Trusted Server settings validation. +pub fn get_settings_from_config_store_with( + config_store: &dyn PlatformConfigStore, + secret_store: &dyn PlatformSecretStore, + store_name: &StoreName, + key: &str, + default_secret_store_name: &StoreName, + extra_integrations: &[IntegrationBuilder], ) -> Result> { let raw_value = read_config_entry(config_store, store_name, key)?; let envelope_json = resolve_fastly_chunk_pointer(config_store, store_name, &raw_value)?; - settings_from_config_blob(&envelope_json, secret_store, default_secret_store_name) + settings_from_config_blob_with( + &envelope_json, + secret_store, + default_secret_store_name, + extra_integrations, + ) } fn read_config_entry( diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index d95e6ebf85..e3d9272e6b 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -310,8 +310,8 @@ impl lol_html::OutputSink for RcVecSink { /// Adapter to use `lol_html` [`HtmlRewriter`](lol_html::HtmlRewriter) as a [`StreamProcessor`]. /// /// Output is emitted incrementally on every [`process_chunk`](StreamProcessor::process_chunk) -/// call. Script rewriters that receive text from `lol_html` must be fragment-safe — -/// they accumulate text fragments internally until `is_last_in_text_node` is true. +/// call. A text handler that receives text from `lol_html` must be fragment-safe, +/// so it accumulates text fragments until the last chunk of the text node arrives. /// /// The adapter is single-use: one adapter per request. Calling [`StreamProcessor::reset`] /// is a no-op because the rewriter consumes its settings on construction. @@ -1731,8 +1731,8 @@ mod tests { } /// Verify that `lol_html` fragments text nodes when input chunks split - /// mid-text-node. Script rewriters must be fragment-safe — they accumulate - /// text fragments internally until `is_last_in_text_node` is true. + /// mid-text-node. A text handler must be fragment-safe, so it accumulates + /// text fragments until the last chunk of the text node arrives. #[test] fn lol_html_fragments_text_across_chunk_boundaries() { use std::cell::RefCell; diff --git a/crates/trusted-server-core/src/tdl.rs b/crates/trusted-server-core/src/tdl.rs new file mode 100644 index 0000000000..8adcddb65d --- /dev/null +++ b/crates/trusted-server-core/src/tdl.rs @@ -0,0 +1,191 @@ +//! Terms Document Locators, the labels saying what terms the data offered for +//! a request is available under. +//! +//! A locator is the address of a published document a person can read, stating +//! the basis on which the data at hand may be collected, shared and used. A +//! recipient reads the locators alongside the data, decides whether those terms +//! are ones it accepts, and decides on the same basis whether it may pass the +//! data on. No locator means no terms are declared, which a recipient must not +//! read as permission. +//! +//! Nothing here interprets a document. Core carries the locators a +//! [`PermissionSignalModule`](crate::permission_signal::PermissionSignalModule) +//! declares for the request and makes them visible to what consumes the data, +//! and the module for a terms scheme decides which document applies. Model +//! Terms for Marketing (MTM) is the first such scheme to arrive and one of many +//! rather than the only one, because a publisher, a trade body or a regulator +//! can each publish terms and each set becomes a module. +//! +//! # The document must not change +//! +//! A locator has to point at a document that is never edited once published, +//! which is why a version belongs in its address. A document that can be +//! rewritten tomorrow means a recipient can never prove what it agreed to, and +//! one edit silently rewrites the basis of every transaction already sent under +//! it. That is a property of how the document is published, so no code here can +//! check it, and it is the reason this type refuses nothing but an address that +//! could not be fetched at all. + +use core::str::FromStr; + +use error_stack::Report; +use url::Url; + +use crate::error::TrustedServerError; + +/// The address of a published terms document. +/// +/// Absolute, and `http` or `https`, because a recipient has to be able to +/// fetch and read the document. See the module documentation for why the +/// document itself must be immutable and versioned. +#[derive(Debug, Clone, PartialEq, Eq, Hash, derive_more::Display)] +pub struct Tdl(String); + +impl Tdl { + /// Builds a locator from an address. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when the address is not an + /// absolute `http` or `https` URL with a host, because a recipient given + /// one of those has nothing it can fetch. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::tdl::Tdl; + /// + /// let tdl = Tdl::new("https://terms.example.com/marketing/2.txt") + /// .expect("should accept an absolute https address"); + /// assert_eq!(tdl.as_str(), "https://terms.example.com/marketing/2.txt"); + /// + /// assert!(Tdl::new("/marketing/2.txt").is_err()); + /// ``` + pub fn new(locator: &str) -> Result> { + let parsed = Url::parse(locator).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "Terms document locator `{locator}` is not an absolute URL: {error}" + ), + }) + })?; + if !matches!(parsed.scheme(), "http" | "https") { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Terms document locator `{locator}` must be http or https, so a \ + recipient can read the document, and not `{}`", + parsed.scheme() + ), + })); + } + if parsed.host().is_none() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("Terms document locator `{locator}` names no host"), + })); + } + Ok(Self(locator.to_owned())) + } + + /// The address, as it is carried to a recipient. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for Tdl { + type Err = Report; + + fn from_str(locator: &str) -> Result { + Self::new(locator) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_absolute_https_address_is_accepted() { + let tdl = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + assert_eq!( + tdl.as_str(), + "https://terms.example.com/marketing/2.txt", + "should carry the address unchanged" + ); + } + + #[test] + fn an_absolute_http_address_is_accepted() { + assert!( + Tdl::new("http://terms.example.com/marketing/2.txt").is_ok(), + "should accept http, because a document served over http is still readable" + ); + } + + #[test] + fn a_relative_address_is_refused() { + let error = Tdl::new("/marketing/2.txt") + .expect_err("should refuse an address a recipient cannot resolve"); + assert!( + format!("{error:?}").contains("absolute"), + "should say the address is not absolute" + ); + } + + #[test] + fn another_scheme_is_refused() { + let error = + Tdl::new("mailto:terms@example.com").expect_err("should refuse a scheme with no page"); + assert!( + format!("{error:?}").contains("http or https"), + "should name the schemes a recipient can read" + ); + } + + #[test] + fn an_empty_address_is_refused() { + assert!( + Tdl::new("").is_err(), + "should refuse an empty address rather than carry it to a recipient" + ); + } + + #[test] + fn parsing_from_a_string_gives_the_same_answer() { + let parsed: Tdl = "https://terms.example.com/marketing/2.txt" + .parse() + .expect("should parse an absolute https address"); + assert_eq!( + parsed, + Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"), + "should match the constructor" + ); + } + + #[test] + fn two_locators_for_the_same_document_compare_equal() { + let one = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + let two = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + assert_eq!( + one, two, + "should compare equal so the same document is carried once" + ); + } + + #[test] + fn versions_of_one_document_are_different_locators() { + let two = Tdl::new("https://terms.example.com/marketing/2.txt") + .expect("should accept an absolute https address"); + let three = Tdl::new("https://terms.example.com/marketing/3.txt") + .expect("should accept an absolute https address"); + assert_ne!( + two, three, + "should distinguish versions, because a recipient agreed to one of them" + ); + } +} diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37b8a9e657..9fea391024 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -1,48 +1,80 @@ -#[cfg(test)] +#[cfg(any(test, feature = "test-utils"))] pub mod tests { use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource, TcfConsent}; - use crate::settings::Settings; + use crate::ec::module::{EcModuleSelection, HMAC_MODULE_KEY, HOST_SIGNALS_MODULE_KEY}; + use crate::redacted::Redacted; + use crate::settings::{Ec, EcModuleBlock, HmacModuleConfig, HostSignalsModuleConfig, Settings}; #[must_use] pub fn crate_test_settings_str() -> String { r#" - [[handlers]] - path = "^/secure" - username = "user" - password = "pass" - - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" origin_url = "https://origin.test-publisher.com" proxy_secret = "unit-test-proxy-secret" - [integrations.prebid] - enabled = true - external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" + [geo] + # A gdpr-eu country, where every permission requires a signal. This + # reproduces the prior no-default floor, so existing tests are + # unaffected by the now-required default. + # Tests run with no geo module, so single-jurisdiction operation + # is acknowledged the same way a deployment would. + assume_single_jurisdiction = true - [integrations.prebid.bundle.modules] - bidder = ["exampleBidderBidAdapter"] - - [integrations.nextjs] - enabled = false - rewrite_attributes = ["href", "link", "url"] + [auction] [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + [request_signing] - config_store_id = "test-config-store-id" - secret_store_id = "test-secret-store-id" + enabled = false "# .to_owned() } + /// The shared fixture TOML with the module `name` selected in `section` + /// as well, written with the section's type folder left off, for a test + /// that appends that module's own table. + #[must_use] + pub fn crate_test_settings_str_running(section: &str, name: &str) -> String { + let written = crate::module_name::short_form(section, name); + if section == "auction" { + return crate_test_settings_str().replace( + "[auction]\n", + &format!("[auction]\nmodules = [\"{written}\"]\n"), + ); + } + format!( + "{}\n[{section}]\nmodules = [\"{written}\"]\n", + crate_test_settings_str() + ) + } + + /// The crate test configuration with its whole `[ec]` section replaced by + /// `ec_section`, which carries its own `[ec]` header and any module + /// blocks. + /// + /// # Panics + /// + /// Panics if the embedded TOML configuration no longer has an `[ec]` + /// section followed by a `[request_signing]` section. + #[must_use] + pub fn crate_test_settings_str_with_ec_section(ec_section: &str) -> String { + let base = crate_test_settings_str(); + let (before, rest) = base + .split_once("[ec]") + .expect("should find the [ec] section in the test settings"); + let (_, after) = rest + .split_once("[request_signing]") + .expect("should find the [request_signing] section in the test settings"); + format!("{before}{ec_section}\n\n[request_signing]{after}") + } + #[must_use] /// Creates test settings from embedded TOML configuration. /// @@ -56,10 +88,73 @@ pub mod tests { settings } + /// Selects the built-in HMAC module under `name` with `passphrase`, + /// replacing whatever Edge Cookie module the settings carried. + /// + /// A `name` other than `hmac` is a label, so the block names the + /// implementation it configures. + pub fn select_hmac_module(ec: &mut Ec, name: &str, passphrase: &str) { + let mut block = EcModuleBlock::from(HmacModuleConfig { + passphrase: Redacted::new(passphrase.to_owned()), + }); + if name != HMAC_MODULE_KEY { + block.implementation = Some(HMAC_MODULE_KEY.to_owned()); + } + ec.module = Some(EcModuleSelection::from(name)); + ec.module_blocks.clear(); + ec.module_blocks.insert(name.to_owned(), block); + } + + /// Selects the built-in host-signal module under its own name with + /// `passphrase`, replacing whatever Edge Cookie module the settings + /// carried. + pub fn select_host_signals_module(ec: &mut Ec, passphrase: &str) { + ec.module = Some(EcModuleSelection::from(HOST_SIGNALS_MODULE_KEY)); + ec.module_blocks.clear(); + ec.module_blocks.insert( + HOST_SIGNALS_MODULE_KEY.to_owned(), + EcModuleBlock::from(HostSignalsModuleConfig { + passphrase: Redacted::new(passphrase.to_owned()), + }), + ); + } + + /// The passphrase the block `name` holds. + /// + /// # Panics + /// + /// Panics if `name` has no block, or if its block configures another + /// module. + #[must_use] + #[allow( + clippy::panic, + reason = "a fixture names the block it could not find, which `expect` cannot" + )] + pub fn hmac_passphrase<'a>(ec: &'a Ec, name: &str) -> &'a str { + ec.module_blocks + .get(name) + .and_then(EcModuleBlock::hmac_settings) + .unwrap_or_else(|| panic!("settings should configure the hmac module under `{name}`")) + .passphrase + .expose() + } + /// A valid EC ID in `{64-hex}.{6-alnum}` format for use in tests. pub const VALID_SYNTHETIC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.Ab1234"; + /// The permissions of a reader who allowed storage and withheld + /// personalised-ad selection: the Edge Cookie stands, partner sharing does + /// not. + #[must_use] + pub fn storage_only_permissions() -> crate::permissions::PermissionState { + crate::permissions::PermissionState::new( + [crate::permissions::Permission::StoreOnDevice] + .into_iter() + .collect(), + ) + } + /// GDPR consent granting TCF Purpose 1 (storage/EC) but denying Purpose 4 /// (personalized ads), so EC is allowed while EID persistence is not. #[must_use] @@ -94,6 +189,71 @@ pub mod tests { } } +/// Page fixtures shared by core's tests and the tests of a module crate. +#[cfg(any(test, feature = "test-utils"))] +pub mod fixtures { + /// A publisher page carrying the tags of several vendors, as an origin + /// would serve it. + pub const PUBLISHER_PAGE_HTML: &str = include_str!("html_processor.test.html"); +} + +/// The operator-facing settings template, for tests that check what it +/// documents. +#[cfg(any(test, feature = "test-utils"))] +pub mod template { + /// The source-controlled template an operator starts from. + pub const EXAMPLE_TEMPLATE: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + /// The template with its required secret-store key references replaced by + /// resolved test values, so parsing it as settings can exercise the + /// optional blocks a test uncomments. + #[must_use] + pub fn template_with_resolved_required_secrets() -> String { + EXAMPLE_TEMPLATE + .replace( + "proxy_secret = \"publisher_proxy_secret\"", + "proxy_secret = \"unit-test-resolved-publisher-proxy-secret-0001\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"unit-test-resolved-ec-passphrase-secret-0001\"", + ) + } + + /// Uncomments the contiguous `#`-prefixed block that begins at the line + /// `# {header}`, leaving the rest of the template untouched. Stops at the + /// first line that is not a comment, so a blank line ends the block. + #[must_use] + pub fn uncomment_block(template: &str, header: &str) -> String { + let header_line = format!("# {header}"); + let mut out = Vec::new(); + let mut uncommenting = false; + + for line in template.lines() { + if line == header_line { + uncommenting = true; + } else if uncommenting && !line.trim_start().starts_with('#') { + uncommenting = false; + } + + if uncommenting { + let bare = line + .strip_prefix("# ") + .or_else(|| line.strip_prefix('#')) + .unwrap_or(line); + out.push(bare.to_owned()); + } else { + out.push(line.to_owned()); + } + } + + out.join("\n") + } +} + /// Shared Next.js + auction origin fixture. /// /// Adapters exercise the buffered publisher path against this fixture in their @@ -132,11 +292,6 @@ pub mod nextjs_auction { pub fn settings() -> Settings { let mut settings = Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -145,30 +300,53 @@ pub mod nextjs_auction { [ec] passphrase = "test-secret-key-32-bytes-minimum" + + # The fixture supplies its own geo module through the runtime + # services, so it selects the host lookup to reach it. Without a + # selection the jurisdiction is unknown, and a server-side auction + # fails closed on an unknown jurisdiction. + [geo] + module = "platform" + + # What Next.js changes in a page is a middleware, which runs on + # the pages an entry names it for. + [[fetch]] + media_type = "text/html" + middleware = ["framework.nextjs"] "#, ) .expect("should parse Next.js auction fixture settings"); settings - .integrations - .insert_config( - "nextjs", + .insert_module_config( + "framework", + "framework.nextjs", &serde_json::json!({ - "enabled": true, "rewrite_attributes": ["href", "link", "url"], }), ) - .expect("should enable the fixture Next.js integration"); + .expect("should select the fixture Next.js integration"); settings.auction.enabled = true; - settings.auction.mediator = None; - settings.auction.providers = serde_json::from_value(serde_json::json!({ - "fixture": { - "protocol": "openrtb-2.6", - "endpoint": "https://auction.example.com/bid", - "routing": "all_eligible", - "timeout_ms": 5000 - } - })) - .expect("should configure the fixture auction provider"); + // One `[demand.fixture]` source, which is where an auction provider is + // configured. `implementation` states the wire format, so there is no + // separate `protocol`. + settings.demand = crate::provider_table::ProviderTable::new( + vec!["fixture".to_owned()], + std::collections::BTreeMap::from([( + "fixture".to_owned(), + serde_json::Map::from_iter([ + ( + "implementation".to_owned(), + serde_json::json!("auction-protocol.openrtb"), + ), + ( + "endpoint".to_owned(), + serde_json::json!("https://auction.example.com/bid"), + ), + ("routing".to_owned(), serde_json::json!("all_eligible")), + ("timeout_ms".to_owned(), serde_json::json!(5000)), + ]), + )]), + ); settings.creative_opportunities = Some( toml::from_str( r#" @@ -312,10 +490,12 @@ pub mod nextjs_auction { struct FixtureGeo; + #[async_trait::async_trait(?Send)] impl PlatformGeo for FixtureGeo { - fn lookup( + async fn lookup( &self, _client_ip: Option, + _services: &RuntimeServices, ) -> Result, Report> { Ok(Some(GeoInfo { country: "AU".to_owned(), diff --git a/crates/trusted-server-core/src/tester_cookie.rs b/crates/trusted-server-core/src/tester_cookie.rs index 0eeda9fcb3..8dc59647ab 100644 --- a/crates/trusted-server-core/src/tester_cookie.rs +++ b/crates/trusted-server-core/src/tester_cookie.rs @@ -1,7 +1,8 @@ //! Tester-cookie endpoint helpers. //! //! The tester routes are intentionally disabled unless configured. When enabled, -//! they set or clear a first-party `ts-tester` cookie scoped to the configured +//! they set or clear a first-party `-tester` cookie, such as +//! `ts-tester`, scoped to the configured //! publisher cookie domain. They are self-service routes rather than admin //! routes; the cookie only affects tester routing and must not gate sensitive //! behavior. @@ -10,31 +11,36 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{HeaderValue, Response, StatusCode, header}; -use crate::constants::COOKIE_TS_TESTER; +use crate::constants::COOKIE_TESTER; use crate::error::TrustedServerError; +use crate::prefix::PublicPrefix; use crate::settings::Settings; /// Formats the tester cookie `Set-Cookie` header value. -fn format_tester_cookie(domain: &str, max_age_seconds: u32) -> String { +fn format_tester_cookie(public: &PublicPrefix, domain: &str, max_age_seconds: u32) -> String { format!( "{}=true; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age={}", - COOKIE_TS_TESTER, domain, max_age_seconds, + public.cookie_name(COOKIE_TESTER), + domain, + max_age_seconds, ) } /// Formats the tester cookie clearing `Set-Cookie` header value. -fn format_clear_tester_cookie(domain: &str) -> String { +fn format_clear_tester_cookie(public: &PublicPrefix, domain: &str) -> String { format!( "{}=; Domain={}; Path=/; Secure; SameSite=Lax; Max-Age=0", - COOKIE_TS_TESTER, domain, + public.cookie_name(COOKIE_TESTER), + domain, ) } /// Handles `GET /_ts/set-tester`. /// /// Returns `404 Not Found` while `[tester_cookie].enabled` is false. When the -/// feature is enabled, returns `204 No Content` with `Set-Cookie: ts-tester=true` -/// scoped to `publisher.cookie_domain` and persisted for +/// feature is enabled, returns `204 No Content` with `Set-Cookie` writing +/// `-tester=true`, such as `ts-tester=true`, scoped to +/// `publisher.cookie_domain` and persisted for /// `[tester_cookie].max_age_seconds` (30 days by default). /// /// # Errors @@ -43,6 +49,7 @@ fn format_clear_tester_cookie(domain: &str) -> String { /// domain cannot be rendered as an HTTP header value. pub fn handle_set_tester( settings: &Settings, + public: &PublicPrefix, ) -> Result, Report> { if !settings.tester_cookie.enabled { let mut response = Response::new(EdgeBody::empty()); @@ -51,6 +58,7 @@ pub fn handle_set_tester( } let set_cookie = HeaderValue::from_str(&format_tester_cookie( + public, &settings.publisher.cookie_domain, settings.tester_cookie.max_age_seconds, )) @@ -74,7 +82,7 @@ pub fn handle_set_tester( /// /// Returns `404 Not Found` while `[tester_cookie].enabled` is false. When the /// feature is enabled, returns `204 No Content` with `Set-Cookie` expiring the -/// `ts-tester` cookie scoped to `publisher.cookie_domain`. +/// tester cookie scoped to `publisher.cookie_domain`. /// /// # Errors /// @@ -82,6 +90,7 @@ pub fn handle_set_tester( /// domain cannot be rendered as an HTTP header value. pub fn handle_clear_tester( settings: &Settings, + public: &PublicPrefix, ) -> Result, Report> { if !settings.tester_cookie.enabled { let mut response = Response::new(EdgeBody::empty()); @@ -90,6 +99,7 @@ pub fn handle_clear_tester( } let set_cookie = HeaderValue::from_str(&format_clear_tester_cookie( + public, &settings.publisher.cookie_domain, )) .change_context(TrustedServerError::InvalidHeaderValue { @@ -119,7 +129,8 @@ mod tests { settings.tester_cookie.enabled = true; settings.publisher.cookie_domain = ".tester.example".to_string(); - let response = handle_set_tester(&settings).expect("should build tester response"); + let response = handle_set_tester(&settings, &PublicPrefix::default()) + .expect("should build tester response"); assert_eq!( response.status(), @@ -154,7 +165,8 @@ mod tests { settings.tester_cookie.max_age_seconds = 3_600; settings.publisher.cookie_domain = ".tester.example".to_string(); - let response = handle_set_tester(&settings).expect("should build tester response"); + let response = handle_set_tester(&settings, &PublicPrefix::default()) + .expect("should build tester response"); let set_cookie = response .headers() @@ -169,11 +181,32 @@ mod tests { ); } + #[test] + fn tester_cookie_is_named_after_the_public_prefix() { + let mut settings = create_test_settings(); + settings.tester_cookie.enabled = true; + let public = PublicPrefix::new(["_ex"]).expect("should accept the prefix"); + + for response in [ + handle_set_tester(&settings, &public).expect("should set the tester cookie"), + handle_clear_tester(&settings, &public).expect("should clear the tester cookie"), + ] { + let set_cookie = response + .headers() + .get(header::SET_COOKIE) + .expect("should write the tester cookie") + .to_str() + .expect("should render set-cookie as utf-8"); + assert!(set_cookie.starts_with("_ex-tester="), "{set_cookie}"); + } + } + #[test] fn tester_cookie_route_is_disabled_by_default() { let settings = create_test_settings(); - let response = handle_set_tester(&settings).expect("should build disabled tester response"); + let response = handle_set_tester(&settings, &PublicPrefix::default()) + .expect("should build disabled tester response"); assert_eq!( response.status(), @@ -192,7 +225,8 @@ mod tests { settings.tester_cookie.enabled = true; settings.publisher.cookie_domain = ".tester.example".to_string(); - let response = handle_clear_tester(&settings).expect("should build clear tester response"); + let response = handle_clear_tester(&settings, &PublicPrefix::default()) + .expect("should build clear tester response"); assert_eq!( response.status(), @@ -224,8 +258,8 @@ mod tests { fn clear_tester_cookie_route_is_disabled_by_default() { let settings = create_test_settings(); - let response = - handle_clear_tester(&settings).expect("should build disabled clear tester response"); + let response = handle_clear_tester(&settings, &PublicPrefix::default()) + .expect("should build disabled clear tester response"); assert_eq!( response.status(), diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index e8275b8144..0ad3b765f2 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -1,24 +1,30 @@ -use trusted_server_js::{concatenated_hash, single_module_hash}; +//! URLs and `", - tsjs_script_src(module_ids), + tsjs_script_src(parts), ) } -/// `/static` URL for the unified bundle when exact module IDs are unavailable. +/// `/static` URL for the unified bundle when the exact parts are unavailable. /// /// This intentionally omits `?v=` because the serving path can only mark a URL /// immutable when the hash matches the exact enabled module set. Use -/// [`tsjs_script_src`] with exact module IDs when [`IntegrationRegistry`] is -/// available. +/// [`tsjs_script_src`] with the registry's parts when [`IntegrationRegistry`] +/// is available. /// /// [`IntegrationRegistry`]: crate::integrations::IntegrationRegistry #[must_use] @@ -59,7 +65,7 @@ pub fn tsjs_unified_script_src() -> String { "/static/tsjs=tsjs-unified.min.js".to_string() } -/// `", - tsjs_deferred_script_src(module_id) + tsjs_deferred_script_src(part) ) } -/// Generate all deferred `"), "should generate exactly one trusted server script tag" ); @@ -204,18 +251,18 @@ mod tests { #[test] fn publisher_tsjs_script_tag_renders_static_attributes() { - let module_ids = ["gpt"]; - let src = tsjs_script_src(&module_ids); + let parts = compile_time_parts(&["gpt"]); + let src = tsjs_script_src(&parts); assert_eq!( - tsjs_script_tag_with_attributes(&module_ids, &[("data-ts-gam-attribution", "true")]), + tsjs_script_tag_with_attributes(&parts, &[("data-ts-gam-attribution", "true")]), format!( "" ), "should render trusted static attributes on the publisher bundle tag" ); assert_eq!( - tsjs_script_tag(&module_ids), + tsjs_script_tag(&parts), format!(""), "should keep the generic tag byte-for-byte unmarked" ); @@ -226,7 +273,10 @@ mod tests { expected = "attribute name should contain only lowercase ASCII letters, digits, and hyphens" )] fn publisher_tsjs_script_tag_rejects_invalid_attribute_name() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-bad_name", "true")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-bad_name", "true")], + ); } #[test] @@ -234,31 +284,43 @@ mod tests { expected = "attribute name should contain only lowercase ASCII letters, digits, and hyphens" )] fn publisher_tsjs_script_tag_rejects_empty_attribute_name() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("", "true")]); + let _ = tsjs_script_tag_with_attributes(&compile_time_parts(&["gpt"]), &[("", "true")]); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_double_quote_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "bad\"value")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad\"value")], + ); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_ampersand_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "bad&value")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad&value")], + ); } #[test] #[should_panic(expected = "attribute value should not contain HTML-sensitive characters")] fn publisher_tsjs_script_tag_rejects_less_than_in_attribute_value() { - let _ = tsjs_script_tag_with_attributes(&["gpt"], &[("data-safe-name", "badvalue")]); + let _ = tsjs_script_tag_with_attributes( + &compile_time_parts(&["gpt"]), + &[("data-safe-name", "bad>value")], + ); } #[test] @@ -278,36 +340,58 @@ mod tests { #[test] fn tsjs_single_module_script_src_formats_known_module_url_with_hash() { - let src = tsjs_single_module_script_src("creative"); + let parts = compile_time_parts(&["creative"]); + let src = tsjs_single_module_script_src(&parts[0]); assert!( src.starts_with("/static/tsjs=tsjs-creative.min.js?v="), "should use per-module static bundle path" ); assert_sha256_hex_hash(hash_query_value(&src)); + assert_eq!( + src, + format!( + "/static/tsjs=tsjs-creative.min.js?v={}", + trusted_server_js::single_module_hash("creative") + .expect("should have compiled creative in") + ), + "should keep today's URL for a compile-time module" + ); } #[test] - fn tsjs_deferred_script_src_hashes_prebid_shim_and_empties_unknown_module() { - let prebid_src = tsjs_deferred_script_src("prebid"); + fn tsjs_deferred_script_src_hashes_prebid_shim() { + let parts = compile_time_parts(&["prebid"]); + let prebid_src = tsjs_deferred_script_src(&parts[0]); + assert!( prebid_src.starts_with("/static/tsjs=tsjs-prebid.min.js?v="), "prebid shim should be served from the deferred tsjs route" ); assert_sha256_hex_hash(hash_query_value(&prebid_src)); + } + + #[test] + fn tsjs_deferred_script_tag_carries_a_carried_parts_own_hash() { + let part = carried_part("probe", "(() => { window.probe = 1; })()"); + assert_eq!( - tsjs_deferred_script_src("unknown-module"), - "/static/tsjs=tsjs-unknown-module.min.js?v=", - "should document current unknown-module hash behavior" + tsjs_deferred_script_tag(&part), + format!( + "", + part.sha256 + ), + "should version a carried deferred module by its own content hash" ); } #[test] fn tsjs_deferred_script_tag_marks_script_defer() { - let src = tsjs_deferred_script_src("prebid"); + let parts = compile_time_parts(&["prebid"]); + let src = tsjs_deferred_script_src(&parts[0]); assert_eq!( - tsjs_deferred_script_tag("prebid"), + tsjs_deferred_script_tag(&parts[0]), format!(""), "should generate a deferred script tag" ); @@ -324,12 +408,14 @@ mod tests { #[test] fn tsjs_deferred_script_tags_preserves_input_order() { + let parts = compile_time_parts(&["prebid", "creative"]); + assert_eq!( - tsjs_deferred_script_tags(&["prebid", "creative"]), + tsjs_deferred_script_tags(&parts), format!( "{}{}", - tsjs_deferred_script_tag("prebid"), - tsjs_deferred_script_tag("creative") + tsjs_deferred_script_tag(&parts[0]), + tsjs_deferred_script_tag(&parts[1]) ), "should preserve caller-provided deferred module order" ); @@ -353,18 +439,9 @@ mod tests { #[test] fn tsjs_script_src_differs_for_different_module_sets() { assert_ne!( - tsjs_script_src(&["lockr"]), - tsjs_script_src(&["lockr", "permutive"]), + tsjs_script_src(&compile_time_parts(&["lockr"])), + tsjs_script_src(&compile_time_parts(&["lockr", "permutive"])), "should bust the cache when the module set content changes" ); } - - #[test] - fn tsjs_deferred_script_src_has_empty_hash_for_unknown_module() { - assert_eq!( - tsjs_deferred_script_src("does-not-exist"), - "/static/tsjs=tsjs-does-not-exist.min.js?v=", - "should fall back to an empty cache-busting hash for an unknown module" - ); - } } diff --git a/crates/trusted-server-core/src/tsjs_bundle.rs b/crates/trusted-server-core/src/tsjs_bundle.rs new file mode 100644 index 0000000000..4e98e491c8 --- /dev/null +++ b/crates/trusted-server-core/src/tsjs_bundle.rs @@ -0,0 +1,413 @@ +//! Composition of the served tsjs script from module parts. +//! +//! `trusted-server-js` knows only the modules compiled into it. A module a +//! vendor crate carries on its registration is not in that map, so the +//! composition of the served script and its cache-busting hash live here, +//! keyed on content rather than on ids. The byte rule is unchanged from +//! `trusted_server_js::concatenate_modules`: core first, then each part in +//! order, joined by `;\n`, so every existing `?v=` hash is preserved. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +use sha2::{Digest as _, Sha256}; + +/// One module of the served script. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JsModulePart { + /// Module id, for example `core` or `lockr`. + pub id: &'static str, + /// The built IIFE. + pub source: &'static str, + /// SHA-256 of `source`, hex encoded. Identifies the content in the memo. + /// + /// The memo in [`compose_hash`] trusts this value rather than hashing + /// `source` again, so a part that declares the wrong hash serves a stale + /// `?v=` under a valid-looking URL. Debug builds and tests check the + /// value against `source`; release builds do not. + pub sha256: &'static str, +} + +impl JsModulePart { + /// Looks up a compile-time module of `trusted-server-js` by id. + /// + /// Returns `None` when no module with that id was compiled in. + /// + /// # Examples + /// + /// ``` + /// use trusted_server_core::tsjs_bundle::JsModulePart; + /// + /// assert!(JsModulePart::compile_time("core").is_some()); + /// assert!(JsModulePart::compile_time("not-a-module").is_none()); + /// ``` + #[must_use] + pub fn compile_time(id: &'static str) -> Option { + let source = trusted_server_js::module_bundle(id)?; + let sha256 = trusted_server_js::single_module_hash(id)?; + Some(Self { id, source, sha256 }) + } +} + +/// Resolves compile-time parts for a list of ids, dropping unknown ids, as +/// `trusted_server_js` does today. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::tsjs_bundle::compile_time_parts; +/// +/// let parts = compile_time_parts(&["core", "not-a-module"]); +/// +/// assert_eq!(parts.len(), 1); +/// assert_eq!(parts[0].id, "core"); +/// ``` +#[must_use] +pub fn compile_time_parts(ids: &[&'static str]) -> Vec { + ids.iter() + .filter_map(|id| JsModulePart::compile_time(id)) + .collect() +} + +/// Concatenates the parts into the served script. +/// +/// Core comes first (the given `core` part, or the compile-time core when +/// none is given), then the remaining parts in input order. Each id appears +/// once, keeping its first occurrence, and parts are joined by `;\n` so one +/// IIFE cannot run into the next. +/// +/// # Examples +/// +/// ``` +/// use sha2::{Digest as _, Sha256}; +/// use trusted_server_core::tsjs_bundle::{JsModulePart, compose}; +/// +/// fn part(id: &'static str, source: &'static str) -> JsModulePart { +/// let sha256 = Box::leak(hex::encode(Sha256::digest(source)).into_boxed_str()); +/// JsModulePart { id, source, sha256 } +/// } +/// +/// let parts = [ +/// part("example", "(() => { window.example = true; })()"), +/// part("core", "(() => { window.tsjs = {}; })()"), +/// ]; +/// +/// assert_eq!( +/// compose(&parts), +/// "(() => { window.tsjs = {}; })();\n(() => { window.example = true; })()" +/// ); +/// ``` +#[must_use] +pub fn compose(parts: &[JsModulePart]) -> String { + let ordered = ordered(parts); + // Every piece the visit yields has a known length before the walk, so + // reserve the exact byte count once rather than letting the pushes grow + // the buffer. A bundle of a dozen parts is hundreds of kilobytes, and + // growing to that size copies roughly twice the bundle on every request + // that serves it. + let mut size = 0; + visit_parts(&ordered, |part| size += part.len()); + let mut body = String::with_capacity(size); + visit_parts(&ordered, |part| body.push_str(part)); + body +} + +/// SHA-256 of [`compose`]'s output, hex encoded, without materializing it. +/// +/// The result is memoized per ordered set of `(id, sha256)` pairs. Because the +/// key carries each part's content hash, a carried module that keeps its id +/// but changes its source gets a new hash. The memo never evicts, so feed it +/// only sets derived from configuration, never sets derived from request +/// input. +/// +/// The memo can only hit where the process outlives the request, which of the +/// four adapters means the Axum dev server alone, because its `main` builds +/// the router once before serving. Fastly starts a fresh Wasm instance per +/// request, and `edgezero_adapter_cloudflare::run_app` and +/// `edgezero_adapter_spin::run_app` both call `build_app` inside the +/// per-request entry point, so on those three every call is a miss. A miss +/// costs a key vector, two mutex locks and a stored copy of the hash, all +/// beside a SHA-256 over the whole bundle, so the memo is close to free where +/// it cannot hit and removes the hash entirely where it can. +/// +/// # Panics +/// +/// In debug builds, panics when a part's `sha256` is not the SHA-256 of its +/// `source`. Release builds trust the declared hash. +/// +/// # Examples +/// +/// ``` +/// use sha2::{Digest as _, Sha256}; +/// use trusted_server_core::tsjs_bundle::{JsModulePart, compose_hash}; +/// +/// fn part(id: &'static str, source: &'static str) -> JsModulePart { +/// let sha256 = Box::leak(hex::encode(Sha256::digest(source)).into_boxed_str()); +/// JsModulePart { id, source, sha256 } +/// } +/// +/// let core = JsModulePart::compile_time("core").expect("should have compiled core in"); +/// let before = [core, part("example", "(() => { window.example = 1; })()")]; +/// let after = [core, part("example", "(() => { window.example = 2; })()")]; +/// +/// assert_eq!(compose_hash(&before).len(), 64); +/// assert_eq!(compose_hash(&before), compose_hash(&before)); +/// assert_ne!(compose_hash(&before), compose_hash(&after)); +/// ``` +#[must_use] +pub fn compose_hash(parts: &[JsModulePart]) -> String { + for part in parts { + debug_assert_eq!( + part.sha256, + hex::encode(Sha256::digest(part.source)), + "should declare the SHA-256 of its source for part `{}`", + part.id + ); + } + + let ordered = ordered(parts); + let key = ordered + .iter() + .map(|part| (part.id, part.sha256)) + .collect::>(); + if let Some(hash) = lock_cache().get(&key).cloned() { + return hash; + } + + let mut hasher = Sha256::new(); + visit_parts(&ordered, |part| hasher.update(part.as_bytes())); + let hash = hex::encode(hasher.finalize()); + lock_cache().insert(key, hash.clone()); + hash +} + +/// Orders the parts for the served script: core first, then every non-core +/// part in input order, keeping the first occurrence of each id. +fn ordered(parts: &[JsModulePart]) -> Vec { + let mut result = Vec::with_capacity(parts.len() + 1); + + let core = parts + .iter() + .find(|part| part.id == "core") + .copied() + .or_else(|| JsModulePart::compile_time("core")); + if let Some(core) = core { + result.push(core); + } + + for part in parts { + if part.id == "core" { + continue; + } + if result.iter().any(|taken| taken.id == part.id) { + continue; + } + result.push(*part); + } + + result +} + +/// Visits the byte pieces of the served script in order, with `;\n` between +/// consecutive parts. +fn visit_parts(parts: &[JsModulePart], mut visit: F) { + let mut first = true; + for part in parts { + if first { + first = false; + } else { + visit(";\n"); + } + visit(part.source); + } +} + +type HashCache = HashMap, String>; + +fn lock_cache() -> MutexGuard<'static, HashCache> { + static CACHE: OnceLock> = OnceLock::new(); + match CACHE.get_or_init(|| Mutex::new(HashMap::new())).lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) + } + + /// Builds a part whose `sha256` really is the hash of `source`, so no two + /// parts with different content ever share a memo key. + fn part(id: &'static str, source: &'static str) -> JsModulePart { + let sha256 = Box::leak(sha256_hex(source.as_bytes()).into_boxed_str()); + JsModulePart { id, source, sha256 } + } + + #[test] + fn compose_puts_core_first_and_joins_with_a_semicolon_and_newline() { + let parts = [ + part("lockr", "L"), + part("core", "C"), + part("permutive", "P"), + ]; + + assert_eq!( + compose(&parts), + "C;\nL;\nP", + "should order core first and join parts" + ); + } + + #[test] + fn compose_hash_matches_the_compile_time_hash_for_built_in_modules() { + let ids = ["lockr", "permutive"]; + let parts = compile_time_parts(&ids); + + assert_eq!( + compose_hash(&parts), + trusted_server_js::concatenated_hash(&ids), + "should reproduce today's hash for a built-in module set" + ); + assert_eq!( + compose(&parts), + trusted_server_js::concatenate_modules(&ids), + "should reproduce today's bytes for a built-in module set" + ); + } + + #[test] + fn compose_of_no_parts_is_the_compile_time_core_alone() { + assert_eq!( + compose(&[]), + trusted_server_js::concatenate_modules(&[]), + "should serve core alone when no parts are given" + ); + assert_eq!( + compose_hash(&[]), + trusted_server_js::concatenated_hash(&[]), + "should hash core alone when no parts are given" + ); + } + + #[test] + fn compose_matches_every_compile_time_module_set_the_old_api_produces() { + let all = trusted_server_js::all_module_ids(); + let non_core = all + .iter() + .copied() + .filter(|id| *id != "core") + .collect::>(); + let mut cases = vec![all.to_vec(), non_core.clone()]; + cases.push(non_core.iter().rev().copied().collect()); + cases.push(vec!["core"]); + + for ids in cases { + let parts = compile_time_parts(&ids); + assert_eq!( + compose(&parts), + trusted_server_js::concatenate_modules(&ids), + "should reproduce today's bytes for {ids:?}" + ); + assert_eq!( + compose_hash(&parts), + trusted_server_js::concatenated_hash(&ids), + "should reproduce today's hash for {ids:?}" + ); + } + } + + #[test] + fn compose_hash_changes_when_a_carried_module_changes() { + let before = [part("core", "C"), part("probe", "A")]; + let after = [part("core", "C"), part("probe", "B")]; + + assert_ne!( + compose_hash(&before), + compose_hash(&after), + "should hash carried content" + ); + } + + #[test] + fn compose_keeps_a_duplicate_id_once_using_its_first_occurrence() { + let parts = [ + part("core", "C"), + part("lockr", "L1"), + part("permutive", "P"), + part("lockr", "L2"), + part("core", "C2"), + ]; + + assert_eq!( + compose(&parts), + "C;\nL1;\nP", + "should keep the first occurrence of each id and drop later ones" + ); + } + + #[test] + fn compose_reserves_the_exact_bundle_size_before_writing_it() { + let parts = compile_time_parts(trusted_server_js::all_module_ids()); + let body = compose(&parts); + + assert_eq!( + body.capacity(), + body.len(), + "should allocate the bundle once at its exact size" + ); + } + + #[test] + fn compose_hash_is_the_hex_sha256_of_compose() { + let parts = [ + part("core", "(() => {})()"), + part("carried", "(() => { window.carried = true; })()"), + part("lockr", "L"), + ]; + + assert_eq!( + compose_hash(&parts), + sha256_hex(compose(&parts).as_bytes()), + "should hash the exact bytes compose produces" + ); + } + + #[test] + #[should_panic(expected = "should declare the SHA-256 of its source for part `lying`")] + fn compose_hash_rejects_a_part_whose_declared_hash_is_wrong() { + let parts = [ + part("core", "C"), + JsModulePart { + id: "lying", + source: "A", + sha256: "not-the-hash-of-a", + }, + ]; + + let _ = compose_hash(&parts); + } + + #[test] + fn compile_time_parts_drops_unknown_ids() { + let parts = compile_time_parts(&["lockr", "not-a-module", "permutive"]); + let ids = parts.iter().map(|part| part.id).collect::>(); + + assert_eq!( + ids, + ["lockr", "permutive"], + "should keep known ids in order and drop unknown ids" + ); + for part in parts { + assert_eq!( + part.sha256, + sha256_hex(part.source.as_bytes()), + "should carry the compile-time hash of {}", + part.id + ); + } + } +} diff --git a/crates/trusted-server-core/src/version_header.rs b/crates/trusted-server-core/src/version_header.rs deleted file mode 100644 index 822e04b26c..0000000000 --- a/crates/trusted-server-core/src/version_header.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! `x-ts-version`: the deployed git version compiled in by `build.rs`. - -use edgezero_core::http::{HeaderValue, Response}; - -use crate::constants::{HEADER_X_TS_VERSION, TS_GIT_VERSION}; - -/// Returns the compiled-in git version as a header value. -/// -/// `None` when no version is known or the value is not a valid header value. -#[must_use] -pub fn git_version_header_value() -> Option { - header_value_from(TS_GIT_VERSION) -} - -/// Converts `version` to a header value, logging and returning `None` when it -/// is not a valid header value. -#[must_use] -fn header_value_from(version: Option<&str>) -> Option { - let version = version?; - match HeaderValue::from_str(version) { - Ok(value) => Some(value), - Err(_) => { - log::warn!("Skipping invalid TRUSTED_SERVER__GIT_VERSION response header value"); - None - } - } -} - -/// Sets `x-ts-version` to the compiled-in git version, if one is known. -pub fn apply_git_version_header(response: &mut Response) { - apply_git_version_header_from(TS_GIT_VERSION, response); -} - -/// Sets `x-ts-version` to `version`, or removes it when `version` is unknown -/// or invalid, so an origin's `x-ts-version` is never reported as ours. -fn apply_git_version_header_from(version: Option<&str>, response: &mut Response) { - if let Some(value) = header_value_from(version) { - response.headers_mut().insert(HEADER_X_TS_VERSION, value); - } else { - response.headers_mut().remove(HEADER_X_TS_VERSION); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use edgezero_core::body::Body; - use edgezero_core::http::response_builder; - - fn empty_response() -> Response { - response_builder() - .body(Body::empty()) - .expect("should build empty test response") - } - - fn response_with_upstream_version() -> Response { - response_builder() - .header(HEADER_X_TS_VERSION, "upstream-v1") - .body(Body::empty()) - .expect("should build test response with an upstream x-ts-version") - } - - fn version_of(response: &Response) -> Option<&str> { - response - .headers() - .get(HEADER_X_TS_VERSION) - .and_then(|v| v.to_str().ok()) - } - - #[test] - fn header_value_from_valid_version() { - assert_eq!( - header_value_from(Some("v1.2.3")), - Some(HeaderValue::from_static("v1.2.3")), - "should convert a valid version" - ); - } - - #[test] - fn header_value_from_unknown_or_invalid_version() { - assert_eq!(header_value_from(None), None, "should be None when unknown"); - assert_eq!( - header_value_from(Some("bad\nvalue")), - None, - "should be None for a non-header-safe version" - ); - } - - #[test] - fn git_version_header_value_matches_compiled_in_version() { - assert_eq!( - git_version_header_value() - .as_ref() - .and_then(|v| v.to_str().ok()), - TS_GIT_VERSION, - "should convert the compiled-in TS_GIT_VERSION" - ); - } - - #[test] - fn sets_header_from_version() { - let mut response = empty_response(); - apply_git_version_header_from(Some("v1.2.3"), &mut response); - assert_eq!( - version_of(&response), - Some("v1.2.3"), - "should set x-ts-version" - ); - } - - #[test] - fn omits_header_when_version_unknown() { - let mut response = empty_response(); - apply_git_version_header_from(None, &mut response); - assert_eq!( - version_of(&response), - None, - "should omit x-ts-version when unknown" - ); - } - - #[test] - fn skips_invalid_header_value() { - let mut response = empty_response(); - apply_git_version_header_from(Some("bad\nvalue"), &mut response); - assert_eq!( - version_of(&response), - None, - "should skip a non-header-safe version" - ); - } - - #[test] - fn replaces_upstream_header_with_version() { - let mut response = response_with_upstream_version(); - apply_git_version_header_from(Some("v2.0.0"), &mut response); - assert_eq!( - version_of(&response), - Some("v2.0.0"), - "should replace an upstream x-ts-version with ours" - ); - } - - #[test] - fn removes_upstream_header_when_version_unknown() { - let mut response = response_with_upstream_version(); - apply_git_version_header_from(None, &mut response); - assert_eq!( - version_of(&response), - None, - "should remove an upstream x-ts-version when ours is unknown" - ); - } - - #[test] - fn removes_upstream_header_when_version_invalid() { - let mut response = response_with_upstream_version(); - apply_git_version_header_from(Some("bad\nvalue"), &mut response); - assert_eq!( - version_of(&response), - None, - "should remove an upstream x-ts-version when ours is invalid" - ); - } - - #[test] - fn default_uses_compiled_in_version() { - let mut response = empty_response(); - apply_git_version_header(&mut response); - assert_eq!( - version_of(&response), - TS_GIT_VERSION, - "should report the compiled-in TS_GIT_VERSION" - ); - } -} diff --git a/crates/trusted-server-integration-tests/Cargo.toml b/crates/trusted-server-integration-tests/Cargo.toml index 39f985860a..c83377fca4 100644 --- a/crates/trusted-server-integration-tests/Cargo.toml +++ b/crates/trusted-server-integration-tests/Cargo.toml @@ -25,6 +25,7 @@ edgezero-core = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } +trusted-server-modules = { workspace = true } [dev-dependencies] async-trait = { workspace = true } diff --git a/crates/trusted-server-integration-tests/README.md b/crates/trusted-server-integration-tests/README.md index 2d3981721e..8ca3655a32 100644 --- a/crates/trusted-server-integration-tests/README.md +++ b/crates/trusted-server-integration-tests/README.md @@ -7,7 +7,7 @@ end-to-end publisher behavior. It is not linked into an adapter artifact. - `tests/parity.rs` calls Axum, Cloudflare, and Spin routers in process and compares their shared route behavior. -- `tests/documentation_snippets.rs` extracts the checked integration-guide +- `tests/documentation_snippets.rs` extracts each checked integration-guide fixture and compiles it in an isolated offline crate. - `tests/integration.rs` exercises the Fastly/Viceroy and Axum paths against the WordPress and Next.js fixture containers. diff --git a/crates/trusted-server-integration-tests/browser/initial-render/run.cjs b/crates/trusted-server-integration-tests/browser/initial-render/run.cjs index 681ba584a1..d69652bda6 100644 --- a/crates/trusted-server-integration-tests/browser/initial-render/run.cjs +++ b/crates/trusted-server-integration-tests/browser/initial-render/run.cjs @@ -67,7 +67,7 @@ const scriptContents = new Map( [ path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), path.join(dist, "tsjs-core.js"), path.join(dist, "tsjs-gpt.js"), @@ -163,7 +163,7 @@ const scenarios = [ scripts.push( path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), ); scripts.push( @@ -474,7 +474,7 @@ const scenarios = [ files: [ path.join( repo, - "crates/trusted-server-core/src/integrations/gpt_bootstrap.js", + "crates/ad-tag/google/src/gpt_bootstrap.js", ), path.join(dist, "tsjs-core.js"), path.join(dist, "tsjs-gpt.js"), diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index c3d9ff53d8..e49c170a2a 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -1,19 +1,23 @@ -[[handlers]] -path = "^/_ts/admin" -username = "admin" -password = "integration_admin_password" - [publisher] domain = "localhost" cookie_domain = "localhost" origin_url = "http://127.0.0.1:8888" proxy_secret = "integration_proxy_secret" +[geo] +# Viceroy maps the loopback client to US/CA (see viceroy-template.toml), so +# the platform geo module resolves a real place and the EC lifecycle +# scenarios exercise the US state opt-out machinery. +module = "platform" + [ec] -passphrase = "integration_ec_passphrase" +module = "hmac" ec_store = "ec_identity_store" pull_sync_concurrency = 3 +[ec.hmac] +passphrase = "integration_ec_passphrase" + [[ec.partners]] name = "Integration Test Partner" source_domain = "inttest.example.com" @@ -28,70 +32,18 @@ api_token = "integration_partner_token_bravo" [request_signing] enabled = false -config_store_id = "app_config" -secret_store_id = "secrets" - -[integrations.prebid] -enabled = false -timeout_ms = 1000 -debug = false -client_side_bidders = [] - -[integrations.nextjs] -enabled = false -rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] -max_combined_payload_bytes = 10485760 - -[integrations.testlight] -enabled = false -endpoint = "https://testlight.example.com/openrtb2/auction" -timeout_ms = 1200 -rewrite_scripts = true -[integrations.didomi] -enabled = false -geo_query_parameters = false -sdk_origin = "https://sdk.example.com" -api_origin = "https://api.example.com" - -[integrations.sourcepoint] -enabled = false -rewrite_sdk = true -cdn_origin = "https://cdn.example.com" -cache_ttl_seconds = 3600 +# Only the GPT diagnostics overlay runs, which is what the browser tests need. +# A settings table its section does not select refuses startup, so no other +# integration has a table here. +[ad-tag] +modules = ["google.diagnostics"] -[integrations.permutive] -enabled = false -organization_id = "" -workspace_id = "" -project_id = "" -api_endpoint = "https://api.example.com" -secure_signals_endpoint = "https://secure-signals.example.com" - -[integrations.lockr] -enabled = false -app_id = "" -api_endpoint = "https://identity.example.com" -sdk_url = "https://identity.example.com/trusted-server.js" -cache_ttl_seconds = 3600 -rewrite_sdk = true - -[integrations.datadome] -enabled = false -sdk_origin = "https://sdk.example.com" -api_origin = "https://api.example.com" -cache_ttl_seconds = 3600 -rewrite_sdk = true - -[integrations.gpt] -enabled = false -gam_attribution_enabled = false -script_url = "https://ads.example.com/gpt.js" -cache_ttl_seconds = 3600 -rewrite_script = true - -[integrations.gpt_diagnostics] -enabled = true +# What the overlay writes into a page is a serve middleware, which runs on +# each reader's copy of the pages an entry names it for. +[[serve]] +media_type = "text/html" +middleware = ["ad-tag.google.diagnostics"] [proxy] certificate_check = false @@ -101,47 +53,31 @@ enabled = false timeout_ms = 2000 allowed_context_keys = [] -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand] +modules = ["pbs_main", "aps_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = false consent_forwarding = "both" -[auction.providers.pbs-main.notifications] +[demand.pbs_main.notifications] suppress_all = false suppress_seats = [] -[auction.providers.aps-main] -protocol = "openrtb-2.6" -profile = "aps" +[demand.aps_main] +implementation = "auction.aps" endpoint = "https://aps.example.com/e/pb/bid" routing = "all_eligible" - -[auction.providers.aps-main.profile_config] account_id = "example-aps-account-id" debug = false allow_script_creatives = false [auction.bidders.example-bidder] -provider = "pbs-main" - -[integrations.google_tag_manager] -enabled = false -container_id = "GTM-EXAMPLE" -upstream_url = "https://tags.example.com" - -[integrations.adserver_mock] -enabled = false -endpoint = "https://adserver.example.com/mediate" -timeout_ms = 1000 - -[integrations.adserver_mock.context_query_params] -example_segments = "segments" +module = "pbs_main" [debug] ja4_endpoint_enabled = false diff --git a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml index eae7220a84..8967f9bfdc 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml @@ -4,6 +4,52 @@ [local_server] + # Maps the loopback client to California, United States, so the platform + # geo module resolves a real place and the permission model applies the + # US/CA rules from the permissions.yaml tree. + [local_server.geolocation] + format = "inline-toml" + + [local_server.geolocation.addresses."127.0.0.1"] + as_name = "Integration Test" + as_number = 64496 + area_code = 0 + city = "Test City" + conn_speed = "broadband" + conn_type = "wired" + continent = "NA" + country_code = "US" + country_code3 = "USA" + country_name = "United States" + latitude = 0.0 + longitude = 0.0 + metro_code = 0 + postal_code = "00000" + proxy_description = "?" + proxy_type = "?" + region = "CA" + utc_offset = -800 + + [local_server.geolocation.addresses."::1"] + as_name = "Integration Test" + as_number = 64496 + area_code = 0 + city = "Test City" + conn_speed = "broadband" + conn_type = "wired" + continent = "NA" + country_code = "US" + country_code3 = "USA" + country_name = "United States" + latitude = 0.0 + longitude = 0.0 + metro_code = 0 + postal_code = "00000" + proxy_description = "?" + proxy_type = "?" + region = "CA" + utc_offset = -800 + [local_server.backends] [local_server.kv_stores] @@ -62,13 +108,6 @@ key = "ts-2025-10-A" data = "NVnTYrw5xoyTJDOwoUWoPJO3A6UCCXOJJUzgGTxxx7k=" - [[local_server.secret_stores.api-keys]] - key = "api_key" - data = "test-api-key" - - [[local_server.secret_stores.ts_secrets]] - key = "integration_admin_password" - data = "integration-admin-password-32-bytes-ok" [[local_server.secret_stores.ts_secrets]] key = "integration_proxy_secret" data = "integration-test-proxy-secret-32-bytes-ok" diff --git a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs index aada5dcf54..f5bb7e80f5 100644 --- a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs +++ b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs @@ -121,6 +121,9 @@ fn build_app_config_envelope( if let Some(origin_url) = origin_url { settings.publisher.origin_url = origin_url.to_string(); } + // The fixture selects modules a stock build ships, so validation needs + // the stock list as the adapters and the `ts` tool have it. + trusted_server_core::config::register_deploy_integrations(trusted_server_modules::builders()); let app_config = TrustedServerAppConfig::new(settings) .map_err(|report| error_box(format!("invalid Trusted Server app config: {report:?}")))?; @@ -165,7 +168,7 @@ mod tests { use super::*; use error_stack::Report; use std::collections::HashMap; - use trusted_server_core::config_payload::settings_from_config_blob; + use trusted_server_core::config_payload::settings_from_config_blob_with; use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; const TEMPLATE: &str = include_str!("../../fixtures/configs/viceroy-template.toml"); @@ -204,10 +207,6 @@ mod tests { fn integration_secret_store() -> IntegrationSecretStore { IntegrationSecretStore { values: HashMap::from([ - ( - "integration_admin_password".to_owned(), - b"integration-admin-password-32-bytes-ok".to_vec(), - ), ( "integration_proxy_secret".to_owned(), b"integration-test-proxy-secret-32-bytes-ok".to_vec(), @@ -317,10 +316,11 @@ mod tests { fn generated_blob_verifies_and_applies_origin_override() { let envelope = build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999")) .expect("should build envelope"); - let settings = settings_from_config_blob( + let settings = settings_from_config_blob_with( &envelope, &integration_secret_store(), &StoreName::from("trusted_server_secrets"), + &trusted_server_modules::builders(), ) .expect("should verify blob"); diff --git a/crates/trusted-server-integration-tests/tests/common/config.rs b/crates/trusted-server-integration-tests/tests/common/config.rs index 0b793833f5..0ee777c35e 100644 --- a/crates/trusted-server-integration-tests/tests/common/config.rs +++ b/crates/trusted-server-integration-tests/tests/common/config.rs @@ -33,6 +33,9 @@ fn app_config_envelope( })?; let mut settings = app_config.into_settings(); settings.publisher.origin_url = origin_url; + // The fixture selects modules a stock build ships, so validation needs + // the stock list as the adapters and the `ts` tool have it. + trusted_server_core::config::register_deploy_integrations(trusted_server_modules::builders()); let app_config = TrustedServerAppConfig::new(settings).map_err(|report| { Report::new(TestError::ConfigGeneration) .attach(format!("invalid generated integration config: {report:?}")) @@ -57,14 +60,18 @@ pub fn cloudflare_config_json(origin_port: u16) -> TestResult { /// Builds a Cloudflare config binding with server-side auctions enabled. /// -/// The fixture's providers are removed so the enabled auction completes -/// through the orchestrator and telemetry without any outbound provider call. +/// The fixture's demand modules and bidder routes are removed so the enabled +/// auction completes through the orchestrator and telemetry without any +/// outbound call, and so Cloudflare, which runs one demand source at a time, +/// accepts it at startup. pub fn cloudflare_auction_config_json(origin_port: u16) -> TestResult { let envelope = app_config_envelope(origin_port, |table| { let auction = table_mut(table, "auction")?; auction.insert("enabled".to_string(), toml::Value::Boolean(true)); - auction.remove("providers"); auction.remove("bidders"); + let mut demand = toml::Table::new(); + demand.insert("modules".to_string(), toml::Value::Array(Vec::new())); + table.insert("demand".to_string(), toml::Value::Table(demand)); Ok(()) })?; cloudflare_binding_json(&serde_json::json!({ CONFIG_BLOB_KEY: envelope })) @@ -153,11 +160,7 @@ mod tests { .as_array() .expect("fastly.toml should define ts_secrets"); - for key in [ - "publisher_proxy_secret", - "ec_passphrase", - "handler_password", - ] { + for key in ["publisher_proxy_secret", "ec_passphrase"] { assert!( entries .iter() diff --git a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs index d9c36d2338..b215d47388 100644 --- a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs +++ b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs @@ -3,15 +3,30 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; const GUIDE: &str = include_str!("../../../docs/guide/integration-guide.md"); -const START: &str = ""; -const END: &str = ""; #[test] fn integration_guide_runtime_services_fixture_compiles() { + assert_documented_fixture_compiles("runtime-services"); +} + +#[test] +fn integration_guide_middleware_fixture_compiles() { + assert_documented_fixture_compiles("middleware"); +} + +/// Compiles the fence the guide marks as the snippet `name`, exactly as +/// written, as an isolated crate that depends on core alone. +#[allow( + clippy::panic, + reason = "a helper of the tests above, which fails the test that called it" +)] +fn assert_documented_fixture_compiles(name: &str) { + let start = format!(""); + let end = format!(""); let marked = GUIDE - .split_once(START) - .and_then(|(_, tail)| tail.split_once(END).map(|(body, _)| body)) - .expect("should contain one bounded runtime-services snippet"); + .split_once(&start) + .and_then(|(_, tail)| tail.split_once(&end).map(|(body, _)| body)) + .unwrap_or_else(|| panic!("should contain one bounded {name} snippet")); let source = marked .trim() .strip_prefix("```rust\n") @@ -23,7 +38,7 @@ fn integration_guide_runtime_services_fixture_compiles() { .expect("should have a current system clock") .as_nanos(); let fixture_root = std::env::temp_dir().join(format!( - "trusted-server-documentation-snippet-{}-{nonce}", + "trusted-server-documentation-snippet-{name}-{}-{nonce}", std::process::id() )); let source_dir = fixture_root.join("src"); @@ -38,18 +53,24 @@ fn integration_guide_runtime_services_fixture_compiles() { let core_path = repository_root.join("crates/trusted-server-core"); let workspace_manifest = fs::read_to_string(repository_root.join("Cargo.toml")) .expect("should read the workspace manifest"); - let error_stack_requirement = workspace_manifest - .lines() - .find_map(|line| { - let assignment = line.trim().strip_prefix("error-stack")?.trim_start(); - let value = assignment.strip_prefix('=')?; - let start = value.find('"')? + 1; - let end = start + value[start..].find('"')?; - Some(value[start..end].to_owned()) - }) - .expect("should find the workspace error-stack requirement"); + let requirement_of = |crate_name: &str| { + workspace_manifest + .lines() + .find_map(|line| { + let assignment = line.trim().strip_prefix(crate_name)?.trim_start(); + let value = assignment.strip_prefix('=')?; + let start = value.find('"')? + 1; + let end = start + value[start..].find('"')?; + Some(value[start..end].to_owned()) + }) + .unwrap_or_else(|| panic!("should find the workspace {crate_name} requirement")) + }; + let error_stack_requirement = requirement_of("error-stack"); + // The documented fixture implements `PlatformGeo`, whose `lookup` is + // asynchronous, so it carries the same attribute the trait does. + let async_trait_requirement = requirement_of("async-trait"); let manifest = format!( - "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", + "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nasync-trait = \"{async_trait_requirement}\"\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", core_path ); fs::write(fixture_root.join("Cargo.toml"), manifest) @@ -76,7 +97,7 @@ fn integration_guide_runtime_services_fixture_compiles() { assert!( output.status.success(), - "documented integration fixture must compile:\nstdout:\n{}\nstderr:\n{}", + "the documented {name} fixture must compile:\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); diff --git a/crates/trusted-server-integration-tests/tests/environments/axum.rs b/crates/trusted-server-integration-tests/tests/environments/axum.rs index 6e1388cea3..0afdfbacc3 100644 --- a/crates/trusted-server-integration-tests/tests/environments/axum.rs +++ b/crates/trusted-server-integration-tests/tests/environments/axum.rs @@ -14,10 +14,6 @@ const AXUM_DEFAULT_PORT: u16 = 8787; /// Secret-store entries referenced by the integration app-config fixture. const INTEGRATION_SECRET_ENV: &[(&str, &str)] = &[ - ( - "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_ADMIN_PASSWORD", - "integration-admin-password-32-bytes-ok", - ), ( "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_PROXY_SECRET", "integration-test-proxy-secret-32-bytes-ok", diff --git a/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs b/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs index c1bbafe8df..a7a45f2f63 100644 --- a/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs +++ b/crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs @@ -445,9 +445,10 @@ pub enum EcScenario { /// returns the scoped UID. FullLifecycle, - /// Consent withdrawal: GPC header triggers EC cookie deletion for a - /// seeded EC in the default US-state test geo. - ConsentWithdrawal, + /// Opt-out suppression: a GPC header suppresses use of a seeded EC + /// (identify answers 403) without expiring the cookie, in the default + /// US-state test geo. Opt-outs are never destructive. + OptOutSuppression, /// Identify without EC cookie returns 204. IdentifyWithoutEc, @@ -470,7 +471,7 @@ impl EcScenario { pub fn all() -> Vec { vec![ Self::FullLifecycle, - Self::ConsentWithdrawal, + Self::OptOutSuppression, Self::IdentifyWithoutEc, Self::IdentifyConsentDenied, Self::ConcurrentPartnerSyncs, @@ -489,7 +490,7 @@ impl EcScenario { pub fn run(&self, base_url: &str) -> TestResult<()> { match self { Self::FullLifecycle => ec_full_lifecycle(base_url), - Self::ConsentWithdrawal => ec_consent_withdrawal(base_url), + Self::OptOutSuppression => ec_opt_out_suppression(base_url), Self::IdentifyWithoutEc => ec_identify_without_ec(base_url), Self::IdentifyConsentDenied => ec_identify_consent_denied(base_url), Self::ConcurrentPartnerSyncs => ec_concurrent_partner_syncs(base_url), @@ -577,37 +578,48 @@ fn ec_full_lifecycle(base_url: &str) -> TestResult<()> { } /// Consent withdrawal: GPC header clears EC cookie. -fn ec_consent_withdrawal(base_url: &str) -> TestResult<()> { +fn ec_opt_out_suppression(base_url: &str) -> TestResult<()> { let client = EcTestClient::new(base_url); allow_ec_generation(&client); let seeded_ec_id = seeded_ec_id('b', "test02"); let ec_id = use_seeded_ec(&client, &seeded_ec_id); - log::info!("EC consent withdrawal: using seeded EC = {ec_id}"); + log::info!("EC opt-out suppression: using seeded EC = {ec_id}"); - // GPC overrides the allow cookie in US-CA, so this is an explicit - // withdrawal and must expire the EC cookie. + // GPC is a US-style opt-out. It suppresses use of the identifier for the + // request but is never destructive: the cookie is not expired and no + // tombstone is written, so lifting the opt-out restores the identity. let resp = client.get_with_headers("/", &[("sec-gpc", "1")])?; - - if !is_ec_cookie_expired(&resp) { + if is_ec_cookie_expired(&resp) { return Err(Report::new(TestError::UnexpectedContent) - .attach("consent withdrawal should expire ts-ec cookie (expected Max-Age=0)")); + .attach("an opt-out must suppress without expiring the ts-ec cookie")); } - if client.ec_cookie_value().is_some() { + if client.ec_cookie_value().is_none() { return Err(Report::new(TestError::UnexpectedContent) - .attach("client should stop tracking ts-ec after explicit withdrawal")); + .attach("the client should keep tracking ts-ec through an opt-out")); } - // 3. With consent still granted and the EC cookie revoked, identify should - // now report no EC present. - let resp = identify(&client, INTTEST_API_TOKEN)?; - assert_status(&resp, 204).attach("identify should return 204 after cookie revocation")?; - - // 4. With GPC still asserted, identify should reflect consent denial. + // With GPC asserted, identify reflects the suppressed permissions. let resp = identify_with_headers(&client, INTTEST_API_TOKEN, &[("sec-gpc", "1")])?; assert_status(&resp, 403) - .attach("identify with GPC should return 403 after consent withdrawal")?; + .attach("identify with GPC should return 403 while the opt-out is asserted")?; + + // Without GPC the identity is usable again: the cookie still carries the + // identifier, so identify answers 200 with consent ok. No row was seeded + // for this identifier, so there is no enrichment, and the absence of a + // 403 proves the opt-out destroyed nothing. + let resp = identify(&client, INTTEST_API_TOKEN)?; + let body = assert_json_response(resp, 200)?; + if body.get("consent").and_then(|v| v.as_str()) != Some("ok") { + return Err(Report::new(TestError::UnexpectedContent).attach(format!( + "identify without GPC should report consent ok, got {body}" + ))); + } + if body.get("uid").is_some() { + return Err(Report::new(TestError::UnexpectedContent) + .attach("no partner UID was seeded, so identify should carry no enrichment")); + } - log::info!("EC consent withdrawal: PASSED"); + log::info!("EC opt-out suppression: PASSED"); Ok(()) } diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 2428d7799b..65b8f9fabd 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -2,7 +2,7 @@ //! //! Sends identical requests to all host-runnable adapters and asserts that: //! - Response status codes match -//! - Critical headers (X-Geo-Info-Available, WWW-Authenticate on 401) match +//! - Critical headers (X-Geo-Info-Available) match //! //! Fastly parity is verified via cargo test-fastly + Viceroy in CI. @@ -28,17 +28,9 @@ use trusted_server_core::test_support::nextjs_auction; /// The settings baked into the binaries contain placeholder secrets that /// `get_settings()` rejects by design, so all routers are built through /// their `routes_with_settings` testing seams from this known-good config. -/// The handler regex is the production-shaped `^/_ts/admin`, matching -/// `Settings::ADMIN_ENDPOINTS` and the default config, so the canonical -/// `/_ts/admin/keys/*` routes are auth-gated exactly as in production. fn test_settings() -> Settings { Settings::from_toml( r#" - [[handlers]] - path = "^/_ts/admin" - username = "admin" - password = "admin-pass" - [publisher] domain = "test-publisher.example.com" cookie_domain = ".test-publisher.example.com" @@ -46,7 +38,13 @@ fn test_settings() -> Settings { proxy_secret = "parity-test-proxy-secret" [ec] + module = "hmac" + + [ec.hmac] passphrase = "test-secret-key-32-bytes-minimum" + + [geo] + assume_single_jurisdiction = true "#, ) .expect("should parse parity test settings") @@ -434,184 +432,231 @@ async fn discovery_route_body_is_json_parity() { } } +/// The configuration view is built by core from the settings alone, so +/// every adapter shows the same one. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn verify_signature_route_parity() { - // known divergence: without real signing-key configuration the handler may - // return 5xx. The parity assertion is that both adapters agree on the status - // (routing and middleware are wired identically). - let (axum_status, _) = axum_post_headers("/verify-signature", "{}").await; - let (cf_status, _) = cf_post_headers("/verify-signature", "{}").await; - let (spin_status, _) = spin_post_headers("/verify-signature", "{}").await; +async fn config_endpoint_parity() { + use http_body_util::BodyExt as _; + use serde_json::Value; - assert_ne!(axum_status, 404, "Axum /verify-signature must be routed"); - assert_ne!( - cf_status, 404, - "Cloudflare /verify-signature must be routed" - ); - assert_ne!(spin_status, 404, "Spin /verify-signature must be routed"); + const PATH: &str = "/_ts/config.json"; + + let (axum_status, axum_body) = { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("GET") + .uri(PATH) + .body(AxumBody::empty()) + .expect("should build GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + let status = resp.status().as_u16(); + let body = resp + .into_body() + .collect() + .await + .expect("should collect body") + .to_bytes(); + (status, body) + }; + let (cf_status, cf_body) = { + let req = request_builder() + .method("GET") + .uri(PATH) + .body(edgezero_core::body::Body::empty()) + .expect("should build GET request"); + let resp = cf_router().oneshot(req).await.expect("should respond"); + let status = resp.status().as_u16(); + (status, resp.into_body().into_bytes().unwrap_or_default()) + }; + let (spin_status, _, spin_body) = spin_get_body(PATH).await; + + assert_eq!(axum_status, 200, "Axum should answer {PATH}"); + assert_eq!(cf_status, 200, "Cloudflare should answer {PATH}"); + assert_eq!(spin_status, 200, "Spin should answer {PATH}"); + + let axum: Value = serde_json::from_slice(&axum_body).expect("Axum should answer JSON"); + let cf: Value = serde_json::from_slice(&cf_body).expect("Cloudflare should answer JSON"); + let spin: Value = serde_json::from_slice(&spin_body).expect("Spin should answer JSON"); + assert_eq!(axum, cf, "{PATH} must match across adapters (axum, cf)"); + assert_eq!(cf, spin, "{PATH} must match across adapters (cf, spin)"); assert_eq!( - axum_status, cf_status, - "/verify-signature must return same status: axum={axum_status} cf={cf_status}" + axum["settings"]["publisher"]["proxy_secret"], "XXXX", + "a secret should be masked" ); - assert_eq!( - cf_status, spin_status, - "/verify-signature must return same status: cf={cf_status} spin={spin_status}" + assert!( + !String::from_utf8_lossy(&axum_body).contains("parity-test-proxy-secret"), + "the view should not carry the secret anywhere" ); } +/// Whether a request is a browser opening the page is core's decision, and +/// none of these three adapters keeps an identity graph, so each refuses and +/// answers alike. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_rotate_unauthenticated_parity() { - // Both adapters must return 401 for unauthenticated admin requests on the - // canonical `/_ts/admin/keys/*` path that production config auth-gates. - // The authenticated-path divergence (Axum→501 no-KV, CF→4xx no-KV) - // is separate and not covered here. - let (axum_status, axum_headers) = axum_post_headers("/_ts/admin/keys/rotate", "{}").await; - let (cf_status, cf_headers) = cf_post_headers("/_ts/admin/keys/rotate", "{}").await; - let (spin_status, spin_headers) = spin_post_headers("/_ts/admin/keys/rotate", "{}").await; +async fn data_endpoint_parity() { + use http_body_util::BodyExt as _; - assert_eq!( - axum_status, 401, - "Axum must return 401 for unauthenticated admin route" - ); - assert_eq!( - cf_status, 401, - "Cloudflare must return 401 for unauthenticated admin route" - ); - assert_eq!( - spin_status, 401, - "Spin must return 401 for unauthenticated admin route" - ); - assert_eq!( - axum_status, cf_status, - "Axum and Cloudflare must return the same status for unauthenticated admin route" - ); - assert_eq!( - cf_status, spin_status, - "Cloudflare and Spin must return the same status for unauthenticated admin route" - ); + const PATH: &str = "/_ts/data"; + const OPENED_AS_A_PAGE: [(&str, &str); 2] = [ + ("sec-fetch-mode", "navigate"), + ("sec-fetch-dest", "document"), + ]; - let axum_www_auth = axum_headers - .get("www-authenticate") - .expect("Axum 401 must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - let cf_www_auth = cf_headers - .get("www-authenticate") - .expect("Cloudflare 401 must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - axum_www_auth, cf_www_auth, - "WWW-Authenticate header value must match across adapters for /admin/keys/rotate" - ); - assert!( - axum_www_auth.starts_with("Basic"), - "WWW-Authenticate must use Basic scheme: {axum_www_auth:?}" - ); - let spin_www_auth = spin_headers - .get("www-authenticate") - .expect("Spin 401 must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - cf_www_auth, spin_www_auth, - "WWW-Authenticate value must match: cf={cf_www_auth:?} spin={spin_www_auth:?}" - ); + async fn edge( + router: RouterService, + headers: &[(&str, &str)], + ) -> (u16, Option, bytes::Bytes) { + let mut builder = request_builder().method("GET").uri(PATH); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder + .body(edgezero_core::body::Body::empty()) + .expect("should build GET request"); + let resp = router.oneshot(req).await.expect("should respond"); + ( + resp.status().as_u16(), + resp.headers().get("content-security-policy").cloned(), + resp.into_body().into_bytes().unwrap_or_default(), + ) + } + + for (headers, expected) in [(&[][..], 403), (&OPENED_AS_A_PAGE[..], 200)] { + let axum = { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let mut builder = AxumRequest::builder().method("GET").uri(PATH); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let req = builder + .body(AxumBody::empty()) + .expect("should build GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + let status = resp.status().as_u16(); + let policy = resp.headers().get("content-security-policy").cloned(); + let body = resp + .into_body() + .collect() + .await + .expect("should collect body") + .to_bytes(); + (status, policy, body) + }; + let cf = edge(cf_router(), headers).await; + let spin = edge(spin_router(), headers).await; + + assert_eq!( + axum.0, expected, + "Axum should answer {PATH} with {expected}" + ); + assert_eq!(axum, cf, "{PATH} must match across adapters (axum, cf)"); + assert_eq!(cf, spin, "{PATH} must match across adapters (cf, spin)"); + assert!( + axum.1 + .as_ref() + .and_then(|policy| policy.to_str().ok()) + .is_some_and(|policy| policy.starts_with("sandbox;")), + "every answer should be in an origin of its own" + ); + } +} + +/// The permissions a request resolves to are core's decision, so the +/// endpoint that shows them answers the same on every adapter. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn permissions_endpoint_parity() { + use http_body_util::BodyExt as _; + use serde_json::Value; + + const PATH: &str = "/_ts/permissions.json"; + + let (axum_status, axum_body) = { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("GET") + .uri(PATH) + .body(AxumBody::empty()) + .expect("should build GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + let status = resp.status().as_u16(); + let body = resp + .into_body() + .collect() + .await + .expect("should collect body") + .to_bytes(); + (status, body) + }; + let (cf_status, cf_body) = { + let req = request_builder() + .method("GET") + .uri(PATH) + .body(edgezero_core::body::Body::empty()) + .expect("should build GET request"); + let resp = cf_router().oneshot(req).await.expect("should respond"); + let status = resp.status().as_u16(); + (status, resp.into_body().into_bytes().unwrap_or_default()) + }; + let (spin_status, _, spin_body) = spin_get_body(PATH).await; + + assert_eq!(axum_status, 200, "Axum should answer {PATH}"); + assert_eq!(cf_status, 200, "Cloudflare should answer {PATH}"); + assert_eq!(spin_status, 200, "Spin should answer {PATH}"); + + let axum: Value = serde_json::from_slice(&axum_body).expect("Axum should answer JSON"); + let cf: Value = serde_json::from_slice(&cf_body).expect("Cloudflare should answer JSON"); + let spin: Value = serde_json::from_slice(&spin_body).expect("Spin should answer JSON"); + assert_eq!(axum, cf, "{PATH} must match across adapters (axum, cf)"); + assert_eq!(cf, spin, "{PATH} must match across adapters (cf, spin)"); assert!( - spin_www_auth.starts_with("Basic"), - "Spin WWW-Authenticate must use Basic scheme: {spin_www_auth:?}" + axum["set"].is_array() && axum["awaiting"].is_array(), + "the resolution should say what is set and what is awaited" ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_deactivate_unauthenticated_parity() { - // Mirror of admin_rotate_unauthenticated_parity for the deactivate endpoint. - let (axum_status, axum_headers) = axum_post_headers("/_ts/admin/keys/deactivate", "{}").await; - let (cf_status, cf_headers) = cf_post_headers("/_ts/admin/keys/deactivate", "{}").await; - let (spin_status, spin_headers) = spin_post_headers("/_ts/admin/keys/deactivate", "{}").await; +async fn verify_signature_route_parity() { + // known divergence: without real signing-key configuration the handler may + // return 5xx. The parity assertion is that both adapters agree on the status + // (routing and middleware are wired identically). + let (axum_status, _) = axum_post_headers("/verify-signature", "{}").await; + let (cf_status, _) = cf_post_headers("/verify-signature", "{}").await; + let (spin_status, _) = spin_post_headers("/verify-signature", "{}").await; - assert_eq!( - axum_status, 401, - "Axum must return 401 for unauthenticated admin/keys/deactivate" - ); - assert_eq!( - cf_status, 401, - "Cloudflare must return 401 for unauthenticated admin/keys/deactivate" - ); - assert_eq!( - spin_status, 401, - "Spin must return 401 for unauthenticated admin/keys/deactivate" + assert_ne!(axum_status, 404, "Axum /verify-signature must be routed"); + assert_ne!( + cf_status, 404, + "Cloudflare /verify-signature must be routed" ); + assert_ne!(spin_status, 404, "Spin /verify-signature must be routed"); assert_eq!( axum_status, cf_status, - "Axum and Cloudflare must return the same status for unauthenticated admin/keys/deactivate" + "/verify-signature must return same status: axum={axum_status} cf={cf_status}" ); assert_eq!( cf_status, spin_status, - "Cloudflare and Spin must return the same status for unauthenticated admin/keys/deactivate" - ); - - let axum_www_auth = axum_headers - .get("www-authenticate") - .expect("Axum 401 on admin/keys/deactivate must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - let cf_www_auth = cf_headers - .get("www-authenticate") - .expect("Cloudflare 401 on admin/keys/deactivate must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - axum_www_auth, cf_www_auth, - "WWW-Authenticate header value must match across adapters for /admin/keys/deactivate" - ); - assert!( - axum_www_auth.starts_with("Basic"), - "WWW-Authenticate must use Basic scheme: {axum_www_auth:?}" - ); - let spin_www_auth = spin_headers - .get("www-authenticate") - .expect("Spin 401 on admin/keys/deactivate must include WWW-Authenticate") - .to_str() - .expect("should be valid UTF-8"); - assert_eq!( - cf_www_auth, spin_www_auth, - "WWW-Authenticate value must match: cf={cf_www_auth:?} spin={spin_www_auth:?}" - ); - assert!( - spin_www_auth.starts_with("Basic"), - "Spin WWW-Authenticate must use Basic scheme: {spin_www_auth:?}" + "/verify-signature must return same status: cf={cf_status} spin={spin_status}" ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn spin_legacy_admin_aliases_are_denied_locally_not_proxied() { - // The production handler regex `^/_ts/admin` only matches the canonical - // `/_ts/admin/keys/*` paths. Legacy aliases must therefore fail closed with a - // local 404 instead of reaching either the admin handlers or the publisher - // fallback. - for alias in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - let (canonical_status, _) = spin_post_headers(&format!("/_ts{alias}"), "{}").await; - assert_eq!( - canonical_status, 401, - "canonical /_ts{alias} must challenge unauthenticated callers" - ); - - for method in ["POST", "GET"] { - let (alias_status, alias_headers) = - spin_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; - assert_eq!( - alias_status, 404, - "Spin legacy {method} {alias} must be denied locally with 404" - ); - assert!( - !alias_headers.contains_key("www-authenticate"), - "Spin legacy {method} {alias} must not issue an admin auth challenge" - ); - } - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn geo_header_parity_on_all_responses() { let routes_to_check: &[(&str, &str, &str)] = &[ @@ -671,14 +716,11 @@ async fn geo_header_parity_on_all_responses() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn auction_not_challenged_by_auth_parity() { +async fn auction_is_routed_parity() { let (axum_status, _) = axum_post_headers("/auction", r#"{"adUnits":[]}"#).await; let (cf_status, _) = cf_post_headers("/auction", r#"{"adUnits":[]}"#).await; let (spin_status, _) = spin_post_headers("/auction", r#"{"adUnits":[]}"#).await; - assert_ne!(axum_status, 401, "Axum /auction must not 401"); - assert_ne!(cf_status, 401, "Cloudflare /auction must not 401"); - assert_ne!(spin_status, 401, "Spin /auction must not 401"); assert_ne!(axum_status, 404, "Axum /auction must be routed (not 404)"); assert_ne!( cf_status, 404, @@ -749,7 +791,6 @@ async fn spin_auction_ignores_spoofed_forwarded_headers() { ) .await; - assert_ne!(spoofed_status, 401, "Spin /auction must not 401"); assert_ne!( spoofed_status, 404, "Spin /auction must be routed (not 404)" @@ -814,177 +855,40 @@ async fn unknown_route_returns_same_status_parity() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn legacy_admin_aliases_are_denied_locally_not_proxied() { - // The production handler regex `^/_ts/admin` only matches the canonical - // `/_ts/admin/keys/*` paths, so the legacy `/admin/keys/*` aliases are not - // auth-gated. They must be denied locally with 404 — never routed to the key - // handlers (which would execute admin operations), and never left unrouted to - // fall through to the publisher fallback, which forwards the request - // (including any `Authorization` header and key-management body) to the origin - // and leaks admin credentials. - // - // Primary guard: assert the route tables directly. The legacy aliases must - // be registered (to the local deny) so they can never reach the publisher - // fallback as an unrouted path. - let cf_registered: Vec<(String, String)> = cf_router() - .routes() - .iter() - .map(|route| (route.method().to_string(), route.path().to_string())) - .collect(); - let spin_registered: Vec<(String, String)> = spin_router() - .routes() - .iter() - .map(|route| (route.method().to_string(), route.path().to_string())) - .collect(); - let cf_is_registered = - |method: &str, path: &str| cf_registered.iter().any(|(m, p)| m == method && p == path); - let spin_is_registered = |method: &str, path: &str| { - spin_registered - .iter() - .any(|(m, p)| m == method && p == path) - }; - - for path in ["/_ts/admin/keys/rotate", "/_ts/admin/keys/deactivate"] { - assert!( - cf_is_registered("POST", path), - "Cloudflare POST {path} must be a registered route" - ); - assert!( - spin_is_registered("POST", path), - "Spin POST {path} must be a registered route" - ); - } - for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - for method in ["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - assert!( - cf_is_registered(method, path), - "Cloudflare {method} {path} must be a registered route" - ); - assert!( - spin_is_registered(method, path), - "Spin {method} {path} must be a registered route" - ); - } - } - - // Secondary guard: confirm runtime behavior. Canonical paths challenge - // unauthenticated callers (401). Legacy aliases are denied locally with 404 - // and carry no auth challenge — a reintroduced key handler at these ungated - // paths would not return 404, and the publisher fallback would not either, so - // 404 proves the local deny ran. - for alias in ["/admin/keys/rotate", "/admin/keys/deactivate"] { - let (canonical_status, _) = cf_post_headers(&format!("/_ts{alias}"), "{}").await; - assert_eq!( - canonical_status, 401, - "Cloudflare canonical /_ts{alias} must challenge unauthenticated callers" - ); - let (spin_canonical_status, _) = spin_post_headers(&format!("/_ts{alias}"), "{}").await; - assert_eq!( - spin_canonical_status, 401, - "Spin canonical /_ts{alias} must challenge unauthenticated callers" - ); - +async fn key_administration_paths_are_answered_here_and_never_proxied() { + // A request to a key administration path can carry an operator's + // credentials and a key payload. Every adapter must answer it with 404 and + // never reach the publisher fallback, which forwards the request, + // `Authorization` header and body included, to the origin. A key handler + // at one of these paths would not return 404, and the publisher fallback + // would not either, so 404 proves the request was answered here. + for path in [ + "/_ts/admin/keys/rotate", + "/_ts/admin/keys/deactivate", + "/admin/keys/rotate", + "/admin/keys/deactivate", + ] { for method in ["POST", "GET"] { let (axum_status, axum_headers) = - axum_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; + axum_authorized_json(method, path, r#"{"key_id":"leak-me"}"#).await; let (cf_status, cf_headers) = - cf_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; + cf_authorized_json(method, path, r#"{"key_id":"leak-me"}"#).await; let (spin_status, spin_headers) = - spin_authorized_json(method, alias, r#"{"key_id":"leak-me"}"#).await; - assert_eq!( - axum_status, 404, - "Axum legacy {method} {alias} must be denied locally with 404" - ); - assert_eq!( - cf_status, 404, - "Cloudflare legacy {method} {alias} must be denied locally with 404" - ); - assert_eq!( - axum_status, cf_status, - "legacy {method} {alias} status must match across adapters" - ); - assert_eq!( - cf_status, spin_status, - "legacy {method} {alias} status must match across adapters" - ); - assert!( - !axum_headers.contains_key("www-authenticate"), - "Axum legacy {method} {alias} must not issue an admin auth challenge" - ); - assert!( - !cf_headers.contains_key("www-authenticate"), - "Cloudflare legacy {method} {alias} must not issue an admin auth challenge" - ); - assert!( - !spin_headers.contains_key("www-authenticate"), - "Spin legacy {method} {alias} must not issue an admin auth challenge" - ); - } - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_cache_purge_not_implemented_parity() { - // The template cache is Fastly-backed, so the other three adapters answer 501 rather - // than letting the path fall through to the publisher origin and 404 — a CMS purge - // webhook needs to tell "not supported here" from "no such endpoint". - let body = r#"{"scope":"all"}"#; - - let (axum_status, _) = axum_authorized_json("POST", "/_ts/admin/cache/purge", body).await; - let (cf_status, _) = cf_authorized_json("POST", "/_ts/admin/cache/purge", body).await; - let (spin_status, _) = spin_authorized_json("POST", "/_ts/admin/cache/purge", body).await; - - assert_eq!(axum_status, 501, "Axum must answer cache purge with 501"); - assert_eq!( - cf_status, 501, - "Cloudflare must answer cache purge with 501" - ); - assert_eq!(spin_status, 501, "Spin must answer cache purge with 501"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_cache_purge_unauthenticated_parity() { - // Guards the test above from passing for the wrong reason. Without credentials the - // path must 401, which proves the 501s were reached through auth rather than being - // the 401s of a probe that never arrived at a handler. - let body = r#"{"scope":"all"}"#; - - let (axum_status, _) = axum_post_headers("/_ts/admin/cache/purge", body).await; - let (cf_status, _) = cf_post_headers("/_ts/admin/cache/purge", body).await; - let (spin_status, _) = spin_post_headers("/_ts/admin/cache/purge", body).await; - - for (adapter, status) in [ - ("Axum", axum_status), - ("Cloudflare", cf_status), - ("Spin", spin_status), - ] { - assert_eq!( - status, 401, - "{adapter} must require auth on the cache purge path" - ); - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn admin_cache_purge_rejects_credential_forwarding_methods() { - // The guard the Fastly route exists for, asserted cross-adapter: a method the route - // does not claim falls through to the publisher with the Authorization header still - // attached. Every method must be answered locally, never forwarded. - for method in ["GET", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"] { - let (axum_status, _) = axum_authorized_json(method, "/_ts/admin/cache/purge", "").await; - let (cf_status, _) = cf_authorized_json(method, "/_ts/admin/cache/purge", "").await; - let (spin_status, _) = spin_authorized_json(method, "/_ts/admin/cache/purge", "").await; - - for (adapter, status) in [ - ("Axum", axum_status), - ("Cloudflare", cf_status), - ("Spin", spin_status), - ] { - assert_eq!( - status, 501, - "{adapter} must answer {method} locally; a fallthrough would ship the \ - admin credential to the origin" - ); + spin_authorized_json(method, path, r#"{"key_id":"leak-me"}"#).await; + for (adapter, status, headers) in [ + ("Axum", axum_status, &axum_headers), + ("Cloudflare", cf_status, &cf_headers), + ("Spin", spin_status, &spin_headers), + ] { + assert_eq!( + status, 404, + "{adapter} {method} {path} must be answered here with 404" + ); + assert!( + !headers.contains_key("www-authenticate"), + "{adapter} {method} {path} must not ask for credentials" + ); + } } } } @@ -1119,3 +1023,176 @@ async fn adapter_buffers_nextjs_auction_output() { } } } + +/// The shared settings with an `[attestation]` section whose endpoint sits +/// under a prefix of its own, so the deployment owns the prefix. +fn attestation_settings() -> Settings { + let mut settings = test_settings(); + settings.attestation = Some(trusted_server_core::attestation::AttestationConfig { + endpoint: "/_ex/attestation".to_owned(), + operator: "Example Operator".to_owned(), + context: "example-attestation:v1".to_owned(), + verify_url: "https://verifier.example/verify".to_owned(), + }); + settings +} + +/// The value of header `name`, or an empty string when it is absent. +fn header_text(headers: &HeaderMap, name: &str) -> String { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned() +} + +/// One adapter's answer to a request. +struct Answer { + adapter: &'static str, + status: u16, + headers: HeaderMap, +} + +/// What each host-runnable adapter answers to `method` at `uri` under +/// `settings`. +async fn answers(settings: &Settings, method: &str, uri: &str) -> Vec { + let mut axum = EdgeZeroAxumService::new( + AxumApp::routes_with_settings(settings.clone()).expect("should build Axum router"), + ); + let request = AxumRequest::builder() + .method(method) + .uri(uri) + .body(AxumBody::empty()) + .expect("should build request"); + let response = axum + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond"); + let mut answers = vec![Answer { + adapter: "axum", + status: response.status().as_u16(), + headers: response.headers().clone(), + }]; + + for (adapter, router) in [ + ( + "cloudflare", + CloudflareApp::routes_with_settings(settings.clone()) + .expect("should build Cloudflare router"), + ), + ( + "spin", + SpinApp::routes_with_settings(settings.clone()).expect("should build Spin router"), + ), + ] { + let request = request_builder() + .method(method) + .uri(uri) + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let response = router.oneshot(request).await.expect("should respond"); + answers.push(Answer { + adapter, + status: response.status().as_u16(), + headers: response.headers().clone(), + }); + } + answers +} + +/// A deployment whose endpoint sits under a prefix of its own answers +/// everything beneath that prefix the same way on every adapter. Reads of the +/// two attestation addresses reach the handler, the prefix lists its pages, +/// and anything else is not found rather than sent to the publisher's origin. +/// Test builds carry no signing key, so the handler answers 503 in JSON. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn attestation_prefix_parity() { + let cases = [ + ("GET", "/_ex", 200, "text/html"), + ("GET", "/_ex/", 200, "text/html"), + ("GET", "/_ex/attestation.json", 503, "application/json"), + ("HEAD", "/_ex/attestation.json", 503, "application/json"), + ("GET", "/_ex/attestation", 503, "application/json"), + ("POST", "/_ex/attestation", 404, "text/html"), + ("PUT", "/_ex/attestation.json", 404, "text/html"), + ("DELETE", "/_ex", 404, "text/html"), + ("GET", "/_ex/unknown/page", 404, "text/html"), + ]; + let settings = attestation_settings(); + for (method, uri, status, content_type) in cases { + for answer in answers(&settings, method, uri).await { + let case = format!("{} {method} {uri}", answer.adapter); + let got_type = header_text(&answer.headers, "content-type"); + assert_eq!(answer.status, status, "should answer {case} alike"); + assert!( + got_type.starts_with(content_type), + "should answer {case} as {content_type}, got {got_type}" + ); + assert_eq!( + header_text(&answer.headers, "cache-control"), + "no-store", + "should keep the answer to {case} out of every cache" + ); + } + } +} + +/// When the endpoint is not the default, every adapter sends a read of the +/// default address to the endpoint, page to page and JSON to JSON, keeping +/// the nonce. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn attestation_default_address_redirect_parity() { + let cases = [ + ("GET", "/_ts/attestation", "/_ex/attestation"), + ("HEAD", "/_ts/attestation", "/_ex/attestation"), + ( + "GET", + "/_ts/attestation.json?nonce=abc123", + "/_ex/attestation.json?nonce=abc123", + ), + ]; + let settings = attestation_settings(); + for (method, uri, location) in cases { + for answer in answers(&settings, method, uri).await { + let case = format!("{} {method} {uri}", answer.adapter); + assert_eq!(answer.status, 301, "should redirect {case}"); + assert_eq!( + header_text(&answer.headers, "location"), + location, + "should send {case} to the endpoint with its query kept" + ); + assert_eq!( + header_text(&answer.headers, "cache-control"), + "no-store", + "should keep the redirect of {case} out of every cache" + ); + } + } +} + +/// Without the section no adapter takes the default address, so a read of it +/// reaches the publisher's origin as any other path does. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn attestation_is_not_served_without_its_section_parity() { + let settings = test_settings(); + let unknown = answers(&settings, "GET", "/a-path-no-route-names").await; + let attestation = answers(&settings, "GET", "/_ts/attestation.json").await; + + for (unknown, attestation) in unknown.iter().zip(&attestation) { + assert_eq!( + attestation.status, unknown.status, + "{} should answer the attestation address as it answers any path of the origin", + attestation.adapter + ); + // The handler answers 404 where there is no section, so a 404 here + // would say a route had been registered for it. + assert_ne!( + attestation.status, 404, + "{} should not answer from the attestation handler", + attestation.adapter + ); + } +} diff --git a/crates/trusted-server-js/lib/build-all.mjs b/crates/trusted-server-js/lib/build-all.mjs index 24f842e764..2a48d7071b 100644 --- a/crates/trusted-server-js/lib/build-all.mjs +++ b/crates/trusted-server-js/lib/build-all.mjs @@ -11,7 +11,7 @@ * The prebid integration builds here as the tsjs shim only — Prebid.js itself * is never bundled into tsjs. Use build-prebid-external.mjs to generate the * pure Prebid.js external bundle (core + adapters + user ID modules) that the - * shim requires at runtime via integrations.prebid.external_bundle_url. + * shim requires at runtime via auction.prebid.external_bundle_url. * * The Rust build script passes `--out-dir` with a private directory under * Cargo's OUT_DIR, so concurrent cargo builds never share (or clean) the same diff --git a/crates/trusted-server-js/lib/build-prebid-external.mjs b/crates/trusted-server-js/lib/build-prebid-external.mjs index a02c5bf60a..7cce1f103b 100644 --- a/crates/trusted-server-js/lib/build-prebid-external.mjs +++ b/crates/trusted-server-js/lib/build-prebid-external.mjs @@ -5,7 +5,7 @@ * * Unlike build-all.mjs, this script is intended to run outside the Cargo build. * It produces an immutable bundle and manifest that can be hosted on an asset - * CDN, then referenced by integrations.prebid.external_bundle_url. + * CDN, then referenced by auction.prebid.external_bundle_url. */ import crypto from 'node:crypto'; @@ -43,19 +43,19 @@ const MODULE_KIND_DEFINITIONS = Object.freeze([ { requestKey: 'bidder', tomlKey: 'bidder', - fieldPath: 'integrations.prebid.bundle.modules.bidder', + fieldPath: 'auction.prebid.bundle.modules.bidder', metadataType: 'bidder', }, { requestKey: 'userId', tomlKey: 'user_id', - fieldPath: 'integrations.prebid.bundle.modules.user_id', + fieldPath: 'auction.prebid.bundle.modules.user_id', metadataType: 'userId', }, { requestKey: 'analytics', tomlKey: 'analytics', - fieldPath: 'integrations.prebid.bundle.modules.analytics', + fieldPath: 'auction.prebid.bundle.modules.analytics', metadataType: 'analytics', }, ]); diff --git a/crates/trusted-server-js/lib/src/core/index.ts b/crates/trusted-server-js/lib/src/core/index.ts index b2c4e41e15..2678a879f1 100644 --- a/crates/trusted-server-js/lib/src/core/index.ts +++ b/crates/trusted-server-js/lib/src/core/index.ts @@ -13,6 +13,7 @@ import { log } from './log'; import { setConfig, getConfig } from './config'; import { requestAds } from './request'; import { installQueue } from './queue'; +import { installPermissions } from './permissions'; const VERSION = '0.1.0'; @@ -42,6 +43,10 @@ api.requestAds = requestAds; // instead of throwing. Injected scripts overwrite these wholesale. api.adSlots ??= []; api.bids ??= {}; +// The edge also injects the request's resolved permission state, either at head +// open (inline mode) or at the seam (shared-template mode). An accessor +// observes the seam's plain assignment so page code can await it either way. +installPermissions(api); // Point global tsjs w.tsjs = api; @@ -68,5 +73,6 @@ log.info('tsjs initialized', { 'addAdUnits', 'renderAdUnit', 'renderAllAdUnits', + 'whenPermissions', ], }); diff --git a/crates/trusted-server-js/lib/src/core/permissions.ts b/crates/trusted-server-js/lib/src/core/permissions.ts new file mode 100644 index 0000000000..6f037b0215 --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/permissions.ts @@ -0,0 +1,89 @@ +// Permission state the edge injects into the page, and the promise page code +// awaits so it can read that state whichever order the injection arrives in. +import { log } from './log'; +import type { PermissionsSnapshot, TsjsApi } from './types'; + +function isSnapshot(value: unknown): value is PermissionsSnapshot { + return typeof value === 'object' && value !== null; +} + +// Page code reads the four lists without checking any exists, so each is an +// array whatever arrived. The edge always sends all four, and a page assigning +// a snapshot by hand, or an older edge, may not. +function normalize(snapshot: PermissionsSnapshot): PermissionsSnapshot { + return { + ...snapshot, + set: Array.isArray(snapshot.set) ? snapshot.set : [], + awaiting: Array.isArray(snapshot.awaiting) ? snapshot.awaiting : [], + signals: Array.isArray(snapshot.signals) ? snapshot.signals : [], + tdls: Array.isArray(snapshot.tdls) ? snapshot.tdls : [], + }; +} + +/** + * Install the `permissions` accessor and `whenPermissions()` on the API object. + * + * The edge injects `window.tsjs.permissions` either before this bundle runs + * (inline mode, at head open) or after it has initialized (shared-template + * mode, as a plain assignment at the `` seam), and on a page with no + * seam it never arrives at all. An accessor observes the plain assignment, so + * the promise resolves in every one of those orders. + */ +export function installPermissions(api: TsjsApi): void { + // A value already on the API object came from the head-open injection, so it + // is the current value; otherwise page code must still read a defined value. + const injected = api.permissions; + let current: PermissionsSnapshot = isSnapshot(injected) + ? normalize(injected) + : { set: [], awaiting: [], signals: [], tdls: [] }; + let settled = false; + let resolvePending: (snapshot: PermissionsSnapshot) => void = () => {}; + const pending = new Promise((resolve) => { + resolvePending = resolve; + }); + + function settle(snapshot: PermissionsSnapshot): void { + if (settled) return; + settled = true; + resolvePending(snapshot); + } + + Object.defineProperty(api, 'permissions', { + get(): PermissionsSnapshot { + return current; + }, + set(value: PermissionsSnapshot) { + current = isSnapshot(value) + ? normalize(value) + : { set: [], awaiting: [], signals: [], tdls: [] }; + log.debug('permissions: received', current); + settle(current); + }, + enumerable: true, + configurable: true, + }); + + if (isSnapshot(injected)) { + log.debug('permissions: present at initialization', injected); + settle(current); + } else if (typeof document !== 'undefined') { + // The seam sits at ``, so anything it was going to assign has run by + // the time the document is parsed. Resolve with whatever the current value + // is rather than leaving page code waiting on a page that has no seam. A + // bundle that initializes after parsing has already missed any seam, so it + // resolves at once instead of waiting for an event that will never fire. + if (document.readyState === 'loading') { + document.addEventListener( + 'DOMContentLoaded', + () => { + settle(current); + }, + { once: true } + ); + } else { + settle(current); + } + } + + api.whenPermissions = () => pending; +} diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 3407d487a0..df563733af 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -27,6 +27,11 @@ export interface AuctionSlot { div_id: string; formats: Array<[number, number]>; targeting?: Record; + /** + * The bands of the core's `CreativeOpportunitySlot::viewport_sizes`, each + * with a `[width, 0]` viewport. Absent when no format names a viewport bound. + */ + viewport_sizes?: Array<[[number, number], Array<[number, number]>]>; } /** Debug-only copy of server-side bid fields exposed for pipeline inspection. */ @@ -365,6 +370,36 @@ export interface GptSlotHandoff { suppressPublisherRefresh: boolean; } +/** + * Permission state the server resolved for this request. + * + * The names in `set` are IAB Privacy Taxonomy Data Use keys, as resolved by the + * server for this request. + * + * `tdls` are the terms documents the data for this request is available under, + * as declared by the permission signal modules the deployment runs, in the + * order they were asked. Each entry addresses a published document a person can + * read. An empty list says no terms were declared, which is not the same as + * terms that permit anything, so page code that needs a basis and finds none + * has none. + */ +export interface PermissionsSnapshot { + set: string[]; + /** + * Permissions still waiting for a signal. Not set, and not refused + * either, which page code must tell apart from `set` being empty. + */ + awaiting: string[]; + /** + * The signals the server's modules read and found valid, each as it + * was received. A signal absent from this list was missing, could not be + * read, or is one the deployment does not act on, and page code relies + * on exactly these and no other. + */ + signals: ValidSignal[]; + tdls: string[]; +} + export type FirstImpressionOwner = 'publisher' | 'trusted_server'; export type FirstImpressionPhase = 'auctioning' | 'delivery_pending' | 'requested' | 'rendered'; @@ -403,6 +438,16 @@ export interface FirstImpressionState { fallbackSlots: Record; } +/** One signal a server-side module read and found valid. */ +export interface ValidSignal { + /** The module that read it, by its configured id. */ + module: string; + /** The scheme it belongs to, for example `tcf` or `gpp`. */ + scheme: string; + /** The signal as received. */ + value: string; +} + export interface TsjsApi { version: string; que: Array<() => void>; @@ -430,6 +475,20 @@ export interface TsjsApi { adSlots?: AuctionSlot[]; /** Winning bid targeting data injected before . */ bids?: Record; + /** + * Permission state resolved by the server for this request, injected at head + * open in inline mode or at the body seam in shared-template mode. + */ + permissions?: PermissionsSnapshot; + /** + * Resolves with the permission state once it arrives. + * + * It resolves straight away when the state was already present at + * initialization, on the first assignment when the body seam makes one, and + * on `DOMContentLoaded` with whatever the current value is when no assignment + * arrives at all. Every later call returns the same resolved promise. + */ + whenPermissions?(): Promise; /** * Bounded client-side Prebid APS renderer capabilities keyed by Prebid's generated * `hb_adid`. The Universal Creative bridge consumes each entry at most once. diff --git a/crates/trusted-server-js/lib/src/integrations/ec_client_fixed/index.ts b/crates/trusted-server-js/lib/src/integrations/ec_client_fixed/index.ts new file mode 100644 index 0000000000..c8dee2ecbc --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ec_client_fixed/index.ts @@ -0,0 +1,102 @@ +// Demonstration client for the client-cycle Edge Cookie module (`client_fixed`). +// +// Client and server share one fixed, known word. When the resolved marker is +// absent, this posts that word to the resolve endpoint. With the `client_fixed` +// module selected, the server verifies the word and, on a match, persists the +// identity-graph row and sets the coded form of the word (cfix~an-ec) as an +// HttpOnly Edge Cookie on the +// response, together with a non-HttpOnly marker cookie. The Edge Cookie itself +// is HttpOnly, so this script can never see it; the marker is what tells it a +// resolve already succeeded, so it does not post again on every page view. +// +// The value is verifiable precisely because it is a known constant, which is the +// point of the demo. It is useless in production, because a fixed value is not an +// identity and every client posts the same word. For demonstration and testing +// only. A real client-cycle module posts and verifies a real payload (for +// example an OWID signature) instead of a shared constant. +import { log } from '../../core/log'; +import { cookieName } from '../../shared/public_prefix'; + +const RESOLVE_ENDPOINT = '/_ts/api/v1/ec/resolve'; + +// The non-HttpOnly companion the server sets alongside the Edge Cookie, named +// after the public prefix the page's bundle tag carries, such as `ts-ecr`. +// Must match COOKIE_EC_RESOLVED in crates/trusted-server-core/src/constants.rs; +// a Rust test asserts the two stay in sync. +const MARKER_COOKIE = 'ecr'; + +// The fixed, known word shared with the server. Must match EXPECTED_VALUE in +// crates/trusted-server-core/src/ec/module.rs; a Rust test asserts the two +// stay in sync. +const FIXED_WORD = 'an-ec'; + +// The permission this module requires, the same declaration the server-side +// module makes in `required_permissions` (crates/trusted-server-core/src/ec/ +// module.rs). A page module is treated like any other module: it declares +// what it requires and checks that against the resolved state the server +// hands the page before it does anything. The server enforces the same gate on +// the resolve endpoint, so this check is the page's half of one decision, not +// a substitute for the server's. A Rust test asserts the two stay in sync. +const REQUIRED_PERMISSION = 'necessary.operations.storage'; + +// Waits for the resolved permission state the edge injects into the page +// (`window.tsjs.permissions`, via `tsjs.whenPermissions()`) and returns +// whether the permission this module requires is set. With no permission state +// on the page there is nothing to check against, so the answer is no. +export async function requiredPermissionIsSet(): Promise { + const whenPermissions = window.tsjs?.whenPermissions; + if (typeof whenPermissions !== 'function') { + log.warn('ec client_fixed: no permission state on the page, not posting'); + return false; + } + const snapshot = await whenPermissions(); + return Array.isArray(snapshot?.set) && snapshot.set.includes(REQUIRED_PERMISSION); +} + +// Returns true when the resolved marker is present in `cookieString`. The Edge +// Cookie itself is HttpOnly and never appears in `document.cookie`, so the +// marker is the only signal the page has. +export function hasResolvedMarker(cookieString: string, marker: string): boolean { + return cookieString.split(';').some((part) => part.trim().startsWith(`${marker}=`)); +} + +// Posts the fixed known word to the resolve endpoint when no resolved marker is +// present and the required permission is set. Returns the word posted, or null +// when nothing was sent or the post failed (a resolve already succeeded, the +// page names no public prefix, the required permission is not set, the +// environment lacks `document`/`fetch`, or the request threw). +export async function resolveEdgeCookie(): Promise { + if (typeof document === 'undefined' || typeof fetch !== 'function') { + return null; + } + const marker = cookieName(MARKER_COOKIE); + if (!marker) { + log.info('ec client_fixed: the page names no public prefix, not posting'); + return null; + } + if (hasResolvedMarker(document.cookie, marker)) { + return null; + } + if (!(await requiredPermissionIsSet())) { + log.info('ec client_fixed: required permission not set, not posting'); + return null; + } + + try { + await fetch(RESOLVE_ENDPOINT, { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'text/plain' }, + body: FIXED_WORD, + }); + log.info('ec client_fixed: posted the known word to the resolve endpoint'); + return FIXED_WORD; + } catch (err) { + log.warn('ec client_fixed: resolve request failed', err); + return null; + } +} + +if (typeof window !== 'undefined') { + void resolveEdgeCookie(); +} diff --git a/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts b/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts index 2072c04032..874748bec8 100644 --- a/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts +++ b/crates/trusted-server-js/lib/src/integrations/google_tag_manager/script_guard.ts @@ -21,7 +21,7 @@ const GTM_URL_PATTERN = /** * Supported paths that the server can proxy. * Must match the route patterns defined in the GoogleTagManagerIntegration handler - * in crates/trusted-server-core/src/integrations/google_tag_manager.rs + * in crates/tag/google-tag-manager/src/lib.rs */ const SUPPORTED_PATHS = ['/gtm.js', '/gtag/js', '/gtag.js', '/collect', '/g/collect']; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index fc18beb5e9..04867d1813 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -85,6 +85,7 @@ interface GoogleTagSlot { clearTargeting?(key?: string): GoogleTagSlot; addService(service: GoogleTagPubAdsService): GoogleTagSlot; getTargeting?(key: string): string[]; + defineSizeMapping?(mapping: unknown): GoogleTagSlot; } interface SlotRenderEndedEvent { @@ -370,9 +371,15 @@ interface GoogleTag { display(target: GoogleTagDisplayTarget): void; setConfig?(config: GoogleTagConfig): void; getConfig?(keys: string | string[]): GoogleTagEffectiveConfig | undefined; + sizeMapping?(): ViewportSizeBuilder; _loaded_?: boolean; } +interface ViewportSizeBuilder { + addSize(viewport: number[], sizes: Array): ViewportSizeBuilder; + build(): unknown; +} + type GptWindow = Window & { googletag?: Partial; __tsjs_slim_prebid_url?: string; @@ -848,6 +855,27 @@ function displayTargetElementId(target: GoogleTagDisplayTarget): string | undefi return (target as Element).id || undefined; } +/** Defines the slot and applies its viewport bands as a size mapping. */ +function defineWithViewportSizes( + api: { + defineSlot?(adUnitPath: string, size: Array, elementId?: string): S | null; + sizeMapping?(): ViewportSizeBuilder; + }, + slot: AuctionSlot, + elementId: string +): S | null | undefined { + const definedSlot = api.defineSlot?.(slot.gam_unit_path, slot.formats, elementId); + const bands = slot.viewport_sizes; + if (definedSlot && bands?.length && api.sizeMapping && definedSlot.defineSizeMapping) { + const builder = api.sizeMapping(); + for (const [viewport, sizes] of bands) { + builder.addSize(viewport, sizes); + } + definedSlot.defineSizeMapping(builder.build()); + } + return definedSlot; +} + function normalizedGptFormats(formats: Array): Array { return formats.length === 2 && formats.every((format) => typeof format === 'number') ? [formats as number[]] @@ -1155,9 +1183,8 @@ function schedulePublisherFirstImpressionFallback( let tsOwned = false; if (!gptSlot) { gptSlot = - withGptSlotHandoffInternal(ts, () => - g.defineSlot?.(slot.gam_unit_path, slot.formats, element.id) - ) ?? undefined; + withGptSlotHandoffInternal(ts, () => defineWithViewportSizes(g, slot, element.id)) ?? + undefined; if (!gptSlot) { releaseTrustedServerFirstImpressionClaim(ts, element, claim); return; @@ -1352,7 +1379,7 @@ export function installTsAdInit(): void { // defineSlot() for this div is handed the same slot by the scoped GPT // wrapper, preventing a competing container-slot request. const defined = withGptSlotHandoffInternal(ts, () => - g.defineSlot?.(slot.gam_unit_path, slot.formats, actualDivId) + defineWithViewportSizes(g, slot, actualDivId) ); if (!defined) { releaseTrustedServerFirstImpressionClaim(ts, el, firstImpression); @@ -1499,10 +1526,10 @@ const PAGE_BIDS_PATH = '/_ts/page-bids'; * Deprecated alias of {@link PAGE_BIDS_PATH}, kept registered server-side so * pre-rename bundles keep working. This bundle falls back to it when the * canonical path does not serve page-bids: a server rolled back to before the - * rename does not register the canonical path, and an operator `[[handlers]]` - * auth regex broad enough to cover `/_ts` answers it with `401` that no - * anonymous browser fetch can satisfy. Without the fallback either case - * silently drops ads on every SPA navigation. + * rename does not register the canonical path, and a gate in front of the + * service that covers `/_ts` answers it with `401` that no anonymous browser + * fetch can satisfy. Without the fallback either case silently drops ads on + * every SPA navigation. * * Removed together with the server-side alias in IABTechLab/trusted-server#970. */ @@ -1547,7 +1574,7 @@ async function fetchPageBids( signal, }); if (!res.ok) { - // 401: an operator auth handler regex covers this path. 404: this server + // 401: a gate in front of the service covers this path. 404: this server // does not know the route. Either way the other path may still answer. // 403 (cross-site gate) and 5xx would repeat on both, so they do not. return { data: null, wrongEndpoint: res.status === 401 || res.status === 404 }; diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 82c0e8c6b7..1f1680a3ff 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -23,6 +23,7 @@ import { resolveFirstImpressionElement, } from '../../core/first_impression'; import { log } from '../../core/log'; +import { cookieName } from '../../shared/public_prefix'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import { registerApsPrebidRenderer, validateApsRenderer } from '../aps/render'; import type { AuctionBid, AuctionEid } from '../../core/auction'; @@ -45,9 +46,9 @@ type PbjsGlobal = typeof _pbjsDefault & { }; // Prebid.js itself is NOT bundled into this module. It is served as the -// external bundle configured via `integrations.prebid.external_bundle_url` -// (required whenever the prebid integration is enabled) and owns the -// `window.pbjs` global. The Rust head injector emits a stub +// external bundle configured via `auction.prebid.external_bundle_url` +// (required whenever the prebid integration runs) and owns the +// `window.pbjs` global. The Rust module's middleware emits a stub // (`window.pbjs = window.pbjs || {que:[],cmd:[]}`) before any script runs and // Prebid.js installs its API onto that same object, so capturing the reference // at module scope is safe regardless of evaluation order. @@ -179,7 +180,7 @@ export interface PrebidNpmConfig { /** * Shape of the server-injected config at `window.__tsjs_prebid`. - * Set by the Rust IntegrationHeadInjector from trusted-server.toml values. + * Set by the Rust module's middleware from trusted-server.toml values. */ interface InjectedPrebidConfig { accountId?: string; @@ -1133,11 +1134,28 @@ function bannerSizesFromGptSlot(slot: RefreshGptSlot): BannerSize[] | undefined return parsedSizes.length > 0 ? parsedSizes : undefined; } +/** + * The sizes the slot's viewport band asks for at the current viewport width, + * or `undefined` when the slot names no band. See `AuctionSlot.viewport_sizes`. + */ +function injectedSlotBandSizes(slot: AuctionSlot | undefined): Array<[number, number]> | undefined { + const bands = slot?.viewport_sizes; + if (!bands?.length) return undefined; + const width = window.innerWidth; + return bands.find(([[minimumWidth]]) => width >= minimumWidth)?.[1] ?? []; +} + function bannerSizesFromInjectedSlot(slot: AuctionSlot | undefined): BannerSize[] | undefined { - const parsedSizes = slot?.formats?.map(parseBannerSize).filter(isDefined) ?? []; + const sizes = injectedSlotBandSizes(slot) ?? slot?.formats; + const parsedSizes = sizes?.map(parseBannerSize).filter(isDefined) ?? []; return parsedSizes.length > 0 ? parsedSizes : undefined; } +/** Whether the slot's viewport band asks for no size at the current width. */ +function asksForNoSizeAtThisWidth(slot: RefreshGptSlot): boolean { + return injectedSlotBandSizes(findInjectedSlotForRefresh(slot))?.length === 0; +} + function refreshSlotElementId(slot: RefreshGptSlot): string | undefined { const elementId = slot.getSlotElementId?.(); return elementId && elementId.length > 0 ? elementId : undefined; @@ -1988,7 +2006,7 @@ function installApsBidResponseRegistry(): void { export function installPrebidNpm(config?: Partial): typeof pbjs { // The prebid integration requires the external Prebid.js bundle - // (integrations.prebid.external_bundle_url). When it failed to load (network + // (auction.prebid.external_bundle_url). When it failed to load (network // error, SRI mismatch) window.pbjs is still the head-injected stub with no // API — installing the adapter is impossible, so bail out loudly. if (!hasPrebidJsApi()) { @@ -2359,8 +2377,9 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const requestScopedBidRequests = [...validBidRequests]; const hasUserIdApi = typeof pbjs.getUserIdsAsEids === 'function'; const auctionEids = collectAuctionEids(); - if (hasUserIdApi && !auctionEids) { - clearPrebidEidsCookie(); + const eidsCookie = cookieName(EID_COOKIE); + if (hasUserIdApi && !auctionEids && eidsCookie) { + clearPrebidEidsCookie(eidsCookie); } const payload = buildAdRequest(validBidRequests, { eids: auctionEids }); return { @@ -2609,7 +2628,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs log.error( `[tsjs-prebid] client-side bidder "${bidder}" has no adapter in the external ` + 'Prebid bundle. Add its exact Prebid module stem to ' + - '[integrations.prebid.bundle.modules].bidder in trusted-server.toml and ' + + '[auction.prebid.bundle.modules].bidder in trusted-server.toml and ' + 'rebuild it with `ts prebid client`.' ); } @@ -2699,7 +2718,9 @@ export function installRefreshHandler(timeoutMs = 1500): void { getInjectedConfig()?.excludedGamAdUnitPathSuffixes ); const auctionSlots = independentSlots.filter( - (slot) => !isExcludedFromRefreshAuction(slot, excludedGamAdUnitPathSuffixes) + (slot) => + !isExcludedFromRefreshAuction(slot, excludedGamAdUnitPathSuffixes) && + !asksForNoSizeAtThisWidth(slot) ); if (!auctionSlots.length) { const immediateSlotCodes = new Map(); @@ -2904,15 +2925,16 @@ export function installUserIdModules(): void { /** Maximum cookie payload size in bytes (leave room for other cookies). */ const MAX_EID_COOKIE_BYTES = 3072; -/** Cookie name for persisted Prebid EIDs. */ -const EID_COOKIE_NAME = 'ts-eids'; +/** The cookie persisted Prebid EIDs are written in, named after the page's + * public prefix, such as `ts-eids`. */ +const EID_COOKIE = 'eids'; /** Cookie max-age in seconds (1 day). */ const EID_COOKIE_MAX_AGE = 86400; -/** Clears any previously persisted Prebid EIDs cookie. */ -function clearPrebidEidsCookie(): void { - document.cookie = `${EID_COOKIE_NAME}=; Path=/; Secure; SameSite=Lax; Max-Age=0`; +/** Clears any previously persisted Prebid EIDs cookie, named `name`. */ +function clearPrebidEidsCookie(name: string): void { + document.cookie = `${name}=; Path=/; Secure; SameSite=Lax; Max-Age=0`; } /** @@ -2976,31 +2998,37 @@ function logTrimmedEidSources(droppedSources: Set, trimmedSources: Set-eids`, such as + * `ts-eids`) for backend ingestion and auction fallback on later requests. A + * page naming no public prefix writes no cookie. */ function syncPrebidEidsCookie(): void { + const name = cookieName(EID_COOKIE); + if (!name) { + log.debug('the page names no public prefix, so the EIDs cookie is not written'); + return; + } try { if (typeof pbjs.getUserIdsAsEids !== 'function') { // Without Prebid EIDs to forward, stale auction fallback IDs must not persist. - clearPrebidEidsCookie(); + clearPrebidEidsCookie(name); return; } const eids = collectAuctionEids(); if (!eids) { - clearPrebidEidsCookie(); + clearPrebidEidsCookie(name); return; } const payload = fitAuctionEidsToCookie(eids); if (!payload) { - clearPrebidEidsCookie(); + clearPrebidEidsCookie(name); return; } const encoded = btoa(JSON.stringify(payload)); - document.cookie = `${EID_COOKIE_NAME}=${encoded}; Path=/; Secure; SameSite=Lax; Max-Age=${EID_COOKIE_MAX_AGE}`; + document.cookie = `${name}=${encoded}; Path=/; Secure; SameSite=Lax; Max-Age=${EID_COOKIE_MAX_AGE}`; log.debug(`[tsjs-prebid] synced ${payload.length} EID sources to cookie`); } catch (err) { diff --git a/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts b/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts index 3e1493bdbb..dd4624aa77 100644 --- a/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts +++ b/crates/trusted-server-js/lib/src/integrations/sourcepoint/script_guard.ts @@ -12,7 +12,7 @@ function normalizeSourcepointUrl(url: string): string | null { if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed; // Keep in sync with Rust `parse_sourcepoint_url()` in: - // crates/trusted-server-core/src/integrations/sourcepoint.rs + // crates/cmp/sourcepoint/src/lib.rs // (protocol-relative + bare-host handling + hostname-only host checks). // Bare domain or path — attempt to parse as https URL. // The hostname === check in isSourcepointUrl rejects non-matching domains. diff --git a/crates/trusted-server-js/lib/src/shared/public_prefix.ts b/crates/trusted-server-js/lib/src/shared/public_prefix.ts new file mode 100644 index 0000000000..2dba302c3d --- /dev/null +++ b/crates/trusted-server-js/lib/src/shared/public_prefix.ts @@ -0,0 +1,36 @@ +// The deployment's public prefix, which every cookie name this script writes +// starts with, such as `ts` for `ts-eids`. +// +// The server writes it on the bundle's own tag, so the script holds no copy of +// its own. A page whose bundle tag carries none has no names to write, and a +// module asking for one is told `null` and writes nothing. + +/** The `id` the server gives the bundle's `"#, - ctx.request_host - )] + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let origin = context.origin_host.to_owned(); + let host = context.request_host.to_owned(); + MiddlewareAction { + element_handlers: vec![Box::new(AttributeRewrite::new( + "data-link", + Rc::new(move |matched| { + if matched.value.contains(&origin) { + AttributeRewriteAction::replace(matched.value.replace(&origin, &host)) + } else { + AttributeRewriteAction::keep() + } + }), + ))], + ..MiddlewareAction::pass() + } } } ``` -**Behavior**: - -- Snippets are prepended into `` after the TSJS bundle tag -- Called once per HTML response -- Multiple integrations can each contribute snippets -- If no snippets are returned, no extra markup is added - -See [Integration Guide](/guide/integration-guide) for creating custom rewriters. +See [Changing a page](/guide/integration-guide#changing-a-page) for what a +middleware can rely on, and +[Placing page changes](/guide/configuration#placing-page-changes) for the +entries that place one. ## TSJS Injection @@ -800,7 +781,8 @@ The Trusted Server JavaScript (TSJS) library is automatically injected: ### Integration bundles -The integration registry selects TSJS modules by enabled integration ID. A +The integration registry selects TSJS modules by the integrations the +settings select, each known to the browser by its integration ID. A registered ID with a compiled module is included in the immediate unified bundle unless its builder calls `.with_deferred_js()`; deferred modules are served separately as `/static/tsjs=tsjs-.min.js`. Builders can call @@ -819,7 +801,7 @@ Each integration is built as a separate IIFE at compile time (into Cargo's `OUT_ - Prebid is built externally with `build-prebid-external.mjs` and served through `/integrations/prebid/bundle.js` - `tsjs-lockr.js`, `tsjs-permutive.js`, `tsjs-didomi.js`, `tsjs-datadome.js`, `tsjs-testlight.js` — Other integrations -At runtime, the server concatenates `tsjs-core.js` + the modules for enabled integrations. The URL stays `/static/tsjs=tsjs-unified.min.js?v=` for backward compatibility. +At runtime, the server concatenates `tsjs-core.js` + the modules for the integrations that run. The URL stays `/static/tsjs=tsjs-unified.min.js?v=` for backward compatibility. ## Performance Optimization diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index cccb04daee..87122b4fac 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -25,9 +25,12 @@ The `replace-with-*` values below are intentionally rejected placeholders: ```toml [ec] -passphrase = "ec_passphrase" +module = "hmac" ec_store = "ec_identity_store" +[ec.hmac] +passphrase = "ec_passphrase" + [[ec.partners]] name = "Mocktioneer SSP" source_domain = "formally-vital-lion.edgecompute.app" @@ -40,10 +43,11 @@ secret store, not the credential values. This workflow calls the inbound identify and batch-sync APIs, so its partner needs `api_token`. Partners that do not call either API may omit it. Provision high-entropy values under `ec_passphrase` and `partner_api_token`; see -[Configuration](/guide/configuration#secret-store-migration). +[Configuration](/guide/configuration#static-secret-references). Required behavior assumptions: +- `module = "hmac"` selects the built-in HMAC module, and its `passphrase` lives under `[ec.hmac]` - The value stored under `ec_passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters - `ec_store` is linked to the active Fastly service version - `ec_store` is the only KV-backed EC lifecycle store; it contains identity graph state, minimal consent metadata, source-domain keyed partner UIDs, and withdrawal tombstones @@ -99,13 +103,13 @@ curl -si "${TS_BASE_URL}/" \ Look for: -- `Set-Cookie: ts-ec=<64hex.6chars>` +- `Set-Cookie: ts-ec=hmac~<64hex.6chars>` (the `hmac~` prefix is the module code, and a bare value with no prefix, as a browser may already hold, still resolves) ## 5) Batch Sync (S2S) Endpoint: `POST /_ts/api/v1/batch-sync` -Important: request field is `ec_id` (full `{64hex}.{6alnum}` value). The `timestamp` field remains required for API compatibility, but it no longer orders writes because EC identity entries do not store per-partner sync timestamps. Within one request, valid mappings are grouped by normalized EC ID and the last valid UID for each group is applied once; unchanged UIDs are accepted without a write. Group outcomes are reported for every original mapping, and infrastructure failures abort the remaining groups. See the [API Reference](/guide/api-reference) for the complete accounting and failure contract. +Important: request field is `ec_id` (the full value as issued, `hmac~{64hex}.{6alnum}`, with the bare form that has no prefix also accepted). The `timestamp` field remains required for API compatibility, but it no longer orders writes because EC identity entries do not store per-partner sync timestamps. Within one request, valid mappings are grouped by normalized EC ID and the last valid UID for each group is applied once; unchanged UIDs are accepted without a write. Group outcomes are reported for every original mapping, and infrastructure failures abort the remaining groups. See the [API Reference](/guide/api-reference) for the complete accounting and failure contract. ```bash BATCH_UID="${PARTNER_UID}-batch" diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..8204b793a4 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -8,12 +8,9 @@ permit EC use. ## Policy Posture -Trusted Server is technology. It is neutral on policy. The Edge Cookie -gives the deployer a cookie slot and configuration over the surrounding -attributes. The deployer determines the policy posture based on the -laws and contractual arrangements that apply to their deployment. -Privacy outcomes follow from that configuration, not from the cookie -mechanism itself. +Trusted Server is technology. It is neutral on policy. The Edge Cookie gives the deployer a cookie slot and configuration over the surrounding attributes. The deployer determines the policy posture based on the laws and contractual arrangements that apply to their deployment. Privacy outcomes follow from that configuration, not from the cookie mechanism itself. + +An Edge Cookie (EC) is a first-party identifier that the built-in module derives on a first site visit with an HMAC of the client IP address plus a short random suffix, created only when the permission model allows it. It is passed in requests on subsequent visits and activity. Trusted Server surfaces the current EC ID via response headers and a first-party cookie. For the exact header and cookie names, see the [API Reference](/guide/api-reference). For full operational onboarding (partner configuration, batch sync, identify, and auction verification), use the [EC Setup Guide](/guide/ec-setup-guide). @@ -42,7 +39,7 @@ maps to a stable base. ### Request Lifecycle -Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. +Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction). Read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds the consent context from request-local cookies, headers, geolocation, and policy defaults. It does not load consent from a separate KV store. ```mermaid sequenceDiagram @@ -60,7 +57,7 @@ sequenceDiagram Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies TS-->>B: Response + Set-Cookie: ts-ec=... else Return Visit (EC cookie present) - Note over TS: Phase 2: Routing
EC exists — skip generation + Note over TS: Phase 2: Routing
EC exists, skip generation Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies TS-->>B: Response
(no cookie refresh) end @@ -70,11 +67,11 @@ sequenceDiagram ### Response Finalization -After routing completes, the server evaluates consent state and cookie presence to decide what to do with the EC cookie on the response. +After routing completes, the server evaluates the permission state and cookie presence to decide what to do with the EC cookie on the response. ```mermaid flowchart TD - Start[ec_finalize_response] --> ConsentCheck{Consent
allows EC?} + Start[ec_finalize_response] --> ConsentCheck{Permissions
allow EC?} ConsentCheck -- "No" --> ExplicitWithdrawal{Explicit
withdrawal?} ExplicitWithdrawal -- "Yes" --> CookiePresent{Cookie was
present?} @@ -87,44 +84,71 @@ flowchart TD WasPresent -- "No, just generated" --> NewEc["Ingest Prebid EID cookies
Set ts-ec cookie"] ``` -When consent cannot be verified for the current request — for example, unknown jurisdiction or missing/undecodable consent signals in a regulated region — Trusted Server fails closed for EC use by stripping EC headers, but it does **not** treat that as authoritative revocation of an already-issued EC. +When the required permissions are not set for the current request (for example a location lookup that failed, or a baseline that requires a signal no module has answered), Trusted Server fails closed for EC use by stripping EC headers, but it does **not** treat that as authoritative revocation of an already-issued EC. + +## Permission Gating + +EC creation is gated through the [permission model](/guide/permission-model), not by a jurisdiction rule baked into the core. The Edge Cookie module advertises the permissions its data use requires, and Trusted Server creates an Edge Cookie only when every required permission is set. The built-in HMAC module requires `necessary.operations.storage` (TCF Purpose 1), because the `Set-Cookie` operation stores information on the device. + +The Edge Cookie code never reads consent. It checks only whether the required **permission** is set. Consent is one of the sources that _set_ a permission, not something the gate reads directly, so the Edge Cookie logic does not change when a consent framework changes. Two sources combine for each request: + +- **A country and region baseline.** The country, and an optional region such as a US state, that the geo module returns. A region rule takes precedence over its country, and when no country is identified, or neither the region nor the country has a rule, the baseline on the top node of the rules tree applies. +- **Consent and privacy signals.** TCF, GPP, a US Privacy string, GPC and the Model Terms for Marketing preference (`euconsent-v2`, `__gpp` / `__gpp_sid`, `us_privacy`, `Sec-GPC`, `__mtm_pref`), each read by its own [permission signal module](/guide/permission-signals) and mapped onto permissions as a **grant or a revoke** on top of that baseline, in the configured order. There is no separate consent KV fallback. + +Every permission is resolved this way, and the Edge Cookie is created only when each permission its module requires is set. A location lookup that fails resolves at the requires-signal floor, so the cookie is not created unless a signal grants the permission. The core encodes no jurisdiction's law. The deployer brings the policy, and the per-country and per-region rules live in the policy file rather than in core logic. See the [permission model](/guide/permission-model) for the full list of permission sources and the resolution order. Writing browser-supplied Prebid EIDs into the identity graph, and server-side pull sync, additionally need the storage and personalised-ad permissions both set, the pair that also gates EIDs in the bid request. With storage alone the Edge Cookie stays first-party and partner sharing is withheld, so the EC ID is not sent to pull-sync partners. See [Prebid EID Flow](#prebid-eid-flow). + +```mermaid +flowchart TD + Start[Resolve country and region] --> Baseline[Country or region rule,
else the top of the rules tree] + Baseline --> Signals[Apply consent/privacy signals
as a grant or revoke] + Signals --> Check{Module's required
permissions all set?} + Check -- "Yes" --> Allow([Create EC]) + Check -- "No" --> Deny([No EC]) +``` + +The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, source-domain keyed partner UIDs, a minimal consent snapshot used for EC entry metadata, withdrawal tombstones, and completion markers that prevent stale point-read misses from rewriting completed tombstones. A marker key records the original tombstone's validity bound and is ignored at or after that time, even if its KV row has not expired yet. If the clock is unusable, the marker is ignored and withdrawal falls back to a strong root-existence check; only a confirmed existing root can be written. This prevents a stale marker from suppressing withdrawal when an expired EC key is created again. With a healthy store, repeated withdrawal of an already tombstoned EC ID leaves the row unchanged instead of refreshing its 24-hour TTL. Permission resolution for each request is based on the live request signals listed above. + +## Module Types: Server-Side and Client-Side -## Consent Model +The Edge Cookie identifier comes from a configurable module, selected by `[ec] module`. A module is one of two types, and the permission gate above applies to both. The two reach the **same outcome** (a `ts-ec` cookie set and carried on every later request) by **different routes**. -EC creation is gated by jurisdiction. The server detects jurisdiction from geolocation data attached to the request and applies the corresponding consent rules. Live consent comes from request-local signals (`euconsent-v2`, `__gpp`, `__gpp_sid`, `us_privacy`, `Sec-GPC`) plus geolocation and policy defaults; there is no separate consent KV fallback. +- **Server-side** (for example the built-in HMAC module, or the built-in `host_signals` module that derives an identifier from the host's TLS JA4 and HTTP/2 signals on a host that supplies them). The module derives the identifier at the edge from request data in `generate()`, and the **page response** sets the cookie. Nothing client-side is involved. +- **Client-side** (for example the `client_fixed` demonstration module). The module cannot derive the identifier at the edge, so `generate()` defers and returns no identifier. The page then runs the module's own JavaScript in the browser, which does its work and posts the result to the resolve endpoint. The module derives an identifier from that value in `resolve_from_client()`, and the **resolve response** sets the cookie. ```mermaid flowchart TD - Start[Detect Jurisdiction] --> J{Jurisdiction?} - - J -- "GDPR
(EU/UK)" --> TCF{TCF string
present?} - TCF -- "Yes" --> P1{Purpose 1
granted?} - P1 -- "Yes" --> Allow([Allow EC]) - P1 -- "No" --> Deny([Deny EC]) - TCF -- "No" --> Deny - - J -- "US State" --> GPC{GPC header
set?} - GPC -- "Yes" --> Deny - GPC -- "No" --> USTCF{TCF from CMP
e.g. Didomi?} - USTCF -- "Yes" --> USP1{Purpose 1
granted?} - USP1 -- "Yes" --> Allow - USP1 -- "No" --> Deny - USTCF -- "No" --> USP{US Privacy
string?} - USP -- "Yes" --> OptOut{Opt-out
sale?} - OptOut -- "No" --> Allow - OptOut -- "Yes" --> Deny - USP -- "No" --> Deny - - J -- "Non-regulated" --> Allow - J -- "Unknown
(no geo data)" --> Deny + Start(["Page request, no Edge Cookie"]) --> Type{"Module type"} + + Type -->|"Server-side (e.g. HMAC)"| SGen["generate() derives at the edge from request data"] + SGen --> SSet["Page response sets ts-ec"] + + Type -->|"Client-side (e.g. client_fixed)"| CDefer["generate() defers, returns no identifier"] + CDefer --> CPage["Page response, no cookie, delivers the module JS"] + CPage --> CBox[["Module JS (black box): runs in the browser and does its work"]] + CBox --> CPost["JS posts the result to POST /_ts/api/v1/ec/resolve"] + CPost --> CResolve["resolve_from_client() verifies and derives"] + CResolve --> CSet["Resolve response sets ts-ec"] + + SSet --> Same(["Same outcome: ts-ec set, carried on every later request"]) + CSet --> Same ``` -- **GDPR**: Opt-in required. TCF Purpose 1 (store/access device) must be explicitly consented. Writing browser-supplied Prebid EIDs into the identity graph additionally requires TCF Purpose 4 (personalized ads); see [Prebid EID Flow](#prebid-eid-flow). Without Purpose 4, server-side pull sync is also skipped, so the EC ID is not sent to pull-sync partners. -- **US State**: Opt-out model with three-tier fallback — GPC always blocks, then TCF if a CMP uses it, then US Privacy string, then fail-closed. -- **Non-regulated**: EC always allowed. -- **Unknown**: Fail-closed when jurisdiction cannot be determined. +The two types differ only in route and in the methods they use: + +| Feature | Server-side | Client-side | +| -------------------- | ------------------------- | ---------------------------------------------- | +| Example | HMAC (`hmac`) | `client_fixed` (demonstration) | +| Created in | `generate()`, at the edge | `resolve_from_client()`, from the posted value | +| `generate()` returns | the identifier | no identifier (defers) | +| Client JavaScript | none | the module JS (black box), which posts back | +| Endpoint | none | `POST /_ts/api/v1/ec/resolve` | +| Cookie set on | the page response | the resolve response | + +The resolve endpoint requires an `Origin` on the publisher's domain and a `text/plain` or `application/json` body, and it answers `409` rather than silently replacing an identity the request already carries. A created identifier is persisted to the identity graph before the cookie is set, so withdrawal reaches a client-set identity the same way it reaches an edge-created one. On success the cookie is set on the endpoint's own first-party `200` response, so the value is live for every subsequent request without a second navigation. The cookie is `HttpOnly`, so the page script never reads it back, and a non-`HttpOnly` marker cookie (`ts-ecr=1`, carrying no identity) tells the script a resolve succeeded so it does not post again on every page view. Every resolve response carries `Cache-Control: no-store`. The `client_fixed` demonstration module is compiled only behind the `client-fixed-demo` cargo feature, so a production build rejects selecting it at startup. + +Because the posted value comes from the browser, **verification is the module's responsibility**. A client-side module must verify the payload (for example a signature) before creating an identifier, or a client could forge an Edge Cookie. The endpoint itself is module-agnostic. It bounds the body, applies the same permission gate as organic generation, calls the module, and writes the cookie. -The `ec_identity_store` KV store is the only EC lifecycle store. It holds identity graph state, source-domain keyed partner UIDs, a minimal consent snapshot used for EC entry metadata, withdrawal tombstones, and completion markers that prevent stale point-read misses from rewriting completed tombstones. A marker key records the original tombstone's validity bound and is ignored at or after that time, even if its KV row has not expired yet. If the clock is unusable, the marker is ignored and withdrawal falls back to a strong root-existence check; only a confirmed existing root can be written. This prevents a stale marker from suppressing withdrawal when an expired EC key is created again. With a healthy store, repeated withdrawal of an already tombstoned EC ID leaves the row unchanged instead of refreshing its 24-hour TTL. Consent interpretation for each request remains based on the live request signals listed above. +A built-in `client_fixed` module demonstrates the client-side type end to end with no vendor coupling. Client and server share one fixed, known word. When no Edge Cookie is present, the page script (shipped in the tsjs bundle when that module is selected) posts that word, the server verifies it matches, and on a match sets it as the Edge Cookie. The value is verifiable because it is a known constant, which is the point of the demo. It is useless in production, because a fixed value is not an identity, so it is for demonstration and testing only. ## Partner Sync Channels @@ -156,7 +180,7 @@ Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finali The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. -These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. +These Prebid EID writes follow the same permission rule as bidstream EID forwarding: the storage and personalised-ad permissions must both be set. A reader who allows storage but not personalised ads keeps their EC, but no EIDs from the request are persisted. ```mermaid sequenceDiagram @@ -176,7 +200,7 @@ sequenceDiagram Note over B,TS: Next eligible EID-sync request B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners - TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) + TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires the storage and personalised-ad permissions) ``` Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shape in `ts-eids`. The server remains backward-compatible with earlier flattened `{source, id, atype}` cookies during rollout, but new cookies use the structured `uids[]` form. @@ -225,7 +249,7 @@ sequenceDiagram DSP-->>PS: OpenRTB bid response PS-->>TS: OpenRTB seatbid response TS-->>B: Auction response + x-ts-eids header when available - TS->>KV: Add missing partner UIDs from request eids[]
+ ts-eids/sharedId cookies (TCF Purpose 1 + 4 under GDPR) + TS->>KV: Add missing partner UIDs from request eids[]
+ ts-eids/sharedId cookies (storage and personalised-ad permissions) ``` The relevant OpenRTB structure forwarded to Prebid Server and downstream partners is: @@ -310,7 +334,7 @@ sets `Path=/`, `Secure`, `HttpOnly`, `SameSite=Lax`, and a `Max-Age`. - Returning requests with consent and an existing `ts-ec` do not refresh the EC cookie or KV TTL. - Newly generated ECs receive `Set-Cookie: ts-ec=...`. -- When consent is blocked but not explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. +- When the permission is not set but nothing was explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. Withdrawal is deliberately narrow: a TCF record refusing storage in a jurisdiction whose baseline did not grant it. US-style opt-outs (GPC, a GPP sale opt-out, or a US Privacy opt-out) suppress use for the request but never expire the cookie or write a tombstone, so lifting the opt-out restores the identity. - `/_ts/api/v1/identify` is read-oriented and returns identity enrichment for the authenticated partner. It computes `cluster_size` only when the EC entry does not already store one. - `/_ts/api/v1/batch-sync` validates every input, groups valid mappings by normalized EC ID, and applies the last valid UID for each group once. Group outcomes still account for every original input; infrastructure failures reject the failing and remaining groups. Mapping timestamps remain required for API compatibility but do not order writes. See the [API Reference](/guide/api-reference) for the complete contract. - Pull sync fills missing partner UIDs only. Existing partner UIDs are not periodically refreshed because EC entries no longer store per-partner sync timestamps. diff --git a/docs/guide/edgezero.md b/docs/guide/edgezero.md index ed9c4f2f2c..7c7e4bc2eb 100644 --- a/docs/guide/edgezero.md +++ b/docs/guide/edgezero.md @@ -69,8 +69,9 @@ store before deletion, especially when using `--store` or `--no-env`. 4. Run `ts config push --adapter ADAPTER`. 5. Start or deploy the adapter and verify a non-health publisher route. A healthy endpoint alone may not prove that application configuration loaded. - Confirm that its `x-ts-version` response header reports the release you - deployed. + Confirm which build is serving, from the attestation document's `build` + fields, or from the `x-ts-version` header on a page where a `[[serve]]` + entry names `service_version`. See [Configuration](/guide/configuration) for the field contract, the [CLI guide](/guide/cli) for exact command help, and diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index c8d48be43d..67c25e0e57 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -75,7 +75,7 @@ TOML file. - `publisher.domain` - `publisher.origin_url` - `publisher.proxy_secret` -- `ec.passphrase` +- `ec.hmac.passphrase` (when `ec.module = "hmac"`) --- @@ -94,15 +94,13 @@ credentials or fragment: ```toml # ❌ Wrong -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "prebid.example.com/openrtb2/auction" # ✅ Correct -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" ``` @@ -118,13 +116,12 @@ Failed to parse environment variable: TRUSTED_SERVER__PUBLISHER__DOMAIN **Cause:** Environment variable format doesn't match expected type -**Solution:** Override an existing scalar leaf with the expected type. Provider -map keys preserve hyphens, so shell users must invoke the CLI through `env`: +**Solution:** Override an existing scalar leaf with the expected type. Demand +source names are snake_case, so each maps straight onto a path segment: ```bash env 'TRUSTED_SERVER__PUBLISHER__DOMAIN=example.com' \ - 'TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__TIMEOUT_MS=1000' \ - 'TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=true' \ + 'TRUSTED_SERVER__DEMAND__PBS_MAIN__TIMEOUT_MS=1000' \ ts config validate ``` @@ -149,10 +146,13 @@ Failed to generate EC ID: HMAC error **Solution:** -1. Ensure `passphrase` names a secret-store entry in `trusted-server.toml`: +1. Ensure the `hmac` module is selected and its `passphrase` names a secret-store entry in `trusted-server.toml`: ```toml [ec] +module = "hmac" + +[ec.hmac] passphrase = "ec_passphrase" ``` @@ -160,7 +160,8 @@ passphrase = "ec_passphrase" passphrase value: ```bash -TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase +TRUSTED_SERVER__EC__MODULE=hmac +TRUSTED_SERVER__EC__HMAC__PASSPHRASE=ec_passphrase ``` 3. Provision a high-entropy value of at least 32 characters under @@ -176,9 +177,9 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase Backend not found: prebid-server ``` -**Cause:** Dynamic backend creation for a configured provider endpoint failed. -Provider backends are derived from `[auction.providers.]`; they are not -manually named static Fastly backends. +**Cause:** Dynamic backend creation for a configured demand endpoint failed. +Demand backends are derived from `[demand.]`; they are not manually +named static Fastly backends. **Solution:** @@ -227,14 +228,14 @@ Upstream request timeout after 1000ms **Solution:** -1. Increase the affected server provider timeout: +1. Increase the affected demand source's timeout: ```toml -[auction.providers.pbs-main] +[demand.pbs_main] timeout_ms = 2000 ``` -Browser `[integrations.prebid].timeout_ms` is independent and does not control +Browser `[auction.prebid].timeout_ms` is independent and does not control Prebid Server transport. 2. Verify upstream service is responsive: @@ -289,14 +290,14 @@ Prebid Server returned 400: Invalid OpenRTB request **Solution:** -1. Enable debug mode on the Prebid Server profile: +1. Enable debug mode on the Prebid Server demand source: ```toml -[auction.providers.pbs-main] -profile_config = { debug = true } +[demand.pbs_main] +debug = true ``` -`[integrations.prebid].debug` controls browser Prebid.js only. +`[auction.prebid].debug` controls browser Prebid.js only. 2. Check logs for request/response details 3. Verify bidders are supported by your Prebid Server @@ -331,8 +332,10 @@ Next.js links still pointing to origin domain 2. Update `rewrite_attributes` to match actual keys: ```toml -[integrations.nextjs] -enabled = true +[framework] +module = "nextjs" + +[framework.nextjs] rewrite_attributes = ["href", "link", "url", "src"] # Add keys you find ``` @@ -360,7 +363,10 @@ Failed to fetch Permutive SDK: 404 Not Found 2. Update configuration: ```toml -[integrations.permutive] +[audience] +module = "permutive" + +[audience.permutive] organization_id = "myorg" workspace_id = "workspace-123" ``` @@ -381,15 +387,15 @@ curl https://myorg.edge.permutive.app/workspace-123-web.js No route matched for /integrations/custom/endpoint ``` -**Cause:** Integration not enabled or route not registered +**Cause:** No section selects the integration's module, or the route is not registered **Solution:** -1. Enable integration: +1. Select the module in the section of its type so it runs: ```toml -[integrations.custom] -enabled = true +[testing] +modules = ["custom"] ``` 2. Verify integration is compiled in (check build logs) @@ -413,12 +419,8 @@ Signing key not found: ts-2025-01-A 1. Check Config Store `current-kid` value 2. Verify corresponding key exists in Secret Store -3. Run key rotation to generate new key: - -```bash -curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ - -u admin:password -``` +3. Rotate to a new key with `ts keys rotate`, as + [Key Rotation](./key-rotation.md) describes. --- @@ -436,13 +438,8 @@ curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ **Solution:** -1. Initialize keys using rotation endpoint: - -```bash -curl -X POST https://edge.example.com/_ts/admin/keys/rotate \ - -u admin:password -``` - +1. Create the first key with `ts keys rotate`, as + [Key Rotation](./key-rotation.md) describes. 2. Verify Config Store has `active-kids` entry 3. Check Secret Store contains the key @@ -502,13 +499,11 @@ Config store not found: jwks_store - Service → Configuration → Config Stores - Add store with exact name from configuration -3. Update `trusted-server.toml`: +3. Switch request signing on in `trusted-server.toml`: ```toml [request_signing] enabled = true -config_store_id = "your-config-store-id" # From Fastly dashboard -secret_store_id = "your-secret-store-id" ``` --- @@ -668,18 +663,21 @@ cargo install viceroy --version 0.17.0 --locked --force ### Enable Debug Logging -Browser Prebid.js debug remains under `[integrations.prebid]`: +Browser Prebid.js debug remains under `[auction.prebid]`: ```toml -[integrations.prebid] +[auction] +modules = ["prebid"] + +[auction.prebid] debug = true ``` -For Prebid Server diagnostics, enable debug in that provider's profile: +For Prebid Server diagnostics, enable debug on that demand source: ```toml -[auction.providers.pbs-main] -profile_config = { debug = true } +[demand.pbs_main] +debug = true ``` **Check Fastly logs:** diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index fa64280542..aefc396190 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -309,6 +309,10 @@ for the initial provisioning and linking sequence, subsequent CLI pushes, and precautions when changing a live mapping. A process-environment override alone does not configure the Fastly runtime. +One service can serve several publishers, each from an app-config blob of its +own that the request's host chooses. See +[One blob for each host](/guide/configuration#one-blob-for-each-host). + Create the separate request-signing store when that feature is enabled: ```bash @@ -336,8 +340,11 @@ Configure the secret-store key name in `trusted-server.toml`: ```toml [ec] -passphrase = "ec_passphrase" +module = "hmac" ec_store = "ec_identity_store" + +[ec.hmac] +passphrase = "ec_passphrase" ``` Store the high-entropy passphrase under that key in `ts_secrets`. The resolved @@ -377,10 +384,10 @@ Run the repository smoke from a clean shell: The script creates an isolated application config, applies its publisher-origin overrides, and runs strict validation. It then executes `ts config push ---adapter fastly --local`, adds all three required entries to +--adapter fastly --local`, adds both required entries to `[local_server.secret_stores.ts_secrets]`, and starts `fastly compute serve` -through Viceroy. The required keys are `handler_password`, -`publisher_proxy_secret`, and `ec_passphrase`. +through Viceroy. The required keys are `publisher_proxy_secret` and +`ec_passphrase`. The check deliberately proves both halves of startup. With no config entry, it requires `/health` to return 200 while the publisher route returns 500 with the diff --git a/docs/guide/gdpr-compliance.md b/docs/guide/gdpr-compliance.md index 556a7abfd2..235825265f 100644 --- a/docs/guide/gdpr-compliance.md +++ b/docs/guide/gdpr-compliance.md @@ -7,9 +7,11 @@ Consent signal handling in Trusted Server. Trusted Server reads consent signals from each request, decodes them, and applies built-in enforcement rules to consent-gated activities such as EC creation and EID forwarding. The publisher configures how -signals are interpreted: which countries and US states map to each -jurisdiction's rules, how Global Privacy Control is read, how -conflicting signals are resolved, and when stored signals expire. +signals are interpreted, meaning how Global Privacy Control is read, +how conflicting signals are resolved, and when stored signals expire. +Which countries and US states fall under which jurisdiction's rules is +not set here, because the `permissions.yaml` rules tree states it. See +the [Permission Model](/guide/permission-model). The per-activity gates and their fail-closed defaults are built in. ## Policy Posture @@ -96,12 +98,6 @@ Configure consent handling in the `[consent]` section of mode = "interpreter" # or "proxy" (forward raw strings without decoding) max_consent_age_days = 365 # expiration check for dated signals -[consent.gdpr] -applies_in = ["DE", "FR"] # countries mapped to the GDPR rules - -[consent.us_states] -privacy_states = ["CA", "CO"] # US states mapped to the US state rules - [consent.us_privacy_defaults] gpc_implies_optout = true # how the Sec-GPC header is interpreted @@ -112,6 +108,17 @@ mode = "restrictive" # or "newest" / "permissive" Each field tunes how signals are interpreted. The per-jurisdiction gates and their fail-closed defaults are built in. +Which jurisdiction applies to a visitor is not configured here. The +`[consent.gdpr] applies_in` and `[consent.us_states] privacy_states` +lists are retired, and a `jurisdiction` attribute on the +`permissions.yaml` rules tree does their job. Every node of that tree +may name the jurisdiction for the places it covers, a node that names +none inherits the nearest one above it, and the top of the tree answers +a visitor whose country cannot be resolved. One file therefore carries +the permission baselines and the jurisdiction assignment together. The +[Permission Model](/guide/permission-model) documents the tree, so this +page does not repeat it. + ## Operational Behavior - Consent checks run before consent-gated activities (EC creation, @@ -120,7 +127,8 @@ gates and their fail-closed defaults are built in. Resolution of conflicting signals is configurable (restrictive, newest, or permissive). - Audit logging records the consent decision per gated activity. -- Regional rules are applied per detected jurisdiction. +- Regional rules are applied per detected jurisdiction, which the rules + tree assigns from the visitor's country and region. ## Best Practices @@ -136,6 +144,7 @@ gates and their fail-closed defaults are built in. ## Next Steps +- [Permission Model](/guide/permission-model) - [Configuration Reference](/guide/configuration) - [Edge Cookies](/guide/edge-cookies) - [Architecture](/guide/architecture) diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index d83b0cc863..60a7e25229 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -90,11 +90,10 @@ export TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG="$( jq -r '.trusted_server_config' .edgezero/local-config-trusted_server_config.json )" -# Populate the three secret references from the starter config for this shell. +# Populate the two secret references from the starter config for this shell. # Use stable values only if you need existing proxy URLs or EC IDs to remain valid. export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_PUBLISHER_PROXY_SECRET="$(openssl rand -base64 32)" export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_EC_PASSPHRASE="$(openssl rand -base64 32)" -export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_HANDLER_PASSWORD="$(openssl rand -base64 32)" # Build and start the dev server in the same shell. cargo run -p trusted-server-adapter-axum @@ -129,8 +128,8 @@ ephemeral secret-store values only into the current shell; do not put secret values in the TOML config, config-store blob, or a source-controlled environment file. -> **Dev server limitations:** The Axum adapter does not support KV store, -> geo lookup, config/secret-store writes, or admin key-management routes. +> **Dev server limitations:** The Axum adapter does not support KV store or +> geo lookup. > See [Architecture](/guide/architecture) for the full list. ### Build the Project @@ -174,18 +173,20 @@ only the asset proxy entries you want to serve or block. Then validate it. Edit `trusted-server.toml` to configure: -- browser integrations under `[integrations.*]`; -- server auction providers under map-shaped `[auction.providers.]`; -- server bidder routes under `[auction.bidders.]`; -- KV store mappings; -- EC configuration; -- consent settings (`[gdpr]`); and -- stable key names for `trusted_server_secrets`. +- the page modules that run, each selected in the section of its type (`[cmp]`, `[tag]`, `[ad-tag]`, `[bot-protection]`, `[identity]`, `[audience]`, `[framework]`, and `[auction]` for Prebid), with their settings under `[
.]` +- the `[[fetch]]` and `[[serve]]` entries that say which pages each selected module's page changes run on, which the audit's draft writes for the modules it selects +- the demand sources, in `[demand] modules`, each with its settings under `[demand.]` +- the ad server, if one runs, in `[ad-server] module`, with its settings under `[ad-server.]` +- server bidder routes under `[auction.bidders.]` +- KV store mappings +- Edge Cookie configuration under `[ec]` +- stable key names for `trusted_server_secrets` Do not put a Prebid Server URL or server bidder list under -`[integrations.prebid]`, and do not put APS account/endpoint/timeout fields under -`[integrations.aps]`. Those server values belong to auction provider common -fields and `profile_config`. +`[auction.prebid]`. Those server values belong to a `[demand.]` table +whose `implementation` is `auction.prebid-server`, and APS has no integration table at +all, because it is selected in `[demand]` too. The rules every one of these +tables follows are in [Configuration Rules](/guide/configuration-rules). Before the first push, provision the physical store mapped from logical `trusted_server_secrets` with the credential values referenced by the config. @@ -211,13 +212,15 @@ Restart or redeploy instances after secret rotation. See TRUSTED_SERVER__GIT_VERSION=v1.3.0 fastly compute publish ``` -The build compiles `TRUSTED_SERVER__GIT_VERSION` into the binary and reports it -in the `x-ts-version` response header. The Fastly service version is in -`x-ts-fastly-version`. Without the variable, a build reports the checked-out -tag, else the branch, else the first 6 characters of the commit, and a build -outside a git checkout omits the header. The value is public on every -response, so pass a neutral value such as the commit hash when a branch name -is sensitive. +The build compiles `TRUSTED_SERVER__GIT_VERSION` into the binary. Without the +variable, a build compiles in the checked-out tag, else the branch, else the +first 6 characters of the commit, and a build outside a git checkout knows no +version. No adapter writes it on a response. A deployment that names core's +`service_version` middleware in a `[[serve]]` entry writes it as `x-ts-version` +on the pages the entry covers, with the Fastly service version as +`x-ts-service-version`, so pass a neutral value such as the commit hash when a +branch name is sensitive. See +[Headers that describe the deployment](/guide/configuration#headers-that-describe-the-deployment). ## Next Steps diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index f127d4207c..82c485f2d3 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -7,14 +7,14 @@ Axum, or Spin SDK types. ## Choose the narrowest hook -- `IntegrationProxy` owns explicit method/path endpoints and receives - `Settings`, `RuntimeServices`, and an EdgeZero-neutral request. -- `IntegrationAttributeRewriter` inspects selected HTML attributes. -- `IntegrationScriptRewriter` handles one declared selector. -- `IntegrationHeadInjector` inserts deterministic head markup. -- `IntegrationHtmlPostProcessor` is for bounded whole-document work that - cannot be performed during streaming. +- `IntegrationProxy` owns explicit method/path endpoints and receives an + EdgeZero-neutral request and its call into the request's module context, + from which it names the `Settings`, the `RuntimeServices` and whatever + else it needs. - `IntegrationRequestFilter` makes an early request decision. +- `Middleware` changes a page, on the pages where the operator's settings + place it, and is the one way a module changes a page. See + [Changing a page](#changing-a-page). Build one `IntegrationRegistration` with only the hooks the feature needs. Use `with_deferred_js()` only for a separately loaded integration module and @@ -23,7 +23,7 @@ namespaced and bounded; do not introduce a general outbound proxy. ## Compiling core-neutral fixture -The fixture below registers an attribute rewriter and constructs every +The fixture below registers a middleware and constructs every required `RuntimeServices` service without an adapter dependency. The documentation test extracts this exact fence and compiles it as an isolated crate. @@ -32,12 +32,16 @@ crate. ```rust use std::net::IpAddr; +use std::rc::Rc; use std::sync::Arc; use error_stack::Report; use trusted_server_core::integrations::{ - AttributeRewriteAction, IntegrationAttributeContext, - IntegrationAttributeRewriter, IntegrationRegistration, + AttributeRewriteAction, IntegrationRegistration, +}; +use trusted_server_core::middleware::{ + AttributeRewrite, AttributeRewriteFn, Middleware, MiddlewareAction, + MiddlewareContext, MiddlewarePhase, }; use trusted_server_core::platform::{ BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, @@ -130,42 +134,46 @@ impl PlatformBackend for FixtureBackend { struct FixtureGeo; +#[async_trait::async_trait(?Send)] impl PlatformGeo for FixtureGeo { - fn lookup( + async fn lookup( &self, _client_ip: Option, + _services: &RuntimeServices, ) -> Result, Report> { Ok(None) } } -struct AssetRewriter; +struct AssetAddress; -impl IntegrationAttributeRewriter for AssetRewriter { - fn integration_id(&self) -> &'static str { - "example" +impl Middleware for AssetAddress { + fn middleware_id(&self) -> &'static str { + "testing.example" } - fn handles_attribute(&self, attribute: &str) -> bool { - matches!(attribute, "src" | "href") + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] } - fn rewrite( - &self, - _attribute: &str, - value: &str, - _context: &IntegrationAttributeContext<'_>, - ) -> AttributeRewriteAction { - value - .strip_prefix("https://assets.example/") - .map(|path| AttributeRewriteAction::replace(format!("/assets/{path}"))) - .unwrap_or_else(AttributeRewriteAction::keep) + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + let decide: Rc = Rc::new(|matched| { + matched + .value + .strip_prefix("https://assets.example/") + .map(|path| AttributeRewriteAction::replace(format!("/assets/{path}"))) + .unwrap_or_else(AttributeRewriteAction::keep) + }); + MiddlewareAction { + element_handlers: AttributeRewrite::each(&["src", "href"], &decide), + ..MiddlewareAction::pass() + } } } pub fn registration() -> IntegrationRegistration { IntegrationRegistration::builder("example") - .with_attribute_rewriter(Arc::new(AssetRewriter)) + .with_middleware(Arc::new(AssetAddress)) .build() } @@ -189,14 +197,121 @@ Production adapters replace every unavailable or fixture service with the target implementation. The builder deliberately panics when a required service is omitted, so adapter startup must construct the complete service graph. +### A middleware for a media type other than HTML + +A middleware works on `text/html` unless it says otherwise, with +`handles_media_type`, and an entry naming it under a media type it does not +handle is refused when the registry is built. On a media type other than +HTML an action may carry a stream processor alone, which is handed the body +after core has moved the origin's addresses in it, chunk by chunk, with the +last chunk marked. Head markup and the element and text handlers are +decisions about HTML, and an action that makes one on another media type is +refused by name when the chain is planned. + +A middleware named in an entry with a path, for a media type other than +HTML, also runs where the origin answers `404 Not Found` for a path under +it, on an empty document of the entry's media type, and what it writes is +the file. A middleware that adds to a file therefore writes the same on an +empty one, with nothing before its first line. + +```rust +struct Sellers; + +impl Middleware for Sellers { + fn middleware_id(&self) -> &'static str { + "example.sellers" + } + + fn handles_media_type(&self, media_type: &str) -> bool { + media_type == "text/plain" + } + + fn phases(&self) -> &[MiddlewarePhase] { + &[MiddlewarePhase::Fetch] + } + + fn create(&self, _context: &MiddlewareContext<'_>) -> MiddlewareAction { + MiddlewareAction { + stream: Some(Box::new(AppendLines::new(["example-ssp.com, 1, DIRECT"]))), + ..MiddlewareAction::pass() + } + } +} +``` + +The entry that places it names the media type and, where the change is for +one file, its path. See "Entries for a media type other than HTML" in the +[configuration guide](/guide/configuration#entries-for-a-media-type-other-than-html). + +### A middleware that prepares, and one that refuses + +A middleware whose decision needs the request's services, or has to wait on +the key-value store or the network, answers `prepare` with a future. Core +runs it in the request's handler, before the document is processed, as the +module the middleware belongs to and with the permissions it declares in +`required_permissions`. What the future answers is kept under the +middleware's name in the context's `prepared`, and `create`, which stays +synchronous, takes it from there as the type it prepared. A middleware that +prepares nothing leaves `prepare` at its default and is asked `create` as +before. + +```rust +impl Middleware for Sellers { + fn prepare<'a>( + &'a self, + call: ModuleCall<'a>, + context: &'a MiddlewareContext<'a>, + ) -> Option> { + Some(Box::pin(async move { + let lines = self.lines_for(call, context.request_host).await?; + Ok(Box::new(lines) as Box) + })) + } + + fn create(&self, context: &MiddlewareContext<'_>) -> MiddlewareAction { + let lines: Vec = context.take_prepared(MODULE).unwrap_or_default(); + MiddlewareAction { + stream: Some(Box::new(AppendLines::new(lines))), + ..MiddlewareAction::pass() + } + } +} +``` + +The future may answer a `Refusal` instead, with a status, an optional +`Retry-After` and a message. The request is then answered with that status +and the message as plain text, no document is processed, and nothing is +stored or served from the store, so a middleware that cannot make the +document it is responsible for fails closed rather than letting an empty or +unchanged one through. `Refusal::service_unavailable` is the one for a +source that cannot answer now. The preparations of one document run in the +order the entry names the middleware, and the first refusal ends them. + +On a document that may be stored the fetch phase's preparations are handed +the request's services and nothing of the reader, because what they answer +is stored with the document. The serve phase is prepared with the reader's +state on every document, the one fetched for this reader as much as a copy +read from the store, so what a serve middleware finds prepared never depends +on whether its reader was the first. On a document that is not stored both +phases are prepared with the reader's state. + ## Proxy implementation rules -An `IntegrationProxy::handle` implementation receives the complete runtime -service graph. Register or predict backends through `services.backend()`, -send through `services.http_client()`, bound request and response bodies, and -return `Report` with integration context. The registry -strips internal identity headers before dispatch; an integration must opt into -any explicit forwarding behavior. +An `IntegrationProxy::handle` implementation receives the request and its +`ModuleCall`. It hands a function of its own to `call.inject_with`, with the +request as that function's own argument, and names what else it needs as +parameters, such as `&Settings` and `&RuntimeServices`, which is the complete +runtime service graph. Register or predict backends through +`services.backend()`, send through `services.http_client()`, bound request and +response bodies, and return `Report` with the route's +context. The registry strips internal identity headers before dispatch, and a +route must opt into any explicit forwarding behavior. + +A route that names a value whose use needs a permission, such as the Edge +Cookie identifier, declares that permission in `required_permissions` and is +passed the value only on a request that grants it. Nothing gates the route +itself, so a route that must not run without a permission reads the +`&PermissionState` and refuses. Streaming support is an adapter capability. Request `PlatformHttpRequest::with_stream_response()` only when the caller has @@ -212,12 +327,602 @@ DOM insertion dispatcher, remain idempotent, and leave unmatched elements untouched. See [Trusted Server JavaScript](/guide/tsjs) and [GPT's guarded handoff](/guide/integrations/gpt#server-slot-handoff). +## Integrations that ship outside core + +Every integration ships in a crate of its own, which a deployment composes in +at startup, and the hooks described above are what that crate implements. +The vendor owns the code, the release cycle and the integration's own rules, +and core never names the vendor. The crates in this repository sit under +`crates///`, and a vendor's crate can equally live in a +repository of its own. + +`crates/testing/seam-probe` is the worked example. It is a test fixture rather +than something to deploy, and it exercises every part of the seam from a +vendor crate's position. The round-trip tests in +`crates/trusted-server-adapter-axum/tests/seam_probe.rs` drive each part +through the Axum adapter, and the Fastly adapter carries the same round trip. + +### What the crate provides + +The crate hands out an `IntegrationBuilder`, which names the integration and +points at the functions that do the work. + +| Part | Purpose | +| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Id | Names the integration in its routes under `/integrations//`, in its browser module and in diagnostics | +| Source | The crate or package name, reported when two builders claim one id, so an operator can tell which crates collided | +| Build function | Reads `Settings` and returns the registration. It is called only when a section selects the module | +| Validate function | The integration's own deploy rules. It runs for every builder, selected or not | +| `.with_module_name(".")` | The name a section selects the module by, which is the crate's path under `crates/` with `.` between the parts. `module_name!()` derives it from the crate's folder | +| `.with_request_preparer(...)` | Optional. Runs before routing, once for a request, whether or not the module is selected | +| `.with_response_finalizer(...)` | Optional. Finishes the response the page path returns, from what the module's request hooks left for the request | +| `.with_auction_token()` | Optional. Declares that the module's browser script reads the token an auction publishes with its winning bids, which is made only when a selected module reads one | +| `.with_secret_settings(...)` | Optional. Lists the settings in the module's own table that hold the name of a secret, so each is looked up as the settings load | +| `.with_plan_registration(...)` | Optional. Registers the module from the compiled auction plan, for a module whose page support follows what the plan selects. It runs whether or not a section selects the module, and its hooks go first | +| `.with_plan_validator(...)` | Optional. Checks the module's settings against the compiled auction plan, when a deployment is validated and as the settings load | +| `.with_demand(...)` and `.with_adserver(...)` | Optional. Registers an auction implementation that `[demand]` or `[ad-server]` can name. A crate that supplies nothing else starts from `IntegrationBuilder::implementations(...)` | +| `.with_loading_cache(...)` | Optional. Offers the loading cache the crate supplies, which a feature selects by the module's name in its `[cache.]` table. An adapter takes the selections with `loading_caches_from` | +| `.with_shared_settings(...)` | Optional. Runs the builder when its vendor's `[shared.]` table is present, for a vendor whose modules all read that one table. A builder the table runs is selected by no section | + +```rust +pub fn module_name() -> &'static str { + trusted_server_core::module_name!() +} + +pub fn builder() -> IntegrationBuilder { + IntegrationBuilder::new(EXAMPLE_ID, EXAMPLE_SOURCE, register, validate) + .with_module_name(module_name()) + .with_request_preparer(prepare_request) +} +``` + +A crate at `crates/cmp/example` is the module `cmp.example`, so a deployment +runs it with `[cmp] module = "example"` and gives it settings in +`[cmp.example]`. + +### Who maintains the crate + +Every crate outside core names its maintainers in its manifest, the way +Prebid.js requires a named maintainer of every adapter. + +```toml +[package.metadata.maintainers] +owner = "Example Vendor" +status = "vendor owned" +``` + +`status` is `vendor owned` once the vendor has adopted the crate, +`seeking vendor owner` while the Trusted Server maintainers hold it for a +vendor, and `project owned` for a crate that is the project's own, such as a +test module. A test in `crates/trusted-server-modules` fails for a crate in +this repository that leaves the declaration out. + +### What a registration can declare + +The build function returns an `IntegrationRegistration`, built with the +builder every integration uses. + +| Declaration | What it does | +| ----------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `.with_proxy(...)` | Routes the paths the proxy declares | +| `.with_request_filter(...)` | Inspects a request and can turn it back before it reaches the origin | +| `.with_middleware(...)` | Offers a page change that an entry may place by the middleware's name | +| `.with_bundle_tag_attribute(name, value)` | Puts an attribute on the script bundle's tag, for a browser module that reads a setting from it | +| `.with_js_module(CarriedJsModule { source, sha256 })` | Carries the integration's own browser script, built outside `trusted-server-js` | +| `.with_deferred_js()` | Serves the script as its own ` ``` -If `rewrite_sdk` is enabled, Trusted Server rewrites existing DataDome script tags in HTML: +If `rewrite_sdk` is enabled, Trusted Server rewrites existing DataDome script tags in HTML, on the pages a `[[fetch]]` entry names the middleware `bot-protection.datadome` for: ```html @@ -161,7 +167,7 @@ When `enable_protection = true`, Trusted Server calls DataDome before normal rou A request is protected when all of the following are true: -1. The DataDome integration is enabled. +1. `[bot-protection] module` selects DataDome. 2. `enable_protection = true`. 3. The method is not listed in `protection_excluded_methods`. 4. The path is not one of Trusted Server's internal routes. @@ -170,7 +176,7 @@ A request is protected when all of the following are true: 7. No `protection_exclusion_rules` match. 8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. -Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. +Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. @@ -182,7 +188,10 @@ Protection API: ```toml # Runtime activation also requires FASTLY_IS_STAGING=1. -[integrations.datadome.protection_test_bypass] +[bot-protection] +module = "datadome" + +[bot-protection.datadome.protection_test_bypass] enabled = true credential_secret_name = "datadome_test_bypass" ``` @@ -262,7 +271,10 @@ fields. For example: Use structured rules for all DataDome protection exclusions. Each rule has an `id`, optional `methods`, and a typed matcher. The default configuration includes a `path_regex` rule for common static assets. ```toml -[[integrations.datadome.protection_exclusion_rules]] +[bot-protection] +module = "datadome" + +[[bot-protection.datadome.protection_exclusion_rules]] id = "legacy-static-get-head" methods = ["GET", "HEAD"] type = "path_regex" @@ -272,7 +284,7 @@ patterns = [ "^/robots\\.txt$", ] -[[integrations.datadome.protection_exclusion_rules]] +[[bot-protection.datadome.protection_exclusion_rules]] id = "next-rsc" methods = ["GET", "HEAD"] type = "query_param_non_empty" @@ -292,7 +304,10 @@ Supported rule types are: Config Store-backed CIDR sources accept newline-, comma-, whitespace-, or JSON-array encoded CIDR lists. They are useful for large or frequently updated vendor crawler lists. ```toml -[[integrations.datadome.protection_excluded_ip_cidr_sources]] +[bot-protection] +module = "datadome" + +[[bot-protection.datadome.protection_excluded_ip_cidr_sources]] config_store = "datadome-ip-bypass" key = "googlebot_ips" ``` @@ -367,14 +382,13 @@ sequenceDiagram Override configuration via environment variables: ```bash -TRUSTED_SERVER__INTEGRATIONS__DATADOME__ENABLED=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__SDK_ORIGIN=https://js.datadome.co -TRUSTED_SERVER__INTEGRATIONS__DATADOME__API_ORIGIN=https://api-js.datadome.co -TRUSTED_SERVER__INTEGRATIONS__DATADOME__CACHE_TTL_SECONDS=3600 -TRUSTED_SERVER__INTEGRATIONS__DATADOME__REWRITE_SDK=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__ENABLE_PROTECTION=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key -TRUSTED_SERVER__INTEGRATIONS__DATADOME__CLIENT_SIDE_KEY=your-client-side-key +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__SDK_ORIGIN=https://js.datadome.co +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__API_ORIGIN=https://api-js.datadome.co +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__CACHE_TTL_SECONDS=3600 +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__REWRITE_SDK=true +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__ENABLE_PROTECTION=true +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key +TRUSTED_SERVER__BOT-PROTECTION__DATADOME__CLIENT_SIDE_KEY=your-client-side-key ``` ## Client-side script guard @@ -393,11 +407,11 @@ This keeps DataDome scripts routed through first-party context, even when insert ### Script not loading -Check that the integration is enabled: +Check that `[bot-protection]` selects the module: ```toml -[integrations.datadome] -enabled = true +[bot-protection] +module = "datadome" ``` If you rely on auto-injection, verify `client_side_key` is non-empty and `inject_client_side_tag = true`. @@ -415,8 +429,10 @@ curl -X POST https://www.example.com/integrations/datadome/js/check Check that both fields are configured: ```toml -[integrations.datadome] -enabled = true +[bot-protection] +module = "datadome" + +[bot-protection.datadome] enable_protection = true server_side_key_secret_name = "datadome_server_side_key" ``` @@ -425,7 +441,7 @@ Also verify the request is not excluded by the default internal/static route exc ### HTML rewriting not working -Ensure `rewrite_sdk = true` and that your pages are being proxied through Trusted Server's HTML processing pipeline. +Ensure `rewrite_sdk = true`, that a `[[fetch]]` entry names `bot-protection.datadome` for the page, and that your pages are being proxied through Trusted Server's HTML processing pipeline. ## See also diff --git a/docs/guide/integrations/didomi.md b/docs/guide/integrations/didomi.md index a2a64e434b..e4669de5bf 100644 --- a/docs/guide/integrations/didomi.md +++ b/docs/guide/integrations/didomi.md @@ -26,8 +26,10 @@ for the upstream contract. Add the integration to the operator-owned `trusted-server.toml`: ```toml -[integrations.didomi] -enabled = true +[cmp] +module = "didomi" + +[cmp.didomi] geo_query_parameters = true # proxy_path = "my-custom-consent" # sdk_origin = "https://sdk.privacy-center.org" @@ -40,13 +42,12 @@ Publish application configuration with: ts config push --adapter fastly ``` -| Field | Type | Required | Default | Description | -| ---------------------- | ------- | -------- | ------------------------------------- | ------------------------------------------------------- | -| `enabled` | boolean | No | `true` in a present integration block | Enables the integration | -| `geo_query_parameters` | boolean | No | `false` | Enables trusted geo canonicalization for notice loaders | -| `proxy_path` | string | No | `integrations/didomi/consent` | Changes the first-party path prefix | -| `sdk_origin` | string | No | `https://sdk.privacy-center.org` | Changes the SDK origin, primarily for testing | -| `api_origin` | string | No | `https://api.privacy-center.org` | Changes the API origin, primarily for testing | +| Field | Type | Required | Default | Description | +| ---------------------- | ------- | -------- | -------------------------------- | ------------------------------------------------------- | +| `geo_query_parameters` | boolean | No | `false` | Enables trusted geo canonicalization for notice loaders | +| `proxy_path` | string | No | `integrations/didomi/consent` | Changes the first-party path prefix | +| `sdk_origin` | string | No | `https://sdk.privacy-center.org` | Changes the SDK origin, primarily for testing | +| `api_origin` | string | No | `https://api.privacy-center.org` | Changes the API origin, primarily for testing | `geo_query_parameters` is disabled by default for compatibility. It currently supports Fastly only because Cloudflare does not expose a trusted region through @@ -62,8 +63,10 @@ already exists in the TOML input. `proxy_path` helps avoid a predictable integration path: ```toml -[integrations.didomi] -enabled = true +[cmp] +module = "didomi" + +[cmp.didomi] geo_query_parameters = true proxy_path = "my-custom-consent" ``` @@ -76,7 +79,10 @@ This serves Didomi at `/my-custom-consent/*`. The path: - may start with `/`; Trusted Server normalizes the leading slash. Trusted Server passes the resolved path to its browser bundle through -`window.__tsjs_didomi.proxyPath`. +`window.__tsjs_didomi.proxyPath`. The script that sets it is the middleware +`cmp.didomi`, which runs on the pages a `[[fetch]]` entry names it for, so +name it for every page that loads Didomi. See +[Placing page changes](/guide/configuration#placing-page-changes). ## Notice-loader geo flow diff --git a/docs/guide/integrations/gam.md b/docs/guide/integrations/gam.md index 304619ede9..5560c7d752 100644 --- a/docs/guide/integrations/gam.md +++ b/docs/guide/integrations/gam.md @@ -1,7 +1,7 @@ # Google Ad Manager integration status -Trusted Server does not ship a direct Google Ad Manager integration or an -`[integrations.gam]` configuration section. +Trusted Server does not ship a direct Google Ad Manager integration, so there +is no `gam` module to select or configure. The TSJS GPT module can place server-side auction results into publisher-owned GPT slots, including APS renderer winners. That browser handoff is not a GAM @@ -10,4 +10,4 @@ GAM demand. Use the [Integrations overview](/guide/integrations-overview) for the current integration inventory and [Auction orchestration](/guide/auction-orchestration) -for shipped provider profiles. +for the demand implementations it ships. diff --git a/docs/guide/integrations/google_tag_manager.md b/docs/guide/integrations/google_tag_manager.md index c60efac227..4e89dafecd 100644 --- a/docs/guide/integrations/google_tag_manager.md +++ b/docs/guide/integrations/google_tag_manager.md @@ -34,8 +34,10 @@ This is a deliberate design choice to limit data forwarded to Google. If your us Add the GTM configuration to `trusted-server.toml`: ```toml -[integrations.google_tag_manager] -enabled = true +[tag] +modules = ["google-tag-manager"] + +[tag.google-tag-manager] container_id = "GTM-XXXXXX" # upstream_url = "https://www.googletagmanager.com" # Optional override (must be https) # allowed_tag_ids = ["G-XXXXXXXX"] # Tag ids besides container_id that gtag/js may serve first-party @@ -45,14 +47,13 @@ container_id = "GTM-XXXXXX" ### Configuration Options -| Field | Type | Required | Description | -| ---------------------- | ------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | boolean | No | Enable/disable integration (default: `false`) | -| `container_id` | string | Yes | Your GTM Container ID (e.g., `GTM-A1B2C3`) | -| `upstream_url` | string | No | Custom upstream, as a credential-free `https` bare origin with a literal host — no path, query, fragment, userinfo or wildcard (default: `https://www.googletagmanager.com`) | -| `allowed_tag_ids` | array | No | Extra tag ids servable on `gtag/js` besides `container_id` (default: none) | -| `cache_max_age` | number | No | Cache duration in seconds (default: `900`, range: `60`-`86400`) | -| `max_beacon_body_size` | number | No | Max POST body size in bytes (default: `65536`, range: `1024`-`1048576`) | +| Field | Type | Required | Description | +| ---------------------- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `container_id` | string | Yes | Your GTM Container ID (e.g., `GTM-A1B2C3`) | +| `upstream_url` | string | No | Custom upstream, as a credential-free `https` bare origin with a literal host, with no path, query, fragment, userinfo or wildcard (default: `https://www.googletagmanager.com`) | +| `allowed_tag_ids` | array | No | Extra tag ids servable on `gtag/js` besides `container_id` (default: none) | +| `cache_max_age` | number | No | Cache duration in seconds (default: `900`, range: `60`-`86400`) | +| `max_beacon_body_size` | number | No | Max POST body size in bytes (default: `65536`, range: `1024`-`1048576`) | ### Upgrading: check `allowed_tag_ids` @@ -101,7 +102,7 @@ A trailing slash is still accepted; it is trimmed before targets are built. **This is a breaking change.** A path-based value such as `https://tags.example.com/gateway` passed validation before this release and -produced targets below that path, so an enabled deployment using one now fails +produced targets below that path, so a deployment using one now fails config validation. `ts config push` rejects it, which is where you should expect to see this. Validation is fail-closed rather than skip-the-integration: a config that reaches the runtime without having been pushed through that check @@ -129,7 +130,7 @@ flowchart TD ### 1. Script Rewriting -When Trusted Server processes an HTML response, it automatically rewrites GTM script tags to point to the local proxy: +On the pages a `[[fetch]]` entry names the middleware `tag.google-tag-manager` for, Trusted Server rewrites GTM script tags to point to the local proxy. See [Placing page changes](/guide/configuration#placing-page-changes). **Before:** @@ -295,7 +296,7 @@ _Expected_: `200 OK` (or 204). ## Implementation Details -See [crates/trusted-server-core/src/integrations/google_tag_manager.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/integrations/google_tag_manager.rs). +See [crates/tag/google-tag-manager/src/lib.rs](https://github.com/IABTechLab/trusted-server/blob/main/crates/tag/google-tag-manager/src/lib.rs). ## Next Steps diff --git a/docs/guide/integrations/gpt-diagnostics.md b/docs/guide/integrations/gpt-diagnostics.md index af6f2e3248..d2a7dad051 100644 --- a/docs/guide/integrations/gpt-diagnostics.md +++ b/docs/guide/integrations/gpt-diagnostics.md @@ -23,19 +23,30 @@ server integration configuration plus `?ts_console=1`; it does not require new publisher JavaScript, React, Next.js, DOM, or GAM configuration. The diagnostics integration is independent of the -[GPT first-party script integration](./gpt.md). Either integration can be enabled +[GPT first-party script integration](./gpt.md). Either integration can run without the other, although Trusted Server creative-progress evidence is available only for slots served through the existing GPT integration. ## Deployment Configuration -The module is unavailable unless explicitly enabled for the deployment: +The module is unavailable unless the deployment names it: ```toml -[integrations.gpt_diagnostics] -enabled = true +[ad-tag] +modules = ["google.diagnostics"] ``` +What the module writes into a page is the serve middleware `ad-tag.google.diagnostics`, +which runs on each reader's copy of the pages a `[[serve]]` entry names it for: + +```toml +[[serve]] +media_type = "text/html" +middleware = ["ad-tag.google.diagnostics"] +``` + +See [Placing page changes](/guide/configuration#placing-page-changes). + Deployment configuration makes the module available; it does not activate any browser session. Inactive browser sessions receive no diagnostics module. When activated, the standalone content-hashed module loads synchronously after the core bundle so it can @@ -477,7 +488,7 @@ trigger GPT or Prebid work, gate an auction, or delay delivery. ### The API or panel is absent -1. Confirm `[integrations.gpt_diagnostics]` is enabled in the deployed configuration. +1. Confirm `[ad-tag] modules` selects `google.diagnostics` in the deployed configuration. 2. Activate the browser session with an exact recognized `ts_console` value. 3. Confirm the Trusted Server script bundle loaded successfully. 4. Use `ts_console=false` and then `ts_console=true` on a new document to reset diff --git a/docs/guide/integrations/gpt.md b/docs/guide/integrations/gpt.md index ae070950f8..128e8154f7 100644 --- a/docs/guide/integrations/gpt.md +++ b/docs/guide/integrations/gpt.md @@ -25,7 +25,7 @@ All of these are served from `securepubads.g.doubleclick.net`. Publisher HTML │ ├─ + + diff --git a/tools/permissions-inspector/wasm/.gitignore b/tools/permissions-inspector/wasm/.gitignore new file mode 100644 index 0000000000..2f7896d1d1 --- /dev/null +++ b/tools/permissions-inspector/wasm/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tools/permissions-inspector/wasm/Cargo.lock b/tools/permissions-inspector/wasm/Cargo.lock new file mode 100644 index 0000000000..8391023d81 --- /dev/null +++ b/tools/permissions-inspector/wasm/Cargo.lock @@ -0,0 +1,2552 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-io", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "build-print" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e6738dfb11354886f890621b4a34c0b177f75538023f7100b608ab9adbd66b" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "time", + "version_check", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core", + "typenum", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "edgezero-core" +version = "0.1.0" +source = "git+https://github.com/stackpop/edgezero?rev=683202c66948146ec360f490126f1d60f920288b#683202c66948146ec360f490126f1d60f920288b" +dependencies = [ + "anyhow", + "async-compression", + "async-stream", + "async-trait", + "bytes", + "edgezero-macros", + "futures", + "futures-util", + "http", + "http-body", + "log", + "matchit", + "ryu", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha2", + "thiserror", + "toml", + "tower-service", + "tracing", + "validator", + "web-time", +] + +[[package]] +name = "edgezero-macros" +version = "0.1.0" +source = "git+https://github.com/stackpop/edgezero?rev=683202c66948146ec360f490126f1d60f920288b#683202c66948146ec360f490126f1d60f920288b" +dependencies = [ + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 3.0.4", + "toml", + "validator", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "error-stack" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b878b3fac9613c3c7f22eb70bc8a3c6ebdc03cc11479ee60fde1692d747fd45f" +dependencies = [ + "anyhow", + "rustc_version", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "iab_gpp" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3be2d0191a3376e0176bb3df53b2754c644ead6edd50d9494ee8fa376a70e02" +dependencies = [ + "bitstream-io", + "fnv", + "iab_gpp_derive", + "num-derive", + "num-iter", + "num-traits", + "prettyplease", + "proc-macro2", + "quote", + "strum_macros", + "syn 2.0.119", + "thiserror", + "walkdir", +] + +[[package]] +name = "iab_gpp_derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5acda598b043c6386d20fffe86c600b63c7ca4980ee9a28f7e9aaa15d749747" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jose-b64" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec69375368709666b21c76965ce67549f2d2db7605f1f8707d17c9656801b56" +dependencies = [ + "base64ct", + "serde", + "subtle", + "zeroize", +] + +[[package]] +name = "jose-jwa" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab78e053fe886a351d67cf0d194c000f9d0dcb92906eb34d853d7e758a4b3a7" +dependencies = [ + "serde", +] + +[[package]] +name = "jose-jwk" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280fa263807fe0782ecb6f2baadc28dffc04e00558a58e33bfdb801d11fd58e7" +dependencies = [ + "jose-b64", + "jose-jwa", + "p256", + "p384", + "rsa", + "serde", + "zeroize", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lol_html" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" +dependencies = [ + "bitflags", + "cfg-if", + "cssparser", + "encoding_rs", + "foldhash", + "hashbrown", + "memchr", + "mime", + "precomputed-hash", + "selectors", + "thiserror", +] + +[[package]] +name = "matchit" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "elliptic-curve", + "primeorder", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "permissions-inspector-wasm" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "trusted-server-core", + "trusted-server-permission-signal-gpc", + "trusted-server-permission-signal-gpp", + "trusted-server-permission-signal-tcf", + "trusted-server-permission-signal-us-privacy", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro-error-attr3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error3" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "selectors" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12c0ba9680044b4ce98d391a62094047eada0d64860b80166c39f4a6b5640785" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "trusted-server-core" +version = "0.1.0" +dependencies = [ + "async-stream", + "async-trait", + "base64", + "brotli", + "bytes", + "chacha20poly1305", + "chrono", + "cookie", + "cssparser", + "derive_more", + "ed25519-dalek", + "edgezero-core", + "error-stack", + "flate2", + "futures", + "getrandom 0.2.17", + "glob", + "hex", + "hmac", + "http", + "httpdate", + "iab_gpp", + "jose-jwk", + "log", + "lol_html", + "matchit", + "mime", + "p256", + "rand", + "regex", + "serde", + "serde_json", + "serde_yaml_ng", + "sha2", + "subtle", + "toml", + "trusted-server-js", + "trusted-server-openrtb", + "url", + "urlencoding", + "uuid", + "validator", + "web-time", +] + +[[package]] +name = "trusted-server-js" +version = "0.1.0" +dependencies = [ + "build-print", + "hex", + "sha2", + "which", +] + +[[package]] +name = "trusted-server-openrtb" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", +] + +[[package]] +name = "trusted-server-permission-signal-gpc" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "trusted-server-permission-signal-gpp" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "trusted-server-permission-signal-tcf" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "trusted-server-permission-signal-us-privacy" +version = "0.1.0" +dependencies = [ + "trusted-server-core", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "validator" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43fb22e1a008ece370ce08a3e9e4447a910e92621bb49b85d6e48a45397e7cfa" +dependencies = [ + "idna", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e4b81c20a1d6d50d1d7265c658dfbd204e8b9ac4d80f3c931f39462196335" +dependencies = [ + "darling", + "proc-macro-error3", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "serde", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/permissions-inspector/wasm/Cargo.toml b/tools/permissions-inspector/wasm/Cargo.toml new file mode 100644 index 0000000000..2871aad40c --- /dev/null +++ b/tools/permissions-inspector/wasm/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "permissions-inspector-wasm" +version = "0.1.0" +edition = "2024" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +trusted-server-core = { path = "../../../crates/trusted-server-core" } +# The inspector runs the same permission signal modules an adapter ships, +# in the same default order, so the page answers as a deployment would. +trusted-server-permission-signal-gpc = { path = "../../../crates/permission-signal/gpc" } +trusted-server-permission-signal-gpp = { path = "../../../crates/permission-signal/gpp" } +trusted-server-permission-signal-tcf = { path = "../../../crates/permission-signal/tcf" } +trusted-server-permission-signal-us-privacy = { path = "../../../crates/permission-signal/us-privacy" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[profile.release] +opt-level = "z" +lto = true +strip = true +codegen-units = 1 + +# Deliberately its own workspace, like the integration-tests crate, so the +# main workspace's wasm32-wasip1 default target does not apply to it. +[workspace] diff --git a/tools/permissions-inspector/wasm/build.rs b/tools/permissions-inspector/wasm/build.rs new file mode 100644 index 0000000000..5e8f11cb32 --- /dev/null +++ b/tools/permissions-inspector/wasm/build.rs @@ -0,0 +1,46 @@ +use std::process::Command; + +fn git(args: &[&str]) -> String { + Command::new("git") + .args(args) + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_default() +} + +/// Reads the workspace version from the repository root manifest, so the page +/// reports the same version as the trusted-server crates it runs. +fn workspace_version() -> String { + let root = std::fs::read_to_string("../../../Cargo.toml").unwrap_or_default(); + let mut in_package = false; + for line in root.lines() { + let line = line.trim(); + if line.starts_with('[') { + in_package = line == "[workspace.package]"; + } else if in_package && line.starts_with("version") { + if let Some(version) = line.split('"').nth(1) { + return version.to_string(); + } + } + } + String::from("unknown") +} + +fn main() { + println!("cargo:rustc-env=TS_CORE_VERSION={}", workspace_version()); + println!( + "cargo:rustc-env=TS_CORE_COMMIT={}", + git(&["rev-parse", "--short=9", "HEAD"]) + ); + println!( + "cargo:rustc-env=TS_CORE_DATE={}", + git(&["show", "-s", "--format=%cs", "HEAD"]) + ); + println!( + "cargo:rustc-env=TS_CORE_BRANCH={}", + git(&["rev-parse", "--abbrev-ref", "HEAD"]) + ); + println!("cargo:rerun-if-changed=../../../Cargo.toml"); +} diff --git a/tools/permissions-inspector/wasm/src/lib.rs b/tools/permissions-inspector/wasm/src/lib.rs new file mode 100644 index 0000000000..a826d8388d --- /dev/null +++ b/tools/permissions-inspector/wasm/src/lib.rs @@ -0,0 +1,182 @@ +//! The permissions bit of Trusted Server, compiled to WebAssembly for the +//! inspector page. Inputs in, resulting permissions out, through the same +//! functions the server runs: `build_context_from_signals` decodes the raw +//! consent signals and `assemble_permissions` resolves the policy. + +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::json; +use trusted_server_core::consent::build_context_from_signals; +use trusted_server_core::consent::types::RawConsentSignals; +use trusted_server_core::ec::consent::{GeoStatus, assemble_permissions}; +use trusted_server_core::evidence::OwnedRequestInfo; +use trusted_server_core::module_context::ModuleContext; +use trusted_server_core::permission_signal::PermissionSignalModule; +use trusted_server_core::permissions::{Permission, PermissionMaps}; +use trusted_server_core::platform::GeoInfo; + +/// The signal modules the inspector asks, in the order every adapter offers +/// them when `[permission-signal] modules` names none. +fn modules() -> Vec> { + vec![ + Arc::new(trusted_server_permission_signal_gpc::GpcModule::new()), + Arc::new(trusted_server_permission_signal_gpp::GppSaleOptOutModule::new()), + Arc::new(trusted_server_permission_signal_us_privacy::UsPrivacyModule::new()), + Arc::new(trusted_server_permission_signal_tcf::TcfModule::new()), + ] +} + +/// The inspector's evaluation request. +#[derive(Deserialize)] +struct EvalInput { + /// `located`, `none`, or `failed`. + geo: String, + country: Option, + region: Option, + tc: Option, + gpp: Option, + us_privacy: Option, + #[serde(default)] + gpc: bool, +} + +fn eval_json(input: &str) -> String { + let input: EvalInput = match serde_json::from_str(input) { + Ok(input) => input, + Err(e) => return json!({"ok": false, "error": e.to_string()}).to_string(), + }; + let signals = RawConsentSignals { + raw_tc_string: input.tc.filter(|s| !s.is_empty()), + raw_gpp_string: input.gpp.filter(|s| !s.is_empty()), + raw_gpp_sid: None, + raw_us_privacy: input.us_privacy.filter(|s| !s.is_empty()), + gpc: input.gpc, + }; + let ctx = build_context_from_signals(&signals); + // The page carries no request, only the consent signals above, and each of + // the four modules answers from the consent record rather than from + // request evidence, so empty evidence changes none of their answers. A + // module that read a header or a cookie would need real evidence here. + let evidence = OwnedRequestInfo::default(); + // The page stands for no request, so the modules are asked in the empty + // context with the signals entered. + let context = ModuleContext::empty() + .with_consent(&ctx) + .with_evidence(&evidence); + let modules = modules(); + let maps = PermissionMaps::standard(); + let (state, jurisdiction) = match input.geo.as_str() { + "failed" => { + let state = assemble_permissions(&context, GeoStatus::Failed, &modules); + (state, "unknown".to_string()) + } + "none" => { + let state = assemble_permissions(&context, GeoStatus::NoLocation, &modules); + (state, jurisdiction_name(maps.default_jurisdiction())) + } + _ => { + let info = GeoInfo { + city: String::new(), + country: input.country.clone().unwrap_or_default(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: input.region.clone().filter(|r| !r.is_empty()), + asn: None, + }; + let state = assemble_permissions(&context, GeoStatus::Located(&info), &modules); + let jurisdiction = jurisdiction_name( + maps.jurisdiction_for(input.country.as_deref(), input.region.as_deref()), + ); + (state, jurisdiction) + } + }; + let set: Vec<&'static str> = Permission::all() + .filter(|p| state.is_set(*p)) + .map(Permission::as_str) + .collect(); + json!({ + "ok": true, + "jurisdiction": jurisdiction, + "set": set, + "tcf_decoded": ctx.tcf.is_some(), + // No `malformed_record`. Whether an absent, unreadable or expired + // signal changes the permissions is each module's own decision, + // taken in its answer, so nothing at this level can report one and + // `false` would read as "none was found". + }) + .to_string() +} + +fn jurisdiction_name(j: trusted_server_core::consent::jurisdiction::Jurisdiction) -> String { + let name = format!("{j:?}").to_lowercase(); + let name = name.split('(').next().unwrap_or(&name).to_string(); + name.replace("usstate", "us-state") + .replace("nonregulated", "non-regulated") +} + +fn validate_json(yaml: &str) -> String { + match PermissionMaps::from_yaml(yaml) { + Ok(_) => json!({"ok": true}).to_string(), + Err(e) => json!({"ok": false, "error": e.to_string()}).to_string(), + } +} + +fn meta_json() -> String { + json!({ + "version": env!("TS_CORE_VERSION"), + "commit": env!("TS_CORE_COMMIT"), + "date": env!("TS_CORE_DATE"), + "branch": env!("TS_CORE_BRANCH"), + }) + .to_string() +} + +/// Leaks a length-prefixed buffer the host reads and then frees. +fn out(s: String) -> *mut u8 { + let bytes = s.into_bytes(); + let mut buf = Vec::with_capacity(4 + bytes.len()); + buf.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(&bytes); + let ptr = buf.as_mut_ptr(); + core::mem::forget(buf); + ptr +} + +#[unsafe(no_mangle)] +pub extern "C" fn ts_alloc(len: usize) -> *mut u8 { + let mut buf = vec![0u8; len]; + let ptr = buf.as_mut_ptr(); + core::mem::forget(buf); + ptr +} + +/// # Safety +/// `ptr` must come from `ts_alloc` or an `out` buffer with capacity `len`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ts_free(ptr: *mut u8, len: usize) { + unsafe { drop(Vec::from_raw_parts(ptr, len, len)) }; +} + +/// # Safety +/// `ptr`/`len` must describe a valid UTF-8 JSON buffer from `ts_alloc`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ts_eval(ptr: *const u8, len: usize) -> *mut u8 { + let input = unsafe { core::slice::from_raw_parts(ptr, len) }; + out(eval_json(core::str::from_utf8(input).unwrap_or("{}"))) +} + +/// # Safety +/// `ptr`/`len` must describe a valid UTF-8 YAML buffer from `ts_alloc`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ts_validate(ptr: *const u8, len: usize) -> *mut u8 { + let input = unsafe { core::slice::from_raw_parts(ptr, len) }; + out(validate_json(core::str::from_utf8(input).unwrap_or(""))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn ts_meta() -> *mut u8 { + out(meta_json()) +} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 066ad659e3..32f43939cd 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -1,21 +1,43 @@ # ============================================================================= -# Trusted Server — application configuration template +# Trusted Server application configuration template # ============================================================================= # # This is the source-controlled starting point for an operator-owned # `trusted-server.toml`. Copy it (`ts config init`), fill in the required # values, and push it (`ts config push`) as an EdgeZero app-config blob. # -# Only three sections are REQUIRED for the server to start and pass validation: -# 1. [[handlers]] covering /_ts/admin (admin authentication) -# 2. [publisher] (domain + origin) -# 3. [ec] passphrase (Edge Cookie identity secret) +# Only one section is REQUIRED for the server to start and pass validation, +# which is [publisher] (domain + origin). # -# Everything below those is OPTIONAL. Most optional blocks are commented out — -# uncomment and edit one to enable it — but a few integrations are kept as active -# `enabled = false` stubs (see the Integrations section for why). All example -# hosts use `example.com`; replace them with your real endpoints in your private -# config, not in this template. +# Edge Cookie identity is optional and stays off until an [ec] module is +# selected, so no identity secret is needed to start. See the Edge Cookie +# section below. +# +# Everything below it is OPTIONAL. Most optional blocks are commented out, so +# uncomment and edit one to switch it on. All example hosts use `example.com`, +# so replace them with your real endpoints in your private config, not in this +# template. +# +# Everything this deployment can switch on is a module, selected the same way, +# with `module` where one runs and `modules` where several run: +# +# [] +# module = "" # modules = [...] where several run +# +# [.] # only when that name has settings +# setting = "value" +# +# The types are ec, geo, device, permission-signal, demand, ad-server, +# robots-txt, analytics and integration. A settings table its type's selector +# does not name refuses startup, so there is no stale block to misread. The +# complete rules, and what is checked when, are in +# docs/guide/configuration-rules.md. +# +# [shared.] holds what one vendor's modules share, such as an endpoint +# and a license key. That vendor's crates alone read it, its presence runs the +# builder they share, and a table no module reads is refused. [prefix] selects +# the module that names the browser's cookies and the shared store keys, and +# core's own `ts` runs when it selects nothing. # # `TRUSTED_SERVER__` env vars can override values when the `ts` CLI builds and # validates this config for push (e.g. TRUSTED_SERVER__PUBLISHER__DOMAIN=...; @@ -28,29 +50,7 @@ # ----------------------------------------------------------------------------- -# REQUIRED — Admin authentication -# ----------------------------------------------------------------------------- -# HTTP Basic-auth handler(s). At least one handler whose `path` regex covers -# the /_ts/admin endpoints is mandatory; startup fails without it. Each handler -# needs a non-placeholder username/password (deploy validation rejects the -# sample password below). -[[handlers]] -# Regex matched against the request path. This one guards the admin surface. -path = "^/_ts/admin" -username = "admin" -password = "handler_password" - -# You can add more handlers to basic-auth-protect other path prefixes. The -# sample password below is a known placeholder that deploy validation rejects, -# so it forces a real secret before push: -# [[handlers]] -# path = "^/secure" -# username = "user" -# password = "replace-with-admin-password" - - -# ----------------------------------------------------------------------------- -# REQUIRED — Publisher / origin +# REQUIRED - Publisher / origin # ----------------------------------------------------------------------------- [publisher] # Public domain Trusted Server is fronting. @@ -70,21 +70,67 @@ proxy_secret = "publisher_proxy_secret" # ----------------------------------------------------------------------------- -# REQUIRED — Edge Cookie (EC) identity +# OPTIONAL - Edge Cookie (EC) identity # ----------------------------------------------------------------------------- [ec] -# Secret Store key for the secret used to derive EC identifiers. -passphrase = "ec_passphrase" +# Edge Cookie identity is OFF by default. With no module selected, Trusted +# Server runs statelessly and generates no Edge Cookie. Activate one by +# uncommenting the selector below, together with that module's own +# [ec.] table when it has settings (a table the selector does not name +# is rejected at startup). Deployment tooling can merge a +# TRUSTED_SERVER__EC__MODULE environment value into the published +# configuration before it is loaded, and the running server itself reads its +# settings from the platform config store rather than the environment. The +# built-in hmac module is host-neutral, and a vendor module ships in its +# own crate that the adapter composes in. +# module = "hmac" # KV store that persists EC identity state. This is the physical store name # bound per adapter (e.g. `ec_identity_store` in fastly.toml); edgezero.toml's # logical KV id is `trusted_server_kv`. ec_store = "ec_identity_store" +# Extra origins allowed to POST the client resolve endpoint. The endpoint +# always accepts https:// and nothing else by default. List +# further origins here if the pages that resolve identity are served from +# another origin, www being the common case. Each entry is a bare origin +# (RFC 6454), being http:// or https:// and a host with an optional port, with +# no path, trailing slash, query or fragment, and any other entry is refused +# when the configuration is pushed or loaded. Entries are compared by the +# same-origin test, so the scheme, host and port all have to match, and a +# subdomain is never accepted just for being a subdomain. +# resolve_allowed_origins = ["https://www.example.com"] # Max concurrent partner pull-sync requests. pull_sync_concurrency = 3 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds +# Built-in HMAC module table. Uncomment it together with the +# `module = "hmac"` selector above. `passphrase` names the Secret Store key +# holding the secret used to derive EC identifiers. That secret must be +# >= 32 bytes and non-placeholder in production. Provision referenced keys in +# trusted_server_secrets before validating/pushing. +# [ec.hmac] +# passphrase = "ec_passphrase" + +# A table can name the implementation it configures, which makes the table's +# own name a label of your choosing. The same module could therefore be +# selected as `module = "primary"` and configured in the table below +# instead, which is how a deployment gives a module a name that means +# something to the people running it. +# A module from a crate is named by its folder below crates/, and may be +# written in full, as edgecookie., or with that type folder left off. +# [ec.primary] +# implementation = "hmac" +# passphrase = "ec_passphrase" + +# Built-in host-signal module table. Creates the identifier from the host's +# TLS JA4 and HTTP/2 signals together with the client IP, so it needs a host +# that supplies those signals (Fastly does). Uncomment it together with +# `module = "host_signals"` above, and give it its own secret on the same +# terms as the hmac table. +# [ec.host_signals] +# passphrase = "ec_host_signals_passphrase" + # Example partner configuration. Provision referenced keys in # trusted_server_secrets before validating/pushing. # [[ec.partners]] @@ -102,26 +148,99 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false +# Which module resolves location. Leave the whole section out to keep the +# host's own lookup, which is what every adapter ships with today. +# [geo] +# `none` resolves no location at all, or name a registered module that declares +# a geo module to have that module resolve location instead of the host. +# module = "none" + # ============================================================================= -# OPTIONAL — Core features (disabled/omitted by default) +# OPTIONAL - Core features (disabled/omitted by default) # ============================================================================= +# Device-detection module. Selects how a request is classified into device +# signals (browser/bot gating). Default "builtin" classifies from the User-Agent +# alone and makes no host call, so a default deployment stays host-neutral. The +# opt-in "fastly" module adds the host's TLS/H2 signals to the gate. +# Override at deployment with TRUSTED_SERVER__device__module. +# [device] +# module = "builtin" # or "fastly" to add TLS/H2 signal evidence + +[geo] +# Geo / IP intelligence module. No module is the default: Trusted Server +# resolves no geolocation and makes no host geo call, so a default deployment is +# not tied to any host geo service. Opt into the host lookup with +# module = "platform". Override with TRUSTED_SERVER__geo__module. +# module = "platform" +# +# The permission baseline for a request the geo module leaves unmatched +# (and, with no geo module, for every request) is the top of the rules tree +# in the permissions.yaml compiled into the build (the repository sample is +# config/permissions/sample.yaml). Edit that file to +# change it. +# +# With no geo module, every request is treated as that top node, so a +# visitor from another jurisdiction receives its permission rules. A +# deployment that runs an Edge Cookie module without a geo module must +# acknowledge that by uncommenting the line below, or select a geo module +# instead. +# assume_single_jurisdiction = true + # Custom headers added to every response (e.g. X-Robots-Tag: noindex). # [response_headers] # X-Robots-Tag = "noindex" -# Sign outbound OpenRTB/API requests. These are platform management-API store -# IDs used for key-rotation WRITES; the runtime reads keys from fixed store names -# (`jwks_store` / `signing_keys`), not these. To keep signing OFF, leave this -# whole block commented out. If you uncomment it - even with `enabled = false` - -# both store IDs are required and must be REAL values: the key rotate/deactivate -# admin routes read them regardless of `enabled`, so deploy validation rejects -# the placeholders below (replace them before pushing). +# What /robots.txt answers. With this section absent the publisher keeps +# their own file and /robots.txt reaches their origin like any other path. +# +# `modules` names what makes the file, in order, as one name or a list. +# refuse_all refuses every crawler and puts X-Robots-Tag: noindex, nofollow on +# the responses it sends, allow_all allows every crawler everywhere, and any +# other name is a module that supplies rules. Modules contribute rules and +# Trusted Server writes the file. A module that cannot answer has its last +# answer served past its age, and with nothing held the answer is 503, never +# an empty file, which a crawler would read as permission. +# +# A module's own settings go in its own block, [robots-txt.]. +# A publisher who sells advertising wants /ads.txt in always_allow, because +# a crawler may ignore an ads.txt that robots.txt disallows. The file is made +# by core's robots_txt middleware where a [[fetch]] entry for text/plain on +# the file's path names it, see the entry below the ads.txt one. +# [robots-txt] +# modules = ["refuse_all"] +# always_allow = ["/ads.txt"] +# sitemap = "https://publisher.example.com/sitemap.xml" + +# The sellers added to the publisher's ads.txt. The origin's file stays +# authoritative and the lines are written at its end, where a [[fetch]] entry +# for text/plain on the file's path names the middleware. Each line is a data +# record or a variable of the ads.txt grammar, checked as the settings load. +# [ads-txt] +# modules = ["sellers"] + +# [ads-txt.sellers] +# lines = ["ssp.example, 12345, DIRECT, f08c47fec0942fa0"] + +# [[fetch]] +# media_type = "text/plain" +# path = "/ads.txt" +# middleware = ["ads-txt.sellers"] + +# The robots.txt the [robots-txt] section describes, made by core's own +# middleware where this entry names it. The file replaces the origin's own +# and is made where the origin has none. +# [[fetch]] +# media_type = "text/plain" +# path = "/robots.txt" +# middleware = ["robots_txt"] + +# Sign outbound OpenRTB/API requests. The runtime reads its keys from fixed +# store names (`jwks_store` / `signing_keys`), and `ts keys` writes them. To keep +# signing OFF, leave this whole block commented out. # [request_signing] # enabled = false -# config_store_id = "" -# secret_store_id = "" # First-party HTML/CSS rewriting controls. # [rewrite] @@ -145,17 +264,17 @@ pull_sync_concurrency = 3 # enabled = false # max_age_seconds = 2592000 -# Consent forwarding. All values below are the defaults — uncomment to override. +# Consent forwarding. All values below are the defaults, so uncomment one to +# override it. # [consent] # mode = "interpreter" # "interpreter" (decode + forward) or "proxy" (raw passthrough) # check_expiration = true # check TCF consent freshness # max_consent_age_days = 395 # max age before consent is treated as expired (~13 months) # -# [consent.gdpr] -# applies_in = ["AT","BE","BG","HR","CY","CZ","DK","EE","FI","FR","DE","GR","HU","IE","IT","LV","LT","LU","MT","NL","PL","PT","RO","SK","SI","ES","SE","IS","LI","NO","GB"] -# -# [consent.us_states] -# privacy_states = ["CA","VA","CO","CT","UT","MT","OR","TX","FL","DE","IA","NE","NH","NJ","TN","MN","MD","IN","KY","RI"] +# Which places fall under GDPR, and which US states have a comprehensive +# privacy law, are no longer listed here. They are the `jurisdiction` values in +# the permissions.yaml rules tree, so one file states the policy for both the +# permission baseline and the consent handling. # # [consent.us_privacy_defaults] # notice_given = true # has the publisher shown CCPA notice? @@ -166,22 +285,115 @@ pull_sync_concurrency = 3 # mode = "restrictive" # "restrictive" | "newest" | "permissive" # freshness_threshold_days = 30 -[integrations.js_asset_proxy] -enabled = false +# Which permission signal modules run, and in what order. +# +# Signals compose rather than select. A request can carry a TCF string and a +# Global Privacy Control header at once, and both have something to say, so +# modules here is a list, whereas in [ec], [geo] and [device] module names +# one module. The order is the policy, because the last module with an +# opinion decides. +# +# Listed below is every module the shipped adapters link, which is also +# exactly what runs when modules is left out. Each is a crate under +# crates/permission-signal, outside the core, and is named by its folder +# there. Remove the ones this +# publisher does not want to act on. A module that is not on the list does +# not run, and there is no separate switch to turn one off. An unknown or +# repeated name is refused at startup rather than quietly ignored. None of the +# four has settings, so none needs a [permission-signal.] table. +# +# [permission-signal] +# modules = [ +# "gpc", # the Sec-GPC request header, Global Privacy Control +# "gpp", # a GPP US sale opt-out +# "us-privacy", # a US Privacy string sale opt-out +# "tcf", # TCF v2, with its purpose mapping in the crate +# ] +# +# Each name is the module's crate folder below crates/permission-signal, and +# may be written in full, as permission-signal.gpp. +# +# A module that gains settings will take them in a [permission-signal.] +# block named for it. None of these four has settings, so such a block is +# refused rather than ignored. +# +# The three opt-outs are separate entries so that a publisher who does not act +# on Global Privacy Control can remove "gpc" and keep the GPP and US Privacy +# opt-outs working. +# +# A consent record that arrives and cannot be read revokes, whichever modules +# are configured. That is error handling rather than a module to choose, so +# it is not in this list and cannot be removed. +# +# The default order asks Global Privacy Control first, being a browser setting +# with no interface of its own, and the three that carry a choice someone made +# through an interface after, so an answer given at a prompt amends the header +# the visitor arrived with. Reorder the list to change that. +# +# The layering, what a module may consult, and how to add a scheme are +# documented in crates/trusted-server-core/src/permission_signal/README.md +# and in the Permission Signals guide. + +# Page integrations are modules, each selected in the section of its type, +# which is [cmp] for a consent prompt, [tag] and [ad-tag] for tags, +# [bot-protection], [identity], [audience], [framework], [auction] for the +# modules the auction runs and [proxy] for the script proxy. A section names +# what runs with `module` where one runs or `modules` where several run, a +# module with settings gets its own [
.] table, and a section +# that selects nothing or a table its section does not select refuses +# startup. Nothing is selected by default. The sections and tables are +# further down, commented out. + +# A module changes a page through a middleware, and a middleware runs only on +# the pages where a [[fetch]] or [[serve]] entry names it. A page takes the +# first entry that covers it and runs that entry's middleware in the order +# written. The two entries below name every middleware the modules further +# down supply, in the order `ts audit` writes them, and core's own +# `service_version` last. Keep the names of the modules that are selected and +# remove the rest, because an entry that names the middleware of a module no +# section selects refuses startup. See "Placing page changes" in the +# configuration guide. +# [[fetch]] +# media_type = "text/html" +# middleware = [ +# "auction.prebid", +# "js_asset_proxy", +# "testing.testlight", +# "framework.nextjs", +# "audience.permutive", +# "identity.lockr", +# "cmp.didomi", +# "cmp.inmobi", +# "cmp.sourcepoint", +# "tag.google-tag-manager", +# "bot-protection.datadome", +# "ad-tag.google", +# ] + +# [[serve]] +# media_type = "text/html" +# middleware = ["tag.google-tag", "bot-protection.datadome.tag", "ad-tag.google.diagnostics", "service_version"] + +# First-party proxying of named third-party scripts. Uncomment the +# `modules` line under [proxy] below to switch it on. +# [proxy.js_asset_proxy] # Uncomment to override upstream cache headers for every asset below. # This replaces upstream directives, including private and no-store. # Use only when each asset's bytes are identical for every visitor. # cache_ttl_seconds = 3600 # Asset fetches use a fixed TrustedServer/1.0 User-Agent. Do not proxy assets # that vary by browser User-Agent or use integrity hashes for UA-specific bytes. - -[[integrations.js_asset_proxy.assets]] -path = "/assets/example-vendor-loader.js" -origin_url = "https://cdn.example.com/vendor-loader.js" -proxy = "enabled" +# +# [[proxy.js_asset_proxy.assets]] +# path = "/assets/example-vendor-loader.js" +# origin_url = "https://cdn.example.com/vendor-loader.js" +# proxy = "enabled" # Proxy behavior and first-party asset routing. Kept active with defaults. [proxy] +# The modules [proxy] selects. Uncomment to run the first-party script proxy +# configured in [proxy.js_asset_proxy] above. +# modules = ["js_asset_proxy"] # Verify TLS certs when proxying to HTTPS origins (default true; false only for # local self-signed dev). # certificate_check = true @@ -197,7 +409,7 @@ proxy = "enabled" # prefix = "/.image/" # origin_url = "https://assets.example.com" # Optional path rewrite before sending upstream (the prefix must match what the -# pattern expects — here both use `/.image/`): +# pattern expects, and here both use `/.image/`): # path_pattern = "^/\\.image/(.*)/[^/]+\\.([^/.]+)$" # target_path = "/image/upload/$1.$2" # @@ -230,6 +442,19 @@ proxy = "enabled" # thumbnail = "width=150&crop=1:1,smart" # medium = "format=auto&width=828" +# The loading cache that holds each robots.txt module's answer, named by the +# module that offers it, and how long the copies last, in seconds: the brief +# copy in process memory, the longest the copy in the shared store is used, and +# how long that copy lasts unused. A module's own refresh ends its answer +# sooner. Memory may be 0, the other two must be at least 1, and none may be +# more than a year. Without `module` the one loading cache module the build +# links is used, and with none or several linked no copy is kept. +# [cache.robots-txt] +# module = "51degrees" +# memory_seconds = 60 +# store_seconds = 604800 +# store_idle_seconds = 2592000 + # Operator-controlled cache header policies for static/rehosted assets. Disabled # rules never match, and matcher/policy validation is deferred until a rule is # enabled; IDs must be non-empty and unique. Keep rules disabled unless the @@ -256,9 +481,11 @@ proxy = "enabled" # edge_ttl_seconds = 31536000 # immutable = true -# Server-side auction. Providers are declared below; mediator names must match -# enabled integrations. Kept active with the creative-processing leaves present -# so the EdgeZero environment override can apply to them. +# Server-side auction. [auction] holds the settings that belong to the auction +# itself, [demand] selects the demand sources, [ad-server] selects the ad server +# that picks the winner, and [auction.bidders.] says which demand source a +# browser bidder code is routed to. Kept active with the creative-processing +# leaves present so the EdgeZero environment override can apply to them. [auction] # Keep disabled until provider endpoints, routes, and profile values below are # replaced with deployment-specific settings. @@ -280,47 +507,69 @@ rewrite_creatives = true # blank slots on a script-heavy demand stack. sanitize_creatives = false timeout_ms = 2000 -# mediator = "adserver_mock" # optional mediator integration # Context keys the JS client may forward into auction requests (allowlist; # empty blocks all). allowed_context_keys = [] - -# Example server-side provider declarations. Remove or customize before enabling auctions. -[auction.providers.pbs-main] -protocol = "openrtb-2.6" -profile = "prebid-server" +# The modules [auction] selects, with their tables under OPTIONAL below. Add +# "testing.testlight" for the Testlight test module. +# modules = ["prebid"] + +# Example demand sources. Remove or customize before enabling auctions. The +# table name is this source's identity, and its `implementation` line names +# the implementation by module path, which is how two Prebid Servers run side +# by side under names of their own. The built-in implementations are +# auction-protocol.openrtb, auction.prebid-server and auction.aps. Every +# endpoint must be HTTPS, or HTTP to 127.0.0.1, ::1 or localhost. +[demand] +modules = ["pbs_main"] + +[demand.pbs_main] +implementation = "auction.prebid-server" endpoint = "https://prebid.example.com/openrtb2/auction" -# `prebid-server` defaults to 1000 ms when omitted. +# `auction.prebid-server` defaults to 1000 ms when omitted. routing = "explicit" - -[auction.providers.pbs-main.profile_config] debug = false test_mode = false consent_forwarding = "both" -[auction.providers.pbs-main.notifications] +[demand.pbs_main.notifications] suppress_all = false suppress_seats = [] [auction.bidders.example-bidder] -provider = "pbs-main" - -# APS server behavior is also provider/profile-owned. Use routing = "all_eligible" -# when every banner-compatible slot should be eligible. The APS profile timeout -# defaults to 800 ms; debug and script creatives default to false. -# [auction.providers.aps-main] -# protocol = "openrtb-2.6" -# profile = "aps" +module = "pbs_main" + +# APS is a demand implementation too, not a page integration, so everything it +# owns lives in its own table. Use routing = "all_eligible" when every +# banner-compatible slot should be eligible. The APS timeout defaults to +# 800 ms, and debug, script creatives and publisher-native rendering all +# default to off. Add "aps_main" to the [demand] modules list above when +# uncommenting this. +# [demand.aps_main] +# implementation = "auction.aps" # endpoint = "https://aps.example.com/e/pb/bid" # routing = "all_eligible" -# -# [auction.providers.aps-main.profile_config] # account_id = "example-aps-account-id" # debug = false # allow_script_creatives = false +# rendering_mode = "trusted_server" # inventory_domain = "publisher.example" # inventory_page_origin = "https://www.publisher.example" +# The ad server that picks the auction winner. With no [ad-server] selection the +# highest bid wins. The only implementation this repository ships is +# ad-server.mock, for development and testing, written here as mock because +# its type is this section's. +# [ad-server] +# module = "mock" +# +# [ad-server.mock] +# endpoint = "https://adserver.example.com/decide" +# timeout_ms = 1000 +# Map auction context keys to decision-URL query params: +# [ad-server.mock.context_query_params] +# example_segments = "segments" + # Server-side ad slot templates + creative-opportunity auction. Kept active. [creative_opportunities] # Set false to disable server-side ad templates while keeping slot definitions @@ -333,7 +582,7 @@ provider = "pbs-main" enabled = true gam_network_id = "123456789" price_granularity = "dense" -# FCP is not affected by this value — body content above has already +# FCP is not affected by this value, because body content above has already # streamed and painted before the hold begins. What this caps is the slip on # DOMContentLoaded and window.load. 500 ms is the recommended default; raise # only if your SSPs need more headroom and analytics confirm the DCL slip is OK. @@ -347,7 +596,7 @@ auction_timeout_ms = 500 # assembly_mode = "inline" # Request headers (besides Accept-Encoding) the origin may name in Vary. Every # emitted Vary name must be covered here or template storage is refused. Never -# include Cookie — shared templates must be reader-neutral. +# include Cookie, because shared templates must be reader-neutral. # template_cache_vary = [ # "rsc", # "next-router-state-tree", @@ -434,28 +683,30 @@ auction_timeout_ms = 500 # "/news/x" -> /123456789/example/news # "/reviews/y" -> /123456789/example/reviews -# Direct Tinybird auction telemetry (off by default). When enabled, `api_host` -# is required and must be a bare host (no scheme or path). -# [tinybird] -# enabled = true -# api_host = "api.us-east.tinybird.example" # required when enabled; host only +# Direct Tinybird auction telemetry, sent only when [analytics] selects it. +# `api_host` is required and must be a bare host (no scheme or path). +# [analytics] +# module = "tinybird" +# +# [analytics.tinybird] +# api_host = "api.us-east.tinybird.example" # required; host only # auction_dataset = "auction_events" # Events API datasource name # auction_token_secret = "tinybird_auction_append_token" # Key in trusted_server_secrets -# Debug endpoints (all default false — never enable in production). +# Debug endpoints (all default false, and never enable one in production). # [debug] # Exposes GET /_ts/debug/ja4 returning TLS/JA4 fingerprint details. Disable # after investigation; it reflects data browser JS cannot normally read. # ja4_endpoint_enabled = false # Injects a `` auction dump before ``. NEVER enable # in production: even the default "redacted" mode still includes bid-level fields -# and creative previews — it is not a fully anonymized dump. +# and creative previews, so it is not a fully anonymized dump. # auction_html_comment = false # # Fine-grained control over the dump (all optional; defaults shown). # [debug.auction_html_comment_options] # include_provider_responses = true -# include_mediator_response = true +# include_adserver_response = true # include_bids = true # Subset of the fixed validated metadata keys shown in "redacted" (and, for these # three keys, "upstream") mode. Any other key fails config load. @@ -471,35 +722,29 @@ auction_timeout_ms = 500 # ============================================================================= -# OPTIONAL — Integrations +# OPTIONAL - Integration settings # ============================================================================= -# Every integration is off by default. Most are fully commented out; uncomment a -# block and set `enabled = true` to activate it. Four (gpt, didomi, datadome, -# google_tag_manager) are kept as active `enabled = false` stubs so the `ts audit` -# CLI can flip them in place — leave those sections present. Integrations whose -# `enabled` defaults to true (prebid, permutive, lockr, ...) still stay OFF while -# their section is commented out. Required fields are noted per block. +# Every module is off by default, because no section below is uncommented. To +# switch one on, uncomment its section's selection and its settings table. A +# module with nothing to set needs no table at all. A table its section does +# not select refuses startup, so a table left behind after a module is +# switched off is caught rather than sitting unread. Required fields are +# noted per table. # ============================================================================= - -# Browser-side Prebid.js integration. Server-side bidder routing belongs under -# [auction.providers] and [auction.bidders]. When enabled, -# `external_bundle_url` is required and its host must be listed in -# [proxy].allowed_domains. Kept active but disabled. -[integrations.prebid] -enabled = false -timeout_ms = 1000 -debug = false -client_side_bidders = [] # bidders running via native Prebid.js adapters +# [auction.prebid] +# timeout_ms = 1000 +# debug = false +# client_side_bidders = [] # bidders running via native Prebid.js adapters # Keep selected GAM inventory out of Trusted Server's Prebid refresh auctions. # Matching slots still refresh through GAM. # excluded_gam_ad_unit_path_suffixes = ["/trackingonly"] -# Runtime bundle metadata — set after running `ts prebid client` and uploading: +# Runtime bundle metadata, set after running `ts prebid client` and uploading: # external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-.js" # external_bundle_sha256 = "" # external_bundle_sri = "" # Bundle build inputs consumed by `ts prebid client`, not by the edge runtime. # Values are exact upstream module stems without `.js`. -# [integrations.prebid.bundle.modules] +# [auction.prebid.bundle.modules] # bidder = ["rubiconBidAdapter"] # Omit user_id for the curated default preset; use [] to select none. # user_id = ["sharedIdSystem"] @@ -514,7 +759,7 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # the checked-in User ID registry. Each `name` must appear only once. # # The module must be present in the built bundle: name it under -# [integrations.prebid.bundle.modules].user_id, or omit that list to take the +# [auction.prebid.bundle.modules].user_id, or omit that list to take the # generator's default preset. `ts prebid client` resolves every managed name # through the checked-in User ID registry and fails if the generated manifest # omits its required module without updating the configured hash or SRI. @@ -524,63 +769,78 @@ client_side_bidders = [] # bidders running via native Prebid.js adapter # OpenRTB EID source; without one the ID still reaches the auction but is never # written to KV. # -# [[integrations.prebid.managed_user_ids]] +# [[auction.prebid.managed_user_ids]] # name = "sharedId" # -# [integrations.prebid.managed_user_ids.storage] +# [auction.prebid.managed_user_ids.storage] # type = "cookie" # or "html5" # name = "_sharedid" # expires = 15 # days; omit to keep Prebid's default # refresh_in_seconds = 1800 # omit to keep Prebid's default # Next.js first-party rewriting for App Router / RSC payloads. -# [integrations.nextjs] -# enabled = true +# [framework] +# module = "nextjs" + +# [framework.nextjs] # rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] # max_combined_payload_bytes = 10485760 # 10 MiB -# Testlight OpenRTB test integration. `endpoint` required when enabled. -# [integrations.testlight] -# enabled = true +# Testlight OpenRTB test integration, selected as "testing.testlight" in +# [auction] modules. `endpoint` is required. +# [auction.testing.testlight] # endpoint = "https://testlight.example.com/openrtb2/auction" # timeout_ms = 1200 # rewrite_scripts = true -# Didomi CMP SDK/API first-party proxy. Kept active but disabled so `ts audit` -# can flip `enabled` to true when Didomi is detected on the audited page. -[integrations.didomi] -enabled = false -geo_query_parameters = false +# Didomi CMP SDK/API first-party proxy. `ts audit` selects "didomi" in [cmp] +# when it detects Didomi on the audited page; the table below stays yours to +# uncomment and fill. +# [cmp] +# module = "didomi" + +# [cmp.didomi] +# geo_query_parameters = false # sdk_origin = "https://sdk.example.com" # api_origin = "https://api.example.com" # Sourcepoint CMP first-party proxy. # `cdn_origin` is intentionally omitted: the upstream CDN origin is pinned to # Sourcepoint's own host and is not operator-selectable. Leave it unset so the -# validated default applies — overriding it with any other host fails config -# validation. -# [integrations.sourcepoint] -# enabled = true +# validated default applies, because overriding it with any other host fails +# config validation. Selected with [cmp] module = "sourcepoint". +# [cmp.sourcepoint] # rewrite_sdk = true # cache_ttl_seconds = 3600 # auth_cookie_name = "sp_auth" # optional: forward a custom authCookie upstream -# Osano consent management (proxy toggle only). -# [integrations.osano] -# enabled = true +# Osano consent management. It has nothing to set, so [cmp] module = "osano" +# is the whole configuration and it needs no table of its own. -# Permutive DMP. `organization_id` and `workspace_id` required when enabled. -# [integrations.permutive] -# enabled = true +# InMobi Choice consent prompt. `script_url` is required, https on cmp.inmobi.com +# or cdn.inmobi.com with no user name or password. Name `cmp.inmobi` in the +# [[fetch]] entry before any middleware that writes a vendor's tag. Selected +# with [cmp] module = "inmobi". +# [cmp.inmobi] +# script_url = "https://cmp.inmobi.com/choice///choice.js?tag_version=V3" +# cmp_id = 10 + +# Permutive DMP. `organization_id` and `workspace_id` are required. +# [audience] +# module = "permutive" + +# [audience.permutive] # organization_id = "your-permutive-organization-id" # required (non-empty) # workspace_id = "your-permutive-workspace-id" # required (non-empty) # project_id = "your-permutive-project-id" # api_endpoint = "https://api.example.com" # secure_signals_endpoint = "https://secure-signals.example.com" -# lockr identity SDK. `app_id` required when enabled. -# [integrations.lockr] -# enabled = true +# lockr identity SDK. `app_id` is required. +# [identity] +# module = "lockr" + +# [identity.lockr] # app_id = "your-lockr-app-id" # required (non-empty) # api_endpoint = "https://identity.example.com" # sdk_url = "https://identity.example.com/trusted-server.js" @@ -588,9 +848,12 @@ geo_query_parameters = false # rewrite_sdk = true # DataDome bot protection. Proxies tags.js + signal-collection API first-party. -# Kept active but disabled so `ts audit` can flip `enabled` when detected. -[integrations.datadome] -enabled = false +# `ts audit` selects "datadome" in [bot-protection] when it detects DataDome; +# the table below stays yours to uncomment and fill. +# [bot-protection] +# module = "datadome" + +# [bot-protection.datadome] # sdk_origin = "https://sdk.example.com" # api_origin = "https://api.example.com" # cache_ttl_seconds = 3600 @@ -612,45 +875,55 @@ enabled = false # API validation and is stripped before the origin sees it. The credential is # loaded from the Secret Store at runtime (>= 32 bytes of high-entropy material). # Never enable in production. -# [integrations.datadome.protection_test_bypass] +# [bot-protection.datadome.protection_test_bypass] # enabled = true # credential_secret_store = "ts_secrets" # credential_secret_name = "datadome_protection_test_bypass" -# Google Publisher Tag (GPT) first-party proxy. Kept active but disabled so -# `ts audit` can flip `enabled` to true when GPT is detected. -[integrations.gpt] -enabled = false -# Kept as an active leaf so the environment override can apply; the overlay -# cannot create a missing configuration leaf. GAM attribution stays off until -# this is set true. -gam_attribution_enabled = false +# Google Publisher Tag (GPT) first-party proxy. `ts audit` selects "google" in +# [ad-tag] when it detects GPT; the table below stays yours. GAM attribution +# stays off until `gam_attribution_enabled` is set true. The environment +# overlay can only replace a leaf the document already has, and it names a +# section as written, so the variable for this leaf is +# TRUSTED_SERVER__AD-TAG__GOOGLE__GAM_ATTRIBUTION_ENABLED, which a shell's +# `export` cannot set. +# [ad-tag] +# modules = ["google"] + +# [ad-tag.google] +# gam_attribution_enabled = false # script_url = "https://ads.example.com/gpt.js" # cache_ttl_seconds = 3600 # rewrite_script = true - -# GPT runtime diagnostics browser overlay. Optional and enabled manually (not -# flipped by `ts audit`); serves a diagnostics module gated behind an activation -# query param + session cookie. -# [integrations.gpt_diagnostics] -# enabled = true - -# APS browser renderer ownership. Server-side APS behavior belongs under an -# [auction.providers] entry with profile = "aps". -# [integrations.aps] -# enabled = true -# Default `trusted_server` uses TS's opaque static renderer route. Set -# `publisher_native` only for a controlled publisher-origin friendly-frame cohort. -# rendering_mode = "trusted_server" - -# Google Tag Manager first-party proxy. Kept active but disabled so `ts audit` -# can fill container_id and flip `enabled` when GTM is detected. `container_id` -# is required when this integration is actually enabled. -[integrations.google_tag_manager] -enabled = false -# Invalid placeholder on purpose: enabling GTM without a real GTM-XXXXXX id -# fails validation. `ts audit` overwrites this when it detects a real container. -container_id = "GTM-REPLACE-ME" +# write_loader = false + +# GPT runtime diagnostics browser overlay. Selected manually rather than by +# `ts audit`, it serves a diagnostics module gated behind an activation query +# param and a session cookie. It has nothing to set, so adding +# "google.diagnostics" to [ad-tag] modules is the whole configuration. + +# Google's tag, Tag Manager container and AdSense loader written on the page +# from these settings, for a publisher whose pages cannot carry them. The +# consent mode defaults are written first, and the four consent types Google +# requires for its own services default to denied. +# Name "google-tag" in [tag] modules and "tag.google-tag" in a [[serve]] entry +# for "text/html". +# [tag.google-tag] +# analytics_tag_id = "G-REPLACE-ME" +# container_id = "GTM-REPLACE-ME" +# adsense_client_id = "ca-pub-0000000000000000" + +# Google Tag Manager first-party proxy. `ts audit` selects "google-tag-manager" +# in [tag] and writes this table with the container_id it finds when it +# detects GTM. +# `container_id` is required. +# [tag] +# modules = ["google-tag-manager"] + +# [tag.google-tag-manager] +# Invalid placeholder on purpose, because selecting GTM without a real +# GTM-XXXXXX id fails validation. +# container_id = "GTM-REPLACE-ME" # Must be a credential-free https bare origin with a literal host: no path, # query, fragment, `user:pass@` or wildcard. Targets are built by appending a # rooted path to this value, so anything beyond an origin corrupts them. @@ -667,11 +940,17 @@ container_id = "GTM-REPLACE-ME" # provides. Find the ids by grepping your rendered HTML for `gtag/js?id=`. # allowed_tag_ids = ["G-EXAMPLE1", "AW-000000", "MC-EXAMPLE1"] -# Mock ad server used for auction mediation in dev/testing. -# [integrations.adserver_mock] -# enabled = true -# endpoint = "https://adserver.example.com/mediate" -# timeout_ms = 1000 -# Map auction context keys to mediation URL query params: -# [integrations.adserver_mock.context_query_params] -# example_segments = "segments" +# Signed evidence of who operates this deployment and which build it runs, +# served at `endpoint` and at the same address with `.json`. Without this +# table the address belongs to the publisher's origin. The signing keys are +# compiled into the build from the file TRUSTED_SERVER_ATTESTATION_KEYS names +# and are never configured, so a build given none answers 503 there. +# See docs/guide/attestation.md. +# [attestation] +# Defaults to "/_ts/attestation". +# endpoint = "/_ts/attestation" +# operator = "Example Operator" +# The text signed ahead of the evidence. A verifier must use the same text. +# context = "example-attestation:v1" +# Where the page sends a reader to check the claim. Must be https. +# verify_url = "https://verifier.example/verify?host=example.com"