From 5fd582746a400e3b0ba0db3b24b8caa7999d281f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Wed, 21 Oct 2026 06:27:58 +0530 Subject: [PATCH 01/89] Add spec for server-side identity providers Define an IdentityProvider trait and coordinator so vendors such as LiveRamp, Lockr, UID2 and ID5 can resolve or capture EIDs server-side, with owner-configured precedence per source. Hold every page load to one KV write: only the navigation writes, stagers save changes into the sealed ts-eids-s cookie and removals into per-batch ts-eids-r cookies, and a failed write is never retried. Include illustrative LiveRamp, Lockr and UID2 providers, the test harness, configuration, rollout steps and the log of nine review rounds. Signed-off-by: dhruv8sh --- ...0-server-side-identity-providers-design.md | 4076 +++++++++++++++++ 1 file changed, 4076 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md diff --git a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md new file mode 100644 index 0000000000..66581d2662 --- /dev/null +++ b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md @@ -0,0 +1,4076 @@ +# Server-side identity providers + +Status: Draft, revision 10. Revised after review rounds 1 to 9. Section 17 +maps each finding to its fix. + +Issue: none filed yet. Related: [#1245](https://github.com/IABTechLab/trusted-server/issues/1245) +(LiveRamp server-to-server), [#90](https://github.com/IABTechLab/trusted-server/issues/90) +and [#353](https://github.com/IABTechLab/trusted-server/issues/353) (ID5), +[#1246](https://github.com/IABTechLab/trusted-server/issues/1246) (Lockr IDs to +KV), [#777](https://github.com/IABTechLab/trusted-server/issues/777) +(pluggable modules). + +Designed against `spec/1246-prebid-eid-fidelity` at `c84a73da7`, which is +`main` at `6ef07ae52` plus the +[Prebid EID fidelity](2026-10-18-prebid-eid-fidelity-design.md) work. Every +`file:line` reference below is to that commit. Core paths are written without +the `crates/trusted-server-core/src/` prefix, and Fastly adapter paths start +with `adapter-fastly/src/`. Prebid.js references are to version 10.26.0, read +from `crates/trusted-server-js/lib/node_modules/prebid.js/modules/`. This spec +depends on the EID fidelity branch for `merge_auction_eids`, the shared +client EID parser and the `ts-eids` cookie carrying the same set the shim +sends. + +This spec does not depend on PR #1260. Its `IdentityCapture` trait, +`identity_owner` field and `ts-eids-s` cookie are not on `main`. Where this +spec reuses a name from #1260, it defines the thing from scratch here. + +Terms used throughout: + +- **EID / UID**: an OpenRTB extended identifier, `{source, uids[]}`, and one + entry of its `uids`, `{id, atype?, ext?}` (OpenRTB 2.6 §3.2.27, §3.2.28). +- **Identity provider**: a vendor that turns request signals into an EID, + such as ID5, LiveRamp ATS, Lockr AIM or UID2. +- **Source**: the EID `source` domain a provider's IDs carry, such as + `id5-sync.com` or `liveramp.com`. It is also the key of the KV `ids` map. A + **registered** source has an `[[ec.partners]]` entry. +- **EC / KV row**: the Edge Cookie ID (`ts-ec`) and the KV identity-graph + entry keyed by it (`ec/kv_types.rs:42`). +- **HEM**: a hashed email, the SHA-256 of a normalized email address. LiveRamp + ATS, Lockr and UID2 resolve from it. It is personal data, not an anonymous + value. +- **Page load**: one top-level document navigation and every request the page + makes because of it: `/auction` calls, SPA view changes reported to + `/_ts/page-bids`, and integration proxy calls. +- **Writer request**: the one request in a page load allowed to write KV + (section 5.1). +- **Stager**: every other request. It reads cookies and KV and saves changes + into `ts-eids-s` for the next writer request to write. +- **KV gap**: the registered sources a browser sent to `/auction` that the + visitor's KV row does not have yet. +- **Hold mode / after-send mode**: whether a provider is called before the + auction, with the auction waiting for it, or after the response has been + sent. +- **Orphan recovery**: what TS does when a visitor's EC cookie has no KV row + (`ec/finalize.rs:282`). +- **`ts-eids`**: the existing first-party cookie the Prebid shim writes with + the browser's EIDs. Unchanged by this spec. +- **`ts-eids-s`**: a new server-owned cookie, named `__Host-ts-eids-s`, that + caches provider-collected IDs and holds staged changes. Defined in section + 6.2. + +## 1. Problem statement + +Trusted Server (TS) collects EIDs three ways today. Browser modules resolve +IDs that the page forwards, partners push UIDs to batch sync, and pull sync +calls partners from the server. Only pull sync calls a provider from the +server, and it speaks one fixed protocol. + +### P1. Browser-only collection fails where browsers block vendors + +Each provider ships a browser module, and TS forwards what those modules +produce. Safari's tracking protection blocks requests to some vendor hosts. +#1245 reports Prebid's `identityLink` module failing this way on Safari, +leaving no LiveRamp envelope in the auction. Calling providers from the edge +avoids the browser request. + +This spec builds that server-side path. In v1 it helps: + +- providers that need no hashed email, such as ID5 +- providers whose traffic TS already proxies, such as Lockr +- generic pull partners +- any owning provider that can refresh an ID a browser module obtained + earlier, so the ID survives when the module later fails in that browser + +It does not fix the #1245 Safari case by itself. LiveRamp's initial +resolution needs a hashed email, and v1 has no way to obtain one (section +2.2). A Safari visitor whose browser module never obtained an envelope has +nothing to refresh. + +### P2. The only server-side path is one fixed protocol + +`ec/pull_sync.rs` calls partners after the response is sent. Its protocol is +fixed: `GET ?ec_id=` with a bearer token +(`pull_sync.rs:255`, `:368-371`), response `{"uid":"..."}` +(`pull_sync.rs:54-57`). It fills missing UIDs only and never refreshes +(`pull_sync.rs:7-9`). + +None of the providers this spec targets speaks that protocol: + +- **ID5** takes a POST body with a partner number, device signals, the page + URL and consent. It returns up to three IDs (`id5-sync.com`, an EUID ID and + `true-link-id5-sync.com`) and a `signature` the caller sends back on its + next call (`id5IdSystem.js:40-41`, `:130-170`, `:454-487`). +- **LiveRamp ATS** takes a placement ID, an identifier type and value, + consent parameters and the end user's IP. It returns one of several + envelope products, envelopes expire after 15 or 30 days, and refresh is a + different call that takes the old envelope (#1245 §9, §10). +- **UID2** returns an advertising token for bidders and a refresh token that + must never reach them. +- **Lockr**'s API traffic already passes through TS's proxy + (`integrations/lockr.rs:179-237`). Its IDs can be read from the responses, + with no outbound call. + +Each provider needs its own request building, response parsing, error +handling and, for some, private state kept between calls. Without a shared +contract, each integration would repeat consent revalidation, CAS handling, +rate limits and timing. #1245 §6 rules out a private KV writer per vendor. + +### P3. A page load writes KV several times + +Today a page load can write the KV row once per step: + +| Step | Request | Write | Code | +| --------------------------------------- | ----------------------------------------- | ------------------ | ----------------------------------------------------------- | +| New visitor gets an EC | publisher navigation, during routing | `create_if_absent` | `adapter-fastly/src/app.rs:845-857`, `ec/mod.rs:424` | +| New visitor gets an EC | integration-proxy navigation | `create_if_absent` | `integrations/registry.rs:1046-1050` | +| Browser EIDs from `ts-eids` or sharedId | navigation, finalize | CAS update | `ec/finalize.rs:83`, `:183` | +| Prebid body EIDs | each `/auction` | CAS update | `auction/endpoints.rs:298-303`, `finalize.rs:183` | +| `ts-eids` on an SPA view | each `/_ts/page-bids` | CAS update | `publisher.rs:6925`, `finalize.rs:183` | +| Pull sync results | after send, on requests with a loaded row | CAS update | `adapter-fastly/src/main.rs:507-517`, `ec/pull_sync.rs:290` | +| Cluster size | `/_ts/api/v1/identify`, once per EC | CAS update | `ec/identify.rs:107`, `ec/kv.rs:1484`, `:1524-1529` | + +Ingest runs only on requests that set an EID sync source: navigation, +`/auction`, `/_ts/page-bids` and new ECs (`ec/mod.rs:98-111`). Asset routes +skip EC finalize entirely (`adapter-fastly/src/app.rs:834-835`). Pull sync +runs after any response whose request loaded a consenting row +(`pull_sync.rs:86`). + +Each of these skips the write when nothing changed, so most are no-ops, but +each can write. A new visitor can cost three writes on the navigation alone, +plus one for each `/auction` and each SPA view. Adding providers naively adds +one more write each. KV writes cost money, Fastly limits how often one key +can be written, and each extra write is another CAS race between tabs. + +## 2. Goals and non-goals + +### 2.1 Goals + +- **G1.** A trait any identity provider implements to be called from the + server. LiveRamp, Lockr, ID5, UID2 and generic pull partners all fit it, + including providers that keep private state between calls. +- **G2.** Providers never do HTTP, KV or cookie work themselves. One + coordinator owns eligibility, consent, transport limits and write-back. +- **G3.** Operators choose per provider between hold mode and after-send + mode. +- **G4.** At most one browser-driven KV write per page load, and a write + that fails is not retried. This is strict. The only page-load exception is + a consent withdrawal's tombstone (section 5.1). Writers outside page loads + keep their own writes: batch sync, the admin endpoints and the deferred + deletion framework (sections 2.2, 5.5). +- **G5.** Provider-collected IDs are stored in both a first-party cookie + (`ts-eids-s`) and KV, and the cookie is read first. +- **G6.** Each source has at most one owning provider, configured by the + operator. A valid ID the owner wrote wins in the auction over the + browser's. +- **G7.** IDs that expire (LiveRamp envelopes) are refreshed, and are never + sent to bidders or returned by `/_ts/api/v1/identify` after expiry. +- **G8.** Today's pull sync becomes one implementation of the trait. Its + protocol and fill-missing contract are unchanged. Section 10.4 lists the + behaviour that does change. +- **G9.** The trait can model a provider that needs a HEM, even though v1 + has no way to obtain one. +- **G10.** No browser EID that reaches KV today is lost. Everything today's + `/auction` ingest writes still reaches KV, through the next writer request. + What today's `/_ts/page-bids` ingest writes comes from `ts-eids`, which the + next navigation ingests; only an SPA session longer than that cookie's + one-day lifetime can lose such an ID. A body-only ID beyond the `ts-eids` + trim is lost only when KV-gap entries alone exceed the `ts-eids-s` cap + and the browser sends no further `/auction` before the next navigation. +- **G11.** Writing a provider is easy, and extending the trait does not + break one. A simple provider is one struct, one descriptor and two short + methods (section 4.2). +- **G12.** Removals win. An ID removed from KV (opt-out, invalidation, + withdrawal, deletion) stops reaching bidders within a bounded time, even + from the cookie. A removal staged by a stager stops that browser's + bidstream at once and reaches KV at the next writer navigation; a client + that never makes one keeps it in `ts-eids-r` until it does. + +### 2.2 Non-goals + +- **HEM intake.** How a hashed email reaches TS (#1245 §4 options A to D) + needs its own spec. The trait has a typed slot for it. In v1 the slot is + always empty and providers that need it are skipped. +- **Concrete vendor implementations.** ID5, LiveRamp and Lockr each get + their own implementation spec. Section 12 shows how each maps onto the + trait, to prove the trait fits. +- **Other adapters.** KV identity runs only on Fastly + (`publisher.rs:6896-6899`). Axum, Cloudflare and Spin get this design when + EC identity is ported to them (#777 rows 52 to 54). +- **Cross-browser linking.** An ID in one browser's EC row does not appear in + another browser's row (#1245 §12). +- **LiveRamp ATS Direct segments** and GAM targeting. +- **Batch sync and admin writes.** They keep their own writes. Section 7.2 + changes only how batch sync treats an owned source. +- **Removing `ts-eids` or `x-ts-eids`.** Both stay. + +## 3. Background + +### 3.1 Where EIDs come from today + +1. **Browser modules.** Prebid's userId modules resolve IDs in the browser. + The `trustedServer` bid adapter sends them in the `/auction` body and the + shim copies the same set into `ts-eids` (EID fidelity spec §4.2, §4.3). +2. **Pull sync.** After the response is sent, TS calls each pull-enabled + `[[ec.partners]]` entry whose UID is missing (`ec/pull_sync.rs:116`). +3. **Batch sync.** Partners push UIDs to `/_ts/api/v1/batch-sync` + (`ec/batch_sync.rs`). + +### 3.2 How the auction builds `user.eids` + +`handle_auction` takes the client set from the body, else from `ts-eids` +(`auction/endpoints.rs:290-296`). It keeps the full body set for finalize +because the cookie is size-capped (`endpoints.rs:298-303`). It reads the KV +row once when a partner registry exists (`endpoints.rs:307-311`), resolves +KV IDs for registered sources (`endpoints.rs:444`), and merges the two with +`merge_auction_eids` (`endpoints.rs:505`). The navigation and +`/_ts/page-bids` merge in `publisher.rs:5547-5568`. The merge forwards +browser EIDs as sent and adds a KV EID only for a source the browser did not +send. The result is consent-gated and reported in `x-ts-eids` +(`ec/eids.rs:17`, 4 KB cap). + +Browser ingest is fill-missing: a different stored value is never replaced, +because a browser value carries no freshness signal (`ec/kv.rs:146-154`). + +### 3.3 The existing provider pattern + +`AuctionProvider` (`auction/provider.rs:84`) is the house pattern for a +pluggable vendor. The provider builds its request and parses the response, +and the orchestrator owns dispatch, timeouts and response handling. This spec +follows the same split. + +### 3.4 Platform and timing facts this design depends on + +- **KV identity is Fastly-only.** Axum, Cloudflare and Spin pass no KV graph + (`publisher.rs:6896-6899`) and run no EC finalize or EC generation. +- **New ECs are created before the cookie goes out.** Generation activates an + ID only after an Add-confirmed create and re-mints on a collision + (`ec/mod.rs:407-455`). Finalize refuses a generated cookie without a + backing row (`ec/finalize.rs:145-149`). This spec keeps that. +- **After-send work starts after the last byte.** `send_edgezero_response` + streams the whole body before returning (`adapter-fastly/src/main.rs:699-702`), + and pull sync runs after that (`main.rs:510-517`). The response has been + moved into the send by then, so after-send code cannot see its status + unless it is captured first (`main.rs:512-517`). +- **State crosses into finalize only through `EcFinalizeState`** + (`adapter-fastly/src/app.rs:318-332`, inserted at `:535`). Response + extensions require `Clone + Send + Sync`, which a pending platform request + is not (`platform/http.rs:178-183`), so an in-flight provider call cannot + outlive the router. +- **The navigation auction is dispatched before the origin fetch** + (`publisher.rs:4886-4889`). The origin overlaps the EC preload only when the + preload runs, the request has no template-cache key and the platform + supports concurrent fan-out and pending streaming responses + (`publisher.rs:4832-4844`). +- **Navigation detection.** `is_navigation_request` checks + `Sec-Fetch-Dest: document`, which excludes iframes, and falls back to + `Accept: text/html`, which a `fetch()` can also send (`http_util.rs:93-118`). + `is_prefetch_request` detects prefetch and prerender and is `pub(crate)` + in core (`publisher.rs:3200`). EC classification must wait until + pre-route filters allow dispatch (`ec/mod.rs:91-96`). +- **Orphan recovery rotates.** When a cookie's row is proven absent + (`finalize.rs:371-400`), recovery mints a new EC ID and sets a new cookie + (`finalize.rs:304`, `:333`), before the response is sent. Only the + publisher fallback may rotate (`adapter-fastly/src/app.rs:885-890`). +- **Today's navigation preload finds orphans.** Every returning GET + navigation preloads the row unless a valid pull-complete marker lets it + skip (`publisher.rs:3110-3117`, `:4819-4831`). +- **Withdrawal tombstones existing rows only,** with a CAS write of up to five + attempts and then a completion-marker write, per withdrawn ID + (`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:490`, `:509-525`). +- **Cookies make responses private.** Any response with `Set-Cookie` gets + `Cache-Control: private, max-age=0` and loses its edge-cache headers + (`response_privacy.rs:122-145`). Returning-visitor navigations set no EC + cookie today (`finalize.rs:124`), though they can set or expire the + pull-complete marker (`pull_sync_marker.rs:132-155`). +- **The rate limiter counts in 60-second windows** and increments on every + check that is under the limit. Any hourly limit below 60 becomes one call + per minute (`ec/rate_limiter.rs:19-27`, + `adapter-fastly/src/rate_limiter.rs:41-74`). Pull sync keys it on `ec_hash` + (`pull_sync.rs:373`), the HMAC of the client IP (`generation.rs:83-93`), so + everyone behind one address shares one budget. +- **The shim's `ts-eids` is small.** It is trimmed to 3,072 encoded bytes, + lives one day and has no EID-level `ext` (EID fidelity spec §4.3). The + server accepts up to 8 KB (`ec/prebid_eids.rs:24`). +- **Prebid Server receives the browser's cookies.** The Prebid transport + copies the inbound `Cookie` header into every Prebid Server call and strips + only consent cookies, only in `OpenrtbOnly` mode (`integrations/prebid.rs:2091-2108`, + `consent_config.rs:137-150`). The publisher proxy forwards the `Cookie` + header to the origin unchanged (`publisher.rs:3124-3147`). + +## 4. Architecture + +### 4.1 Overview + +```mermaid +flowchart TD + subgraph Writer["Writer request: a top-level navigation"] + A[Read ts-ec, consent, ts-eids, ts-eids-s] --> B{Cookies answer
everything?} + B -- no --> C[KV snapshot read] + B -- yes --> D + C --> D[Pick eligible providers] + D --> E[Hold-mode providers, if the
navigation runs an auction] + E --> F[Finalize: queue changes,
set cookies only if needed] + F --> G[Send response] + G --> H[Returning EC: re-read live row] + H --> I[After-send providers] + I --> J[Apply IdentityMutation:
one conditional write] + end + subgraph Stagers["Stagers: /auction, /_ts/page-bids, integration proxy"] + K[Read cookies, and KV where
the route reads it today] --> L[Hold-mode providers,
passive capture] + L --> M[Save results and the KV gap
into ts-eids-s; never write KV] + end +``` + +The design has four parts: + +1. **`IdentityProvider` trait** (sections 4.2 to 4.6). One per vendor. +2. **`IdentityCoordinator`** (section 4.7). Shared. Chooses providers, + dispatches them, turns results into changes. +3. **`IdentityMutation`** (section 5.4). A request-scoped accumulator. Every + KV change in the writer request is added to it, and it is applied with one + write. +4. **`ts-eids-s` cookie** (section 6.2). The cookie-first copy of + provider-collected IDs and the place stagers save changes. + +New code lives in `crates/trusted-server-core/src/ec/identity/`: + +| File | Contents | +| ----------------- | ---------------------------------------------------------------------- | +| `mod.rs` | Module docs, re-exports | +| `provider.rs` | The `IdentityProvider` and `IdentityProviderConfig` traits | +| `descriptor.rs` | `ProviderDescriptor`, `Resolves`, `Captures`, `Input`, `ConsentPolicy` | +| `input.rs` | `ResolveInput`, `CaptureInput`, `StoredId`, `HashedEmail`, `UnixTime` | +| `exchange.rs` | `ProviderRequest`, `ProviderResponse` | +| `resolution.rs` | `Resolution`, `ResolvedId`, `IdentityProviderError` | +| `coordinator.rs` | Eligibility, dispatch, circuit breaker, result-to-change map | +| `mutation.rs` | `IdentityMutation` and its merge into a live `KvEntry` | +| `cookie.rs` | `ts-eids-s` and `ts-eids-r`: seal, open, encode, decode, size caps | +| `generic_pull.rs` | `GenericPullProvider`, today's pull sync on the trait | +| `config.rs` | `[ec.identity]` settings and ownership validation | +| `test_support.rs` | `ProviderHarness`, a fake HTTP client and a write-counting KV store | + +### 4.2 The trait + +The trait follows five rules: + +- **One required method.** A provider implements `descriptor`. It adds a + capability by declaring it in the descriptor and overriding the matching + methods. Every other method has a default body. +- **Declarative.** What a provider is, may write, needs and must respect + lives in one `ProviderDescriptor`. It is built from config with no + secrets, so `ts config validate` and the running coordinator read the same + data. +- **No I/O.** The coordinator sends requests, buffers, decodes and caps + responses, and does all KV and cookie work. Providers only build requests + and read buffered bytes, so the trait is synchronous and needs no + `async_trait`. `AuctionProvider` parses asynchronously so that it can drain + streaming bodies. The coordinator buffers instead, because it has to cap + response bytes anyway. +- **Pure.** `request`, `response` and `capture` have no side effects and no + hidden inputs. Randomness comes only from `ResolveInput::random_bytes`, + and anything `response` needs from `request` travels in the call context + (section 4.5). The coordinator may call `request` twice in one request, + first as a dry run (section 5.2). `input.is_dry_run()` lets a provider skip + expensive work in the dry run. A provider with private state gets no dry + run. +- **Safe defaults.** The default consent policy is the strictest one, a + provider can read only what it declared, and a capability declared but not + implemented fails loudly in tests (section 13). + +```rust +/// A vendor that turns server-side request signals into EIDs. +/// +/// Only [`descriptor`](Self::descriptor) is required. A provider adds a +/// capability by declaring it in its descriptor and overriding the matching +/// methods: +/// +/// - active resolution: [`Resolves`], plus [`request`](Self::request) and +/// [`response`](Self::response) +/// - passive capture: [`Captures`], plus [`capture`](Self::capture) +/// +/// Methods added to this trait later get default bodies, so existing +/// providers keep compiling. No method takes generic parameters, so the +/// trait stays object-safe. Every default body returns the +/// [`IdentityProviderError::not_implemented`] marker, so a capability that is +/// declared but not implemented is caught by the harness. +pub trait IdentityProvider: Send + Sync { + /// What this provider is, what it may write, what it needs and which + /// consent it requires. + fn descriptor(&self) -> &ProviderDescriptor; + + /// Builds the call for this request, or `None` to skip it this time. + /// + /// Skipping is not a failure: the coordinator waits `refresh_after_sec` + /// before asking again (section 8). + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::UndeclaredInput`] when it reads + /// something its descriptor did not declare, or another + /// [`IdentityProviderError`] when the request cannot be built. + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let _ = input; + Err(IdentityProviderError::not_implemented("request")) + } + + /// Turns the buffered response to [`request`](Self::request) into a + /// resolution. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Rejected`] for a refusal and + /// [`IdentityProviderError::Malformed`] for a body it cannot parse. + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let _ = (response, input); + Err(IdentityProviderError::not_implemented("response")) + } + + /// Reads IDs from a response TS proxied for this provider, with no + /// outbound call. [`Resolution::no_match`] means the response held + /// nothing to capture, and costs nothing: capture never backs off. + /// + /// # Errors + /// + /// Returns an [`IdentityProviderError`] only for a provider bug. A body + /// it does not recognise is `Ok(Resolution::no_match())`, not an error. + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + let _ = input; + Err(IdentityProviderError::not_implemented("capture")) + } +} + +/// Implemented by each identity integration's config type. +/// +/// Builds the provider's descriptor from config alone, with no secrets, so +/// `ts config validate` checks ownership against the same descriptor the +/// running provider uses. +pub trait IdentityProviderConfig: IntegrationConfig { + /// Describes the provider this config enables. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + fn build_descriptor(&self) -> Result>; +} +``` + +A provider's constructor calls its config's `build_descriptor()` and stores +the result, so the validated descriptor and the running one are the same +value. The two methods have different names so that a type implementing both +traits never hits an ambiguous call. `IntegrationConfig` is the existing +config trait (`settings.rs:235`), so identity configs load like every other +integration config. + +**How each change stays additive.** Vendor providers live in +`trusted-server-core` today, and `#[non_exhaustive]` has no effect inside its +defining crate. So additivity rests on construction rules, with +`#[non_exhaustive]` as extra protection for providers that move into their +own crates under #777: + +| Change | Why existing providers keep compiling | +| ----------------------- | ---------------------------------------------------------------------------------------------------------- | +| A new capability | A new trait method with a default body, and a new descriptor builder method | +| A new input | A new `Input` variant and a new accessor. Providers list inputs; they never match on `Input` | +| A new field on any type | Every public struct has private fields and is built through constructors or builders | +| A new outcome | A new `Resolution` constructor. Providers build resolutions, never match on them | +| A new consent signal | Providers keep compiling. By design they fail closed: `strict()` blocks the signal until a vendor opts out | +| A new resolve kind | Kinds are opt-in: a provider is dispatched only for the kinds it lists (default `Initial`, `Refresh`) | +| A new error | Errors are built through constructors such as `IdentityProviderError::rejected(status)` | + +Providers compare kinds with predicates (`input.is_refresh(&source)`) rather +than `match`. Operator settings (`[ec.identity.providers.]`: mode, +timeouts, source narrowing, caps) are applied by the coordinator on top of +the descriptor. Providers never read them. + +**Conventions for the listings below.** Every builder method and getter is +`#[must_use]`. `Resolves`, `Captures` and `EidProvenance` implement +`Default`. `Input`, `Access`, `ResolveKind` and `UsOptOut` derive +`Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display`. +`When` derives `Debug, Clone, Copy, PartialEq, Eq` only, because `Duration` +has no `Display`. `ProviderRequest`, `StoredId` and `HashedEmail` derive +`Debug, Clone`; `HashedEmail`'s `Debug` prints a redacted marker. +`StateChange` derives `Debug, PartialEq, Eq`. `ProviderDescriptor`, +`Resolves`, `Captures`, `ConsentPolicy` and `EidProvenance` derive +`Debug, Clone, PartialEq, Eq`, so tests can compare descriptors. `UnixTime` derives +`Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord`. `IdentityProviderId` +and `SourceDomain` derive `Debug, Clone, PartialEq, Eq, Hash, PartialOrd, +Ord, derive_more::Display` and implement `FromStr` (with +`type Err = Report`), `Serialize` and `Deserialize`. +Every fallible function documents `# Errors`, and every public item has a +doc comment. The listings abbreviate these. + +### 4.3 The descriptor + +```rust +let descriptor = ProviderDescriptor::builder("exampleid".parse()?) + .with_source("exampleid.example.com".parse()?) + .with_resolves( + Resolves::new() + .with_endpoint("https://api.exampleid.example.com/v1/fetch") + .with_needs([Input::ClientIp, Input::UserAgent]) + .with_wants([Input::PageUrl]), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(9999)) + .build()?; +``` + +```rust +impl ProviderDescriptor { + pub fn builder(id: IdentityProviderId) -> ProviderDescriptorBuilder; + pub fn id(&self) -> &IdentityProviderId; + pub fn sources(&self) -> &[SourceDomain]; + /// The first source added. Every built descriptor has one. + pub fn primary_source(&self) -> &SourceDomain; + /// Finds a declared source by a vendor's spelling of it, normalized. + pub fn source_named(&self, name: &str) -> Option<&SourceDomain>; + pub fn resolves(&self) -> Option<&Resolves>; + /// The endpoint declared with `Resolves::with_endpoint`, if any. + pub fn endpoint(&self) -> Option<&Url>; + pub fn captures(&self) -> Option<&Captures>; + pub fn consent_policy(&self) -> &ConsentPolicy; + pub fn provenance(&self, source: &SourceDomain) -> Option<&EidProvenance>; + /// Whether IDs of `source` are declared as expiring. + pub fn is_expiring(&self, source: &SourceDomain) -> bool; +} + +impl ProviderDescriptorBuilder { + /// Adds a source this provider may write. Repeatable. + pub fn with_source(self, source: SourceDomain) -> Self; + /// Declares active resolution. + pub fn with_resolves(self, resolves: Resolves) -> Self; + /// Declares passive capture. + pub fn with_captures(self, captures: Captures) -> Self; + /// Replaces the default policy, [`ConsentPolicy::strict`]. + pub fn with_consent_policy(self, policy: ConsentPolicy) -> Self; + /// Marks the IDs of every declared source as expiring (section 6.1). + pub fn with_expiring_ids(self) -> Self; + /// Marks the IDs of one declared source as expiring, for a provider + /// whose sources differ. + pub fn with_expiring_source(self, source: &SourceDomain) -> Self; + /// EID-level fields for a declared source. The coordinator adds them to + /// every server-side value of that source this provider wrote. No KV + /// field stores them. + pub fn with_provenance(self, source: &SourceDomain, provenance: EidProvenance) -> Self; + /// # Errors + /// + /// Returns `Config` when the descriptor has no source, no capability, a + /// resolver with no endpoint or host, an endpoint that is not an `https` + /// URL, a host that is not a bare hostname, or provenance or an + /// expiring mark for a source it does not declare. + pub fn build(self) -> Result>; +} + +impl Resolves { + pub fn new() -> Self; + /// The endpoint the provider calls, as config spells it. `build()` + /// parses it and requires `https`. Its host becomes an allowed host. + pub fn with_endpoint(self, endpoint: &str) -> Self; + /// Adds another host the provider may call. Repeatable. + pub fn with_host(self, host: &str) -> Self; + /// Inputs required for every kind. Without them the provider is skipped. + pub fn with_needs(self, inputs: impl IntoIterator) -> Self; + /// Inputs required for one kind only. + pub fn with_needs_for(self, kind: ResolveKind, inputs: impl IntoIterator) -> Self; + /// Inputs passed when available, never required. + pub fn with_wants(self, inputs: impl IntoIterator) -> Self; + /// The kinds this provider handles. Default: `Initial` and `Refresh`. + pub fn with_kinds(self, kinds: impl IntoIterator) -> Self; + /// This provider keeps private state, so it runs only on the writer + /// request, where state can be stored (section 5.3). + pub fn with_private_state(self) -> Self; +} + +impl Captures { + pub fn new() -> Self; + /// Offers proxied responses to exactly this method and path. Repeatable. + pub fn with_path(self, method: Method, path: &str) -> Self; + /// Offers proxied responses whose path starts with `prefix`. Repeatable. + pub fn with_path_prefix(self, prefix: &str) -> Self; + /// JSON pointers to consent strings in the proxied request's JSON body + /// (TCF, GPP or US Privacy, detected by format). The coordinator + /// extracts and decodes only these fields, and gates capture by the + /// stricter of the cookie consent and the body consent. The provider + /// never sees the body. + pub fn with_request_consent(self, pointers: impl IntoIterator) -> Self; +} + +/// EID-level fields (OpenRTB 2.6 §3.2.27) that bidders read, such as who +/// inserted an ID and how it was matched. +impl EidProvenance { + pub fn new() -> Self; + pub fn with_inserter(self, inserter: &str) -> Self; + pub fn with_matcher(self, matcher: &str) -> Self; + pub fn with_mm(self, mm: i32) -> Self; + + pub fn inserter(&self) -> Option<&str>; + pub fn matcher(&self) -> Option<&str>; + pub fn mm(&self) -> Option; +} + +/// An input a provider can ask for. Consent strings, the provider's stored +/// IDs, the publisher domain, the time and the declared endpoint are always +/// available. Private state is available to providers declared +/// `with_private_state()`. +#[non_exhaustive] +pub enum Input { + ClientIp, + UserAgent, + /// The top-level page URL without query string or fragment. + PageUrl, + /// Country and subdivision only. + Region, + HashedEmail, +} + +/// Something a provider read without declaring it. +#[non_exhaustive] +pub enum Access { + Input(Input), + Endpoint, + PrivateState, +} + +impl ConsentPolicy { + /// The default. Blocked when any consent signal present cannot be + /// decoded. Where TCF applies, requires Purposes 1 and 4. Blocked by every + /// US opt-out TS decodes (sale, sharing, targeted advertising, and the + /// GPC and known-child fields of GPP sections), by `Sec-GPC: 1` and by + /// `DNT: 1`. + pub fn strict() -> Self; + /// Also requires these TCF purposes, where TCF applies. + pub fn with_tcf_purposes(self, purposes: impl IntoIterator) -> Self; + /// Requires vendor consent for this IAB Global Vendor List ID. + pub fn with_tcf_vendor(self, vendor_id: u16) -> Self; + /// Relaxations. Each needs a documented legal basis in the vendor spec. + pub fn allowing_dnt(self) -> Self; + pub fn allowing_gpc(self) -> Self; + pub fn allowing_us_opt_out(self, opt_out: UsOptOut) -> Self; +} + +#[non_exhaustive] +pub enum UsOptOut { + Sale, + Sharing, + TargetedAdvertising, +} + +impl SourceDomain { + pub fn as_str(&self) -> &str; + /// Whether a vendor's spelling of a source names this one, after + /// normalization. + pub fn matches(&self, name: &str) -> bool; +} +``` + +The full EC ID is not an `Input`. A public variant would let any provider +declare it, and only generic pull may receive it. `Resolves` has a +`pub(in crate::ec::identity)` method, `with_ec_id()`, and `ResolveInput` an +accessor with the same visibility. Only `generic_pull.rs` uses them. + +Every capture path and prefix must lie under the registering integration's +own proxy routes (`IntegrationProxy::routes`, `integrations/registry.rs:316`). +`IntegrationRegistration::build()` cannot fail (`integrations/registry.rs:720-722`), +so the check runs in `IntegrationRegistry::with_plan` +(`integrations/registry.rs:842-865`), which already returns an error and +merges every registration. The error names the integration. + +`IdentityProviderId` is separate from `crate::auction::ProviderId` +(`auction/plan.rs:24-77`) but follows the same pattern. `FromStr` accepts +`^[a-z][a-z0-9_-]{0,62}$`, so an ID can be a bare TOML key and can equal an +integration ID that uses `_`. `SourceDomain::from_str` goes through +`normalize_partner_source_domain` (`ec/partner.rs:20`), mapping its `String` +error to `Config`, so `"…".parse()?` composes inside `build_descriptor`. + +```rust +impl ProviderConsent { + pub fn gdpr_applies(&self) -> bool; + pub fn raw_tc_string(&self) -> Option<&str>; + pub fn raw_gpp_string(&self) -> Option<&str>; + pub fn gpp_section_ids(&self) -> Option<&[u16]>; + pub fn raw_us_privacy(&self) -> Option<&str>; +} +``` + +### 4.4 Inputs + +The coordinator gives a provider only what its descriptor declared. When the +sources in one request have different kinds, the inputs of every kind +present are available, and the provider uses `is_initial` or `is_refresh` to +decide per source. A required input is read with `?` and a wanted one with +`.ok()`: + +```rust +let ip = input.client_ip()?; +let page = input.page_url().ok(); +``` + +Accessors fail in two ways: + +- **`UndeclaredInput { access }`**: the descriptor never declared the input, + the endpoint or private state. That is a provider bug. `.ok()` would hide + it, so every accessor also records an undeclared read on the + `ResolveInput`. After `request` or `response` returns, the coordinator logs any recorded + read as `Config`, and the harness fails the test (section 13), whatever + the provider did with the error. +- **`InputUnavailable { input }`**: the input was declared but is missing + this time, for example a wanted input or one declared only for another + kind. That is a normal skip, handled like `request` returning `None`. + +```rust +impl ResolveInput<'_> { + /// The sources to resolve this time, each with its own kind. + pub fn sources(&self) -> impl Iterator + '_; + pub fn is_initial(&self, source: &SourceDomain) -> bool; + pub fn is_refresh(&self, source: &SourceDomain) -> bool; + /// The descriptor's primary source, for single-source providers. + pub fn primary_source(&self) -> &SourceDomain; + /// The stored record for one of this provider's sources, valid or + /// expired. + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId>; + /// This provider's private state. The raw JSON is loaded once per + /// request and decoded on each call. `Ok(None)` when there is none. + /// + /// # Errors + /// + /// `InvalidState` when the stored state does not decode as `T`, for + /// example after a schema change. The failed decode is recorded on the + /// `ResolveInput`, like an undeclared read, and the coordinator clears + /// the stored state whatever the provider returns, so the next call + /// starts fresh (section 8). Without `with_private_state()` it is + /// `UndeclaredInput`. + pub fn state(&self) -> Result, Report>; + /// True during the dry run of section 5.2. The request is discarded, so + /// a provider may skip expensive work and return `Some` early. Always + /// false for a provider declared `with_private_state()`, which gets no + /// dry run. + pub fn is_dry_run(&self) -> bool; + /// The endpoint declared with `Resolves::with_endpoint`. + pub fn endpoint(&self) -> Result<&Url, Report>; + pub fn consent(&self) -> &ProviderConsent; + pub fn publisher_domain(&self) -> &str; + pub fn now(&self) -> UnixTime; + /// Random bytes from the platform's generator, the one EC generation + /// already uses (`ec/generation.rs:52-56`). Fixed by the harness, so + /// tests are deterministic. + pub fn random_bytes(&self) -> [u8; N]; + + pub fn client_ip(&self) -> Result>; + pub fn user_agent(&self) -> Result<&str, Report>; + pub fn page_url(&self) -> Result<&Url, Report>; + pub fn region(&self) -> Result<&Region, Report>; + pub fn hashed_email(&self) -> Result<&HashedEmail, Report>; +} + +impl CaptureInput<'_> { + /// The proxied request path, without query string. + pub fn path(&self) -> &str; + pub fn response(&self) -> &ProviderResponse; + pub fn primary_source(&self) -> &SourceDomain; + pub fn source_named(&self, name: &str) -> Option<&SourceDomain>; + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId>; + /// The stricter of the cookie consent and the request-body consent. + pub fn consent(&self) -> &ProviderConsent; + pub fn publisher_domain(&self) -> &str; + pub fn now(&self) -> UnixTime; +} + +#[non_exhaustive] +pub enum ResolveKind { + /// No stored record for the source. + Initial, + /// A stored record exists, valid or expired, and is due (section 4.7, + /// step 4). The provider decides whether it can refresh an expired one; + /// if not, it returns `None` for that source. + Refresh, +} + +impl StoredId { + pub fn uid(&self) -> &str; + pub fn atype(&self) -> Option; + /// UID-level ext, as bidders read it in `uids[].ext`. + pub fn ext(&self) -> Option<&serde_json::Value>; + /// The provider that wrote it. `None` for browser ingest and legacy rows. + pub fn writer(&self) -> Option<&IdentityProviderId>; + pub fn acquired_at(&self) -> Option; + pub fn expires_at(&self) -> Option; + pub fn refresh_at(&self) -> Option; + pub fn is_expired(&self, now: UnixTime) -> bool; +} + +impl Region { + /// ISO 3166-1 alpha-2, upper case, for example `"US"`. + pub fn country(&self) -> &str; + /// The ISO 3166-2 subdivision without the country prefix, upper case, + /// for example `"CA"`. `None` when the platform does not know it. + pub fn subdivision(&self) -> Option<&str>; + /// Whether this is `country` and `subdivision`. + pub fn is(&self, country: &str, subdivision: &str) -> bool; +} + +impl HashedEmail { + pub fn as_bytes(&self) -> &[u8; 32]; + /// Lowercase hex, as LiveRamp takes it. + pub fn to_hex(&self) -> String; + /// Standard base64, as UID2 takes it. + pub fn to_base64(&self) -> String; +} + +impl UnixTime { + pub fn from_secs(secs: u64) -> Self; + /// For vendors such as Lockr and UID2 that send milliseconds. + pub fn from_millis(millis: u64) -> Self; + pub fn as_secs(self) -> u64; +} + +impl Add for UnixTime { + type Output = UnixTime; +} +``` + +`ProviderConsent` is a view of the request's `ConsentContext` +(`consent/types.rs:109-148`) with the raw strings providers forward and +`gdpr_applies`. It leaves out the decoded detail and `jurisdiction`. +Providers never decide consent: the coordinator has already applied the +descriptor's policy. The raw GPP string and its section IDs still show which +US state section applies, so withholding `jurisdiction` hides only TS's +decoded reading of it. + +Where each input comes from. Phase 1 captures the user agent, DNT and the +page URL into `EcContext`, because after-send code no longer has the request +(section 3.4): + +- **`ClientIp`**: `services.client_info().client_ip`, the platform's address + (as `publisher.rs:5574-5578` does), never a request header and never the + normalized EC IP. +- **`PageUrl`**: the navigation URL; on `/auction`, the Referer as the + auction sanitizer reduces it (`auction/formats.rs:106-155`); on + `/_ts/page-bids`, the request origin plus the normalized `path` parameter + (`publisher.rs:6978-6989`). Always without query string or fragment. A + path can still carry identifiers on some sites, so `PageUrl` is a wanted + input for most providers, never needed. +- **`Region`**: from the request's `GeoInfo` (`adapter-fastly/src/app.rs:423-432`). + `GeoInfo.region` is a free-form `Option` (`platform/types.rs:32`), + so the coordinator normalizes it to the ISO 3166-2 subdivision and upper + case. City, coordinates, metro and ASN are never passed. + +`HashedEmail` holds the 32 raw bytes of a SHA-256 digest. Its `Debug` output +is redacted. No code in v1 constructs one. + +The coordinator supplies `now` from `checked_current_timestamp()` +(`ec/mod.rs:736-748`) and skips identity work for the request when the clock +fails, because a zero time would make every expiry check pass. + +### 4.5 Requests, responses and results + +```rust +impl ProviderRequest { + pub fn get(url: &Url) -> Self; + pub fn post_json(url: &Url, body: &impl Serialize) -> Result>; + /// A POST with a `text/plain` body, for APIs that send base64 text. + pub fn post_text(url: &Url, body: &str) -> Self; + /// Any method, for APIs the helpers above do not fit. `get`, `post_json` + /// and `post_text` are shorthands for this. + pub fn new(method: Method, url: &Url) -> Self; + /// Sets the body and its `Content-Type`. + /// + /// # Errors + /// + /// Returns `Config` when `content_type` is not a valid header value. + pub fn with_body(self, content_type: &str, body: Vec) -> Result>; + pub fn with_query(self, name: &str, value: &str) -> Self; + /// Both header helpers accept any header. A `Cookie` header, from + /// either, makes the coordinator refuse to send the request, and the + /// harness's conformance check fails it (section 13). + pub fn with_header(self, name: HeaderName, value: HeaderValue) -> Self; + /// For dynamic values. Rejects an invalid name or value. + pub fn try_with_header(self, name: &str, value: &str) -> Result>; + /// Adds `Authorization: Bearer `. + pub fn with_bearer(self, token: &Redacted) -> Self; + /// Data `response` needs from this call, such as a nonce or which + /// endpoint was used. Kept in memory with the call, never sent. + /// + /// # Errors + /// + /// Returns `Config` when it does not serialize, or is over 1,024 bytes + /// of JSON. + pub fn try_with_call_context(self, context: &impl Serialize) -> Result>; + pub fn method(&self) -> &Method; + pub fn url(&self) -> &Url; + pub fn headers(&self) -> &HeaderMap; + pub fn body(&self) -> &[u8]; +} + +impl ProviderResponse { + pub fn status(&self) -> StatusCode; + pub fn header(&self, name: &str) -> Option<&str>; + /// The decoded body. The coordinator undoes `Content-Encoding` first. + pub fn body(&self) -> &[u8]; + /// Parses the body as JSON, whatever the status. A parse failure is + /// `Malformed`. + pub fn json(&self) -> Result>; + /// Requires a 2xx status, else `Rejected`, then parses the body as JSON. + pub fn success_json(&self) -> Result>; + /// Requires a 2xx status, else `Rejected`, and a UTF-8 body, else + /// `Malformed`. + pub fn success_text(&self) -> Result<&str, Report>; + /// The context the request carried, decoded. `Ok(None)` when there was + /// none; a context that does not decode as `T` is `Config`. + pub fn call_context(&self) -> Result, Report>; +} + +impl Resolution { + /// One or more IDs, at most one per source: KV holds one UID per + /// source. The coordinator keeps the first ID for a source, discards the + /// rest with a log, and the harness fails the test. An empty set becomes + /// `no_match`, and `is_no_match()` then returns true. + pub fn found(ids: impl IntoIterator) -> Self; + /// The provider recognized no one, or a capture held nothing. + pub fn no_match() -> Self; + /// The stored ID for `source` is no longer valid. + pub fn invalidate(source: &SourceDomain) -> Self; + /// The provider reports the user opted out. + pub fn opted_out() -> Self; + /// Also invalidates `source`. Chainable after `found`, so one response + /// can return some sources and drop others. + pub fn also_invalidate(self, source: &SourceDomain) -> Self; + /// The earliest the coordinator may call again. Default: 24 hours after + /// `no_match`, none otherwise. + pub fn with_retry_in(self, delay: Duration) -> Self; + /// Replaces the provider's private state. + /// + /// The state is stored as embedded JSON of at most 1,024 bytes. A larger + /// value is not stored: the old state stays, the IDs are kept and a + /// state-size warning is logged. From a provider not declared + /// `with_private_state()`, the whole resolution is a `Config` error. + /// `capture` never changes state: a state change in a capture result is + /// ignored with a warning, and the harness fails the test. + pub fn replacing_state(self, state: &impl Serialize) -> Self; + pub fn clearing_state(self) -> Self; + + pub fn ids(&self) -> &[ResolvedId]; + pub fn invalidated(&self) -> &[SourceDomain]; + pub fn is_no_match(&self) -> bool; + pub fn is_opted_out(&self) -> bool; + pub fn retry_in(&self) -> Option; + pub fn state_change(&self) -> StateChange<'_>; +} + +pub enum StateChange<'a> { + Keep, + /// The new state as JSON. + Replace(&'a str), + Clear, +} + +impl ResolvedId { + pub fn new(source: &SourceDomain, uid: impl Into) -> Self; + pub fn with_atype(self, atype: i32) -> Self; + /// UID-level ext. Over 256 serialized bytes it is dropped and the ID + /// kept. + pub fn with_ext(self, ext: serde_json::Value) -> Self; + pub fn with_expires_in(self, delay: Duration) -> Self; + pub fn with_expires_at(self, at: UnixTime) -> Self; + pub fn with_refresh_in(self, delay: Duration) -> Self; + pub fn with_refresh_at(self, at: UnixTime) -> Self; + + pub fn source(&self) -> &SourceDomain; + pub fn uid(&self) -> &str; + pub fn atype(&self) -> Option; + pub fn ext(&self) -> Option<&serde_json::Value>; + pub fn expiry(&self) -> Option; + pub fn refresh(&self) -> Option; +} + +/// A time as a provider gave it: relative to `now`, or absolute. +pub enum When { + In(Duration), + At(UnixTime), +} + +/// A provider failure. Built through the constructors below. +#[derive(Debug, derive_more::Display)] +#[non_exhaustive] +pub enum IdentityProviderError { + /// Configuration or provider code makes the call impossible. + #[display("identity provider configuration error: {message}")] + Config { message: String }, + /// The provider read something its descriptor did not declare. + #[display("`{access}` was read but not declared")] + UndeclaredInput { access: Access }, + /// A declared input is not available for this request. + #[display("input `{input}` is unavailable for this request")] + InputUnavailable { input: Input }, + /// The provider refused the request. + #[display("provider rejected the request with status {status}")] + Rejected { status: StatusCode }, + /// The provider's response could not be parsed. + #[display("provider response could not be parsed")] + Malformed, + /// The stored private state does not decode as the provider's type. + #[display("stored provider state could not be decoded")] + InvalidState, + /// Set by the coordinator only. Providers never see transport failures. + #[display("provider request failed in transport")] + Transport, +} + +impl IdentityProviderError { + pub fn config(message: impl Into) -> Report; + pub fn rejected(status: StatusCode) -> Report; + pub fn malformed() -> Report; + /// The default body of a capability method: a `Config` error with a + /// fixed marker message, so the harness can tell a missing override from + /// a real failure. + pub fn not_implemented(method: &'static str) -> Report; +} + +impl core::error::Error for IdentityProviderError {} +``` + +Relative times (`with_expires_in`, `with_refresh_in`, `with_retry_in`) are +resolved against the request's `now` by the coordinator, so providers rarely +need the clock. + +The coordinator discards a `ResolvedId` whose source the provider does not +claim (or that the operator narrowed away), or whose `uid` is empty or over +`MAX_UID_LENGTH` (`ec/kv_types.rs:35`). It logs each discard with the +provider ID and source only. An ID of an expiring source that carries no +expiry keeps the stored `exp` when its UID is unchanged, and is discarded +otherwise, because an owned expiring ID without `exp` is never served +(section 6.1). That fallback is for a vendor that omits an expiry at +runtime; the harness fails a fixture that does. + +`replacing_state` with a value that does not serialize is handled like an +oversized one: the old state stays, the IDs are kept and a warning is +logged. + +### 4.6 A complete provider + +A fictional provider, `exampleid`, shows the whole surface: a descriptor +built from config, a required and a wanted input, consent strings, a +provider state round trip, a UID-level `ext` and a refresh hint. + +```rust +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct ExampleIdConfig { + pub enabled: bool, + #[validate(url)] + pub endpoint: String, + pub partner_id: u32, + pub gvl_id: u16, +} + +impl IntegrationConfig for ExampleIdConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for ExampleIdConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(EXAMPLEID_INTEGRATION_ID.parse()?) + .with_source("exampleid.example.com".parse()?) + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::ClientIp, Input::UserAgent]) + .with_wants([Input::PageUrl]) + .with_private_state(), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(self.gvl_id)) + .build() + } +} + +pub struct ExampleIdProvider { + descriptor: ProviderDescriptor, + partner_id: u32, +} + +#[derive(Serialize, Deserialize)] +struct ExampleIdState { + signature: String, +} + +#[derive(Serialize)] +struct FetchBody<'a> { + partner: u32, + ip: String, + ua: &'a str, + page: Option<&'a str>, + gdpr: bool, + consent: Option<&'a str>, + signature: Option, +} + +#[derive(Deserialize)] +struct FetchReply { + uid: Option, + signature: Option, + link_type: Option, + max_age_sec: Option, +} + +impl ExampleIdProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub fn new(config: &ExampleIdConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + partner_id: config.partner_id, + }) + } +} + +impl IdentityProvider for ExampleIdProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let consent = input.consent(); + let body = FetchBody { + partner: self.partner_id, + ip: input.client_ip()?.to_string(), + ua: input.user_agent()?, + page: input.page_url().ok().map(Url::as_str), + gdpr: consent.gdpr_applies(), + consent: consent.raw_tc_string(), + signature: input.state::()?.map(|state| state.signature), + }; + ProviderRequest::post_json(input.endpoint()?, &body).map(Some) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let reply: FetchReply = response.success_json()?; + let Some(uid) = reply.uid else { + return Ok(Resolution::no_match()); + }; + + let mut id = ResolvedId::new(input.primary_source(), uid).with_atype(1); + if let Some(link_type) = reply.link_type { + id = id.with_ext(json!({ "linkType": link_type })); + } + if let Some(max_age) = reply.max_age_sec { + id = id.with_refresh_in(Duration::from_secs(max_age)); + } + + let resolution = Resolution::found([id]); + Ok(match reply.signature { + Some(signature) => resolution.replacing_state(&ExampleIdState { signature }), + None => resolution, + }) + } +} +``` + +Registration uses the existing builder (`integrations/registry.rs:636`) from +an integration's `register` function, which returns +`Report` (`integrations/lockr.rs:321-323`). error-stack +converts only `C` into `Report` (`error-stack-0.6.0/src/context.rs:101`), +so the builder does that conversion once for every vendor: + +```rust +impl IntegrationRegistrationBuilder { + /// Registers a provider built by a fallible constructor. A failure + /// becomes `TrustedServerError::Integration`, naming the integration the + /// builder already holds. + pub fn try_with_identity_provider( + self, + provider: Result>, + ) -> Result>; +} + +Ok(Some( + IntegrationRegistration::builder(EXAMPLEID_INTEGRATION_ID) + .try_with_identity_provider(ExampleIdProvider::new(&config))? + .build(), +)) +``` + +A capture-only provider overrides one method. This fictional one reads +tokens from the API it proxies, and returns `no_match` for any response it +does not recognise: + +```rust +impl IdentityProvider for ExampleCaptureProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + let Ok(reply) = input.response().json::() else { + return Ok(Resolution::no_match()); + }; + Ok(Resolution::found(reply.tokens.into_iter().filter_map(|token| { + let source = input.source_named(&token.source)?; + Some(ResolvedId::new(source, token.value).with_expires_at(UnixTime::from_secs(token.expires))) + }))) + } +} +``` + +Its descriptor declares +`.with_captures(Captures::new().with_path(Method::POST, "/integrations/examplecapture/api/token"))` +and no `resolves`, so it is never dispatched. + +### 4.7 The coordinator + +`IdentityCoordinator` is built per request from the registered providers and +`[ec.identity]` settings. Its dependencies (settings, registry, KV graph, +rate limiter, runtime services) travel in one `IdentityDeps` struct, because +`handle_auction` and `handle_publisher_request` already take the maximum of +seven arguments (`auction/endpoints.rs:114-122`, `publisher.rs:4433-4441`). + +**The eligibility view.** Eligibility reads one merged view of stored state, +on every route: + +- KV is the base when the request read it, else the cookie's last + confirmed copy. +- Cookie entries with `k: 0` override it when their `pv` matches the base + revision. Removals and opt-outs in the removals cookie `ts-eids-r` always + apply (section 6.2). +- `ps.n` is the later of the cookie's and KV's. `ps.iv` is the union. +- `ps.o`: a staged opt-out (in `ts-eids-r`, not yet in KV) always counts, + unless `ts-eids-s` stages a clear `ps.oc` later than it (section 8). A + mirrored opt-out (a copy in `ts-eids-s`) yields to KV when this request + read KV, so a clear in KV (section 8) is not undone by the browser's copy. +- Pending markers come from the cookie only. + +**Eligibility.** A provider that declares `resolves` is eligible for this +request when all of these hold: + +1. The request is a real browser (`device_signals.looks_like_browser()`, + `adapter-fastly/src/app.rs:410`), the same gate pull sync uses. +2. Global EC consent allows EC and EID persistence (`allows_eid_persistence`, + as `pull_sync.rs:72-77` does). +3. The descriptor's consent policy is met (section 9.1). +4. At least one claimed source has a kind the provider lists, computed per + source: + - **Owned by this provider.** `Initial` when there is no stored record. + `Refresh` when there is one, valid or expired, and it is due. A stored + record is due when it is expired, when this provider did not write it, + when its `refresh_at` has passed, or, if it has none, when + `acquired_at + refresh_after_sec` has passed or `acquired_at` is + missing. A browser-seeded value whose UID matches `ps.iv` is never due + (section 7.2). An expired record is never served (section 7.1), so the + provider either refreshes it or returns `None`. + - **Unowned.** `Initial` only, and only when KV has no unexpired ID for the + source. An expired record counts as absent. Unowned sources are never + refreshed (section 7.2). + - Sources whose kind's needs are missing are left out. If none remain, + the provider is skipped and the skip is logged at debug. +5. `ps.n` (next eligible time) has passed, and no sticky `ps.o` opt-out is + set (section 8). +6. No unanswered pending marker `p` is set for the provider (section 6.2). +7. The provider's circuit breaker is closed (below). +8. The request mode allows it, subject to the restrictions on stateful + providers and first navigations below: + - Hold-mode providers run on any request that runs an auction: a + navigation with matched slots, `/auction` or `/_ts/page-bids`. + - After-send providers run only on the writer request for a returning + EC. A new visitor's navigation runs none, so its create stays the page + load's one write (section 5.2). Its after-send providers start at the + next navigation. + - A provider declared `with_private_state()` runs only on the writer + request. In hold mode it runs on writer navigations that run an + auction, and after send on writer navigations that do not, so a site + whose auctions all go through `/auction` still refreshes it. + - No provider runs on a new visitor's first navigation, in either mode. + Its row is created before routing, carrying only browser ingest. +9. The per-visitor burst cap is not exceeded. This check comes last, just + before dispatch, because the Fastly limiter increments on every check under + the limit (`adapter-fastly/src/rate_limiter.rs:54-66`). The cap is + `max_calls_per_minute` on the key `identity::`, where + `visitor` is an HMAC of the full EC ID with a key derived from + `ec.passphrase`. It is not `ec_hash`, which everyone behind one IP shares + (section 3.4). + +Pacing per visitor comes from `ps.n` and the refresh rules in step 4, not +from the limiter. The limiter only caps bursts, because its windows are 60 +seconds long. + +**Pre-disclosure revalidation.** Before any byte goes to a provider for a +returning EC, the request must have read the live KV row and found it live +and consenting. Eligibility steps 4 to 6 are then recomputed from that live +row, so an opt-out or removal that the cookie has not caught up with still +stops the call. That read is the writer's phase 1 read for hold mode, its +phase 4 read for after-send mode (section 5.2), or the read `/auction` and +`/_ts/page-bids` already do. A new visitor's first navigation runs no +provider (step 8), so every provider call follows a confirmed row. + +**Dispatch.** For each eligible provider, the coordinator builds a +`ResolveInput` holding only the declared inputs and calls `request`. It then +checks the request before sending it: + +- The scheme is `https`. +- The host is one of the descriptor's `resolves` hosts. +- No `Cookie` header is present, and every header name and value is valid. + +A request that fails a check is dropped and logged as `Config`. The +coordinator sends it through `RuntimeServices::http_client().send_async`, up +to `max_concurrency` at once, with a backend first-byte timeout of the +provider's timeout (for hold mode, `min(timeout_ms, remaining budget)`, +rounded to buckets as `auction/orchestrator.rs:453-473` does). It does not +follow redirects. It reads at most `max_response_bytes`, checked against +`Content-Length` first (`pull_sync.rs:412`) and against the buffered body +(`pull_sync.rs:472-474`). It undoes `Content-Encoding` with the same cap +applied to the decoded size, then passes the `ProviderResponse` to +`response`. + +**Hold-mode timing.** Hold-mode providers share one wall-clock budget, +`hold_budget_ms` (default 100 ms), measured from the first dispatch. Each +waits for `min(timeout_ms, remaining budget)`. The auction starts when all +have answered or the budget is spent. + +Why 100 ms: + +- It is a fifth of the navigation auction's recommended 500 ms timeout + (`auction_timeout_ms`, `trusted-server.example.toml:336-340`). A 300 ms + budget would take 60% of it. +- Browser-side guidance runs higher, from 100 to 500 ms. Prebid's + `auctionDelay` defaults to 500 ms (EID fidelity spec D4), and data vendors + recommend at least 100 or 250 ms (`adlooxRtdProvider.md:86`, + `51DegreesRtdProvider.md:101`). Those figures include the visitor's device + connecting to the vendor. An edge call skips that: it goes from the POP, + often over a connection Fastly already holds open. +- Missing the budget costs that one auction the ID. A call cut off by the + budget is not a failure: it does not count toward `ps.f` or the breaker. + It sets `ps.n = now + hold_retry_sec` (default 300 seconds), so the next + request does not wait on the same slow provider straight away. + +The coordinator logs each provider's response time. An operator raises the +budget only when a provider's measured p95 latency at the edge exceeds it. + +On the navigation, holding has a latency cost. The navigation auction is +dispatched before the origin fetch (section 3.4), so a due hold-mode +provider delays bid dispatch. In modes where the origin fetch waits for +dispatch, it also delays the page's first byte by up to `hold_budget_ms`. A +provider is due rarely (step 4), so few navigations pay this. + +**Results and headers.** A hold-mode call either answers within its budget +or is cut off; no call outlives the router (section 3.4). An answer is saved +in that response's `ts-eids-s` with `k: 0`, on the writer request as on +stagers, and a writer also adds it to its mutation. So a later request reads +it from the cookie first, and a failed flush loses nothing. After-send +results exist only in phase 4, after headers are sent. The writer marks each +after-send provider with a pending marker before headers go out (section +5.2). + +**Circuit breaker.** The rate-limiter trait (`ec/rate_limiter.rs:16-47`) +gains `record(key)` and `count(key)`, backed by Fastly's counter `increment` +and `lookup_count`. Only transport failures, timeouts, 5xx and 429 responses +are recorded, under `identity-fail:`. While `count` is at or above +`breaker_failures_per_minute`, the provider is skipped for every visitor. +Input-caused 4xx responses and `Malformed` back off only that visitor, so no +visitor can open the breaker for everyone by seeding bad values. Fastly +counters are kept per POP, so the breaker is too. + +**Passive capture.** `IntegrationRegistry::handle_proxy` +(`integrations/registry.rs:1031-1060`) holds the `EcContext`. +`IntegrationProxy::handle` itself has no EC context +(`integrations/registry.rs:319-324`). For each provider with a capture +rule (`with_path` or `with_path_prefix`) matching the request: + +- **Before `handle`.** When the descriptor's `Captures` declares `with_request_consent` and the + request has a body, `handle_proxy` buffers it with the existing + `collect_body_bounded` (`integrations/mod.rs:182`, 256 KiB cap), extracts + only the named JSON pointers, and re-wraps the bytes with + `EdgeBody::from` before calling `handle`, so the integration forwards them + unchanged. Today the request moves into `handle` unread + (`integrations/registry.rs:1061`), and Lockr buffers its own copy inside + `handle_api_proxy` (`integrations/lockr.rs:208-210`), out of the + coordinator's reach. A body over the cap or not valid JSON allows no + positive IDs; removals still apply. +- **The consent to apply.** `consent::stricter_of(cookie, body)` is a new + function. It combines two consent contexts field by field: any opt-out, + missing purpose or missing vendor consent in either one wins. +- **After `handle`,** on 2xx responses only, the coordinator decodes + `Content-Encoding` under the byte caps, builds a `CaptureInput` and calls + `capture`. +- **It applies the result.** Removals and opt-outs always apply, whatever the + consent, because a revocation arrives after consent is denied. Positive + IDs apply only when eligibility steps 1 to 3 pass under the stricter + consent, and the provider has no sticky `ps.o`, unless this result clears + it: a capture-only provider without a GVL ID that returns IDs under + consenting cookie and body consent stages a clear, `ps.oc = now`, beside + its IDs in `ts-eids-s` (section 8). + +The result is staged on `EcContext`. `ec_finalize_response`, which already +runs for proxy responses (`adapter-fastly/src/app.rs:909`, +`main.rs:606-614`), writes positive IDs into `ts-eids-s` and removals and +opt-outs into `ts-eids-r`, including on the unusable-consent path (section +6.2). On a writer request the result also joins the mutation. + +Capture never touches `ps.n`, `ps.f` or the breaker: a proxied call that +held nothing is not a provider failure. Only an opt-out and its IDs +persist. Capture never changes provider state: a state change in its result +is ignored with a warning, on writers and stagers alike, and the harness's +conformance check fails a `capture` that returns one (section 13). This +holds whether or not the provider declares `with_private_state()`. + +Lockr's API responses are already buffered on Fastly: a non-streaming send +reads the body whole, up to 10 MiB (`adapter-fastly/src/platform.rs:413-429`). +The Lockr proxy forwards the browser's `Accept-Encoding` +(`integrations/lockr.rs:259-265`), so its responses can be compressed. The +coordinator decodes them, as above. + +## 5. One KV write per page load + +### 5.1 Writer requests and stagers + +A request is the **writer request** when all of these hold. The decision is +made once pre-route filters have allowed dispatch, because a challenged or +blocked request must not persist anything (`ec/mod.rs:91-96`). It sits where +routing already decides EC generation (`adapter-fastly/src/app.rs:846-851`, +`integrations/registry.rs:1048`) and is stored on `EcContext`: + +- It is an organic route: the publisher fallback, or an integration proxy + route. +- `Sec-Fetch-Dest` is `document`. The `Accept: text/html` fallback of + `is_navigation_request` does not count, because `fetch()` can send it + (`http_util.rs:105-117`). +- It is not a prefetch or prerender. `is_prefetch_request` moves to + `http_util` as a public function so the adapter can call it. +- The request is a real browser (`adapter-fastly/src/app.rs:410`). + Integration-proxy EC generation gains the same explicit check: today it + is gated only by a missing KV graph (`integrations/registry.rs:1048-1049`, + `adapter-fastly/src/app.rs:458`). + +An integration-proxy writer never rotates an EC, as today +(`adapter-fastly/src/app.rs:885-890`). + +The adapter captures the response status before `send_edgezero_response` +consumes the response, and passes it to phase 4 (`main.rs:512-517`, `:618`). +A returning EC's writer flushes only when that status is 2xx or 304. For any +other status it drops its mutation and dispatches no after-send provider. It +saves the answers of hold-mode calls it already made in the response's +cookie, because those calls happen before the origin status is known +(`publisher.rs:4887-4894`, `:5151-5175`). A redirect is followed by another +navigation, which recomputes everything from the same cookies. + +A new visitor's create (section 5.2, phase 1, before routing) is written whatever the +status, as today, because the response sets the EC cookie. + +Every other request is a **stager**: `/auction`, `/_ts/page-bids`, +integration proxy calls that are not navigations (including Lockr's API +proxy), prefetches, iframes, clients that send no `Sec-Fetch-Dest`, and every +other route. Stagers read cookies and KV but never write KV. Section 5.3 +covers what they save instead. + +`/_ts/api/v1/identify` stops writing too. The once-per-EC `cluster_size` +evaluation (`ec/identify.rs:107`, `ec/kv.rs:1484`) moves into the writer's +mutation. `/identify` uses a stored `cluster_size` when present and computes +one without writing when not. + +**SPA views.** A single-page app changes views with JavaScript, and TS's +script reports each change to `/_ts/page-bids` (`publisher.rs:6890-6894`). +An SPA view is part of the page load that loaded the document, so +`/_ts/page-bids` is a stager. Today it ingests `ts-eids` into KV +(`publisher.rs:6925`). Under this design it stops. A visitor who stays in an +SPA has their staged changes written at their next full navigation. +Auctions keep using the staged entries from the cookie in the meantime. + +**Exception: consent withdrawal.** A withdrawal writes its tombstone on +whichever request carries it, before the response is sent, exactly as today. +That includes stagers and clients the bot gate does not treat as real +browsers (`adapter-fastly/src/app.rs:446-452`). Revocation cannot wait for a +navigation that may never come. The exception covers the whole of today's +withdrawal path: a CAS tombstone with up to five attempts, then its +completion-marker write, for each withdrawn EC ID (one or two; +`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). It is the only +page-load exception to G4. The tombstone is not part of any `IdentityMutation`. A +writer whose phase 4 re-read finds the tombstone drops its mutation (section +5.4, step 1). + +### 5.2 Phases of the writer request + +**Phase 1, at the routing decision. No KV write.** + +- Read `ts-ec`, consent, `ts-eids`, sharedId and `ts-eids-s`. Verify + `ts-eids-s` (section 6.2). +- Decide whether this is the writer request (section 5.1). +- Capture the user agent, DNT and page URL into `EcContext` (section 4.4). +- Read the KV snapshot when section 6.3 says to. A missing row for a + returning EC starts today's orphan recovery: confirm the absence, then + rotate before send (`ec/finalize.rs:371-400`). The rotated EC's create + carries this request's browser ingest, as today (`ec/finalize.rs:323`), + and its removals and sticky `ps.o`, `ps.oc` and `ps.iv`, but never + positive staged IDs. It is the page load's one write (section 5.6): a + request that rotates drops the rest of its mutation and its pending + markers, and skips phase 4. When the create is `Written`, its response + deletes the `ts-eids-r` cookies, whose contents the create carried, and + sets a new `ts-eids-s` from the created row. Otherwise both are kept. If the confirm + read finds the row after all, no rotation happens and that read is the + phase 1 snapshot (today's code writes at once there, `ec/finalize.rs:383-386`; + here the change joins the mutation). +- For a new visitor, `generate_if_needed` creates the row before routing, as + today (`ec/mod.rs:407-455`), so no unconfirmed ID is ever used. Today that + create is an empty row and browser ingest is a second write in finalize + (`ec/mod.rs:412`, `ec/finalize.rs:144`). Here the create also carries + browser ingest from `ts-eids` and sharedId, which are read before routing, + so it is the page load's one write. + +**Phase 2, routing.** + +- The coordinator computes eligible providers from the merged view. +- If the navigation runs an auction, due hold-mode providers are dispatched + before it, after eligibility is recomputed from the phase 1 read, and + awaited within the hold budget. Their answers feed the auction. +- For a returning EC, the coordinator calls `request` for each eligible + after-send provider as a dry run, with `input.is_dry_run()` true. `request` + is pure and does no I/O. A provider that returns `Some` gets a pending + marker in phase 3. One that returns `None` is skipped (section 8). The real + request is rebuilt in phase 4 from the live row. +- A provider declared `with_private_state()` gets no dry run, because its + state is only in KV. When it is due on the cookie view it gets a marker, + and phase 4 rebuilds its request with the state from the live row. A + rebuilt `None` sets `ps.n` (section 8). This saves a phase 1 read on each + of its refreshes. +- Any other dry run sees what KV holds only when phase 1 read it. Section + 6.3 makes phase 1 read KV whenever an after-send provider is eligible on + the cookie view and has a candidate source that is unowned or only a + stub. + +**Phase 3, finalize, before send.** + +- **New visitor.** Nothing to write: the row was created before routing + (phase 1). `ts-ec` is set because the create was `Written`, as today + (`ec/finalize.rs:145-149`). Phase 4 does nothing for a new visitor, and no + provider runs on its first navigation (section 4.7, step 8). +- **Returning EC.** + - Browser ingest from `ts-eids` and sharedId adds `FillMissing` changes to + the mutation, instead of calling `sync_eid_cookie_updates` + (`ec/finalize.rs:183`). It covers owned sources too, which seeds them + for their owner (section 7.2). + - Staged entries from `ts-eids-s` pass the flush checks (section 5.3) and + are added to the mutation. + - Hold-mode answers and this request's captures join the mutation. +- **Consent withdrawal** runs today's synchronous path (section 5.1). +- **Cookies.** Removals and opt-outs from this request go into a new + `__Host-ts-eids-r-` cookie (section 6.2). `ts-eids-s` is set on the HTML response only + when needed: + - for a new EC, from its created row + - with the results of hold-mode providers that answered, saved with + `k: 0`, except a provider with private state: its answer stays in the + mutation only, because its ID and its state must land together + - with a pending marker for each after-send provider whose dry run + returned `Some`, only when the response status is 2xx or 304 (finalize + already holds the response) + - when the phase 1 read changed the cookie under the section 6.3 merge + - when the cookie was missing, failed to open or was past its + revalidation age + +**Phase 4, after send. Returning ECs only. The only write.** + +1. If the mutation is empty and no after-send provider was marked, stop. No + read, no write. +2. If phase 1 read the row successfully and no after-send provider is + marked, do not read again (a failed phase 1 read counts as no read, and + phase 4 re-reads): apply the mutation to the phase 1 snapshot and, if anything + changes, write with that generation. A concurrent change since then + fails the CAS, as any conflict does. Otherwise re-read the live row. + Either way, the snapshot or the live row decides: + - A tombstone, or `consent.ok = false`: drop the mutation and stop. + - Missing: drop the mutation and stop. The next request that reads KV + finds the orphan (section 6.3). This covers an integration-proxy + writer, which never rotates. + - Failed: drop the mutation and stop. Staged entries stay in the cookie. +3. Recompute eligibility steps 4 to 6 from the live row, rebuild each marked + provider's request, dispatch the after-send providers and turn their + results into changes. This happens after the re-read, so no EC ID is + disclosed before consent is confirmed (`ec/pull_sync.rs:100-110` does the + same today). +4. Apply the mutation to the live entry (section 5.4). If anything changed, + write it once with `IfGenerationMatch`. +5. If the write fails, on a precondition failure, Fastly's per-key rate + limit or any other error, log and stop. It is not retried. Fastly limits + writes to one key to about one per second, and the limit can be stricter, + so an immediate retry after a conflict would usually fail too. Nothing is + lost for good: + - browser ingest is re-derived from `ts-eids` by the next navigation + - staged entries and stateless hold-mode answers are still `k: 0` in the + cookie + - removals and opt-outs stay in `ts-eids-r` until a KV read shows them + applied + - an after-send provider's result is lost, an `invalidate` or `opted_out` + included. The ID it rejected stays served, as it was before the call, + until the provider is called again once its pending marker passes its + deadline and repeats the removal. Nothing after send can save it, + because the KV write is what failed + - a stateful hold-mode provider's answer, state change and `ps.n`/`ps.f` + are lost with the mutation. Its answer never went into the cookie + (phase 3), so no ID can flush later without its state, and the next + writer calls it again with the state KV still holds. If the vendor has + rotated that state (a used UID2 refresh token, for example), the call + fails, the stored ID ages out, and the browser or a later initial call + re-seeds it + +**Cost per page load.** Writes are at most one, and never retried. Reads, +compared with today: + +| Navigation | Today | This design | +| --------------------------------------------------------------- | ---------------------- | ---------------------------------------------------------------------- | +| Runs an auction | 1 (preload) | 1 (phase 1). Phase 4 reuses it unless an after-send provider is marked | +| No auction, registered source in `ts-eids` | 1 (ingest compare) | 1 (phase 4) | +| No auction, nothing due, staged or pending, valid recent cookie | 0 or 1 | 0 | +| An after-send provider dispatches | 2 (preload, pull sync) | 1 or 2: the phase 4 re-read, plus phase 1 when section 6.3 requires it | + +Today's preload never skips a navigation whose auction needs the row +(`publisher.rs:3110-3116`, `:4825-4828`), and the cookie does not cache +unowned values, so such a navigation reads KV once, as today. The 6.3 +trigger list states the rule. +Each `/auction` and `/_ts/page-bids` keeps its one existing read. Orphan +recovery adds the reads it adds today, in the rare case it runs. + +A digest of the last-compared browser set in `ts-eids-s` could remove the +read on navigations with no auction. It is left out until measurements show +the read matters. + +### 5.3 Stagers + +A stager never writes KV. It saves positive changes into `ts-eids-s` and +removals and opt-outs into `ts-eids-r`, on its own response (section 6.2). +`ts-eids-s` is set in the returning-visitor branch of `ec_finalize_response` +(`ec/finalize.rs:81-127`), replacing today's ingest write at +`finalize.rs:110-112`. That branch already has the request's client EIDs +(`ec/mod.rs:568-578`) and its KV snapshot. `ts-eids-r` is also written from +`finalize_unusable_consent` (`ec/finalize.rs:457`), because a revocation can +arrive while consent is unusable but not explicitly withdrawn, and that path +returns before the returning-visitor branch (`finalize.rs:62-78`). A stager +saves: + +- **The KV gap, on `/auction`.** `/auction` holds the full body set + (`auction/endpoints.rs:298-303`) and the KV row it already reads + (`endpoints.rs:307-311`). For each registered source in the consent-gated + body set that the KV row lacks or holds only expired, it saves + `{s, u, b: 1, k: 0}`. It picks the + UID with today's ingest rules (`collect_eid_updates`, + `ec/prebid_eids.rs:151`). It saves only absent or expired sources, + because ingest is fill-missing (sections 3.2 and 5.4), and skips a UID that matches `ps.iv` (section + 7.2). It costs no extra KV read. + - When the client set came from the `ts-eids` cookie rather than a body, + nothing is saved: the next navigation ingests that cookie anyway. + - When the KV read failed, found the row missing or found a tombstone, + nothing is saved. + - `/auction` returns early on consent denial, before it reads KV + (`endpoints.rs:231-275`). Then nothing is saved. +- **Hold-mode results** on `/auction` and `/_ts/page-bids`, from providers + without state: IDs as entries with `pv` (the revision they replace) in + `ts-eids-s`, `ps` changes marked `sg: 1`, and invalidations and opt-outs + in `ts-eids-r`. `/_ts/page-bids` reads KV only when its auction + runs (`publisher.rs:7077-7090`), so it runs hold mode only then. +- **Passive capture results** on integration proxy responses (Lockr), the + same way. +- **Cookie upkeep from the KV read** that `/auction` and `/_ts/page-bids` + already do (section 6.3). These responses are JSON and already private, so + cookie upkeep costs no HTML cacheability. + +Providers with state run only on the writer request (section 4.7, step 8), +because state lives in KV only and a stager cannot write it. A stager that +sees one of them due leaves it to the next navigation. + +**No lost EIDs (G10).** If the KV gap does not fit in the cookie's size cap, +the next `/auction` finds the same gap, since KV still lacks it, and saves it +again. An ID is delayed by a page view, never dropped, as long as the browser +keeps sending it. Today's `/auction` ingest writes the same fill-missing set +(`ec/kv.rs:146-154`), so everything it writes still reaches KV. + +**Concurrent stagers.** Parallel `/auction` or `/_ts/page-bids` calls each +rewrite the whole cookie from their own copy, and the last `Set-Cookie` +wins. A saved entry can be lost this way: + +- A lost KV-gap entry is saved again by the next `/auction`. +- A lost hold-mode result costs one later provider call. The per-visitor + burst cap, one call per minute by default, makes near-simultaneous + duplicate calls rare. The limiter is best-effort under concurrency + (`ec/rate_limiter.rs:25-27`), so they are not impossible. +- Removals and opt-outs are never lost this way. Each batch lives in its own + `ts-eids-r` cookie, which no other response rewrites and only a writer + whose KV read shows it applied deletes (section 6.2), so a concurrent + rewrite of `ts-eids-s` or a second removal cannot drop them. A capture cannot be re-derived, so this matters most for + Lockr's `revoke-consent`. + +The loss of positive entries is accepted. + +**Flush checks for staged entries.** In phase 3, the writer adds a staged +entry to its mutation only when it passes all of these: + +These checks apply to positive entries. Removals and staged opt-outs keep +only the binding and revision checks (section 5.6). + +- The cookie opens with this EC's ID as associated data (section 6.2). +- The entry is fresh: its staged-at time `sa` is no more than + `staged_max_age_sec` ago. Older entries are dropped from the cookie on its + next rewrite. +- Its writer `w` is an enabled provider that currently claims the source and, + for an owned source, owns it. +- It passes `gate_eid_updates_by_consent` (`ec/finalize.rs:268`) and the + writing provider's consent policy, both against this request's consent, + which may have changed since staging. +- A provider entry is dropped when `ps.o` marks that provider opted out, + unless the cookie also stages a clear `ps.oc` later than that opt-out's + `at`. A replayed old cookie therefore cannot re-insert an opted-out ID: its + clear predates the opt-out. +- A provider entry becomes an `OwnerWrite` with its `pv` as the expected + revision when its writer currently owns the source, and a `FillMissing` + otherwise, so an unowned source keeps the fill-missing contract. The cookie + may hold `k: 0` entries for unowned sources; it caches only owned sources + once they are in KV. A removal from `ts-eids-r` becomes an `OwnerRemove`, + or a `WriterRemove` when it carries no revision. A browser entry becomes a + `FillMissing`. A staged `ps` change merges as described in + section 6.2. + +### 5.4 `IdentityMutation` + +```rust +pub(crate) struct IdentityMutation { + partner_ids: Vec, + provider_state: BTreeMap, + cluster_size: Option, +} + +pub(crate) enum PartnerIdChange { + /// From browser ingest, a KV-gap entry or an unowned provider. + /// Applies only if the source is absent or its record is expired. + FillMissing { source: SourceDomain, record: StoredRecord }, + /// From the owning provider. Applies if the live record's revision equals + /// `expected`, or if `expected` is `None` and the source has no valid ID + /// that this provider wrote. A browser-seeded or legacy value never + /// blocks the owner. + OwnerWrite { source: SourceDomain, record: StoredRecord, expected: Option }, + /// From `Resolution::invalidate` or `also_invalidate`. Applies only if + /// the revision matches. + OwnerRemove { source: SourceDomain, expected: Revision }, + /// From `Resolution::opted_out`, or a staged removal without a revision. + /// Removes the source's record if this provider wrote it, at any + /// revision. + WriterRemove { source: SourceDomain, writer: IdentityProviderId }, +} + +/// A keyed hash of the `(uid, acquired_at)` pair a request read. Comparing +/// the UID alone is not enough: a refresh can extend expiry without changing +/// the UID. The cookie's `pv` stores the same hash, so staged entries compare +/// against the live record's hash. +pub(crate) struct Revision(RevisionHash); +``` + +A new visitor's create is not a mutation: `generate_if_needed` builds the +seed row before routing (section 5.2). + +Applying the mutation to a live entry, in order: + +1. If the live entry is a tombstone, drop everything. +2. Apply removals first: every `OwnerRemove` and `WriterRemove`. Applying a + new `ps.o` for a provider also removes every live record that provider + wrote, owned or unowned, at any revision. Removals are never skipped. + An opt-out applies only when its `at` is after both the live `ps.o.at` + and the live `ps.oc`. Then apply each staged clear `ps.oc` that is later + than the provider's opt-out `at`, live or staged, and store it as the + live `ps.oc`, so the next step sees the provider as not opted out and a + later replay of the old opt-out is stale. A clear that is not later is + dropped. +3. Apply the positive changes by their own rules. Every change written by + a provider the live `ps.o` marks opted out is skipped, `OwnerWrite` and + `FillMissing` alike, and so is a staged entry whose `sa` is before that + provider's latest opt-out `at`, even after a later clear. A `FillMissing` whose + UID matches the live `ps.iv` is skipped. Changes that no longer apply are + dropped silently: a newer write won. +4. Apply `provider_state` changes (`n`, `f`, `o`, `iv`, `st`) and + `cluster_size`. Drop expired records and the `ps` entries of providers no + longer configured. +5. Validate the result with `KvEntry::validate`. If it fails, keep only the + removals and opt-outs and validate again, so a full row can always + shrink. If that also fails, drop the mutation and log. An invalid row is + never written. +6. If nothing changed, skip the write. + +Because the rules are checked against the live entry, a later writer can +re-apply the same changes, re-derived or still staged, without losing a +concurrent tab's newer data. + +### 5.5 Orphans, collisions and deletion + +Creating a new visitor's row before its cookie goes out keeps today's three +safety properties, which a deferred create would lose: + +- **Withdrawals stick.** A withdrawal can only arrive with a cookie, and the + cookie is set only after the row exists, so the tombstone always has a row + to land on. +- **Collisions are caught.** An `AlreadyExists` on the first create re-mints + the ID before any cookie is set, so no browser adopts another visitor's + row. +- **Orphans are found.** A row that goes missing later is found by a KV read + and recovered by today's rotation, before send (section 5.2, phase 1). + Section 6.3 guarantees such a read at least every `kv_revalidate_sec`, and + any stager that sees the row missing replaces `ts-eids-s` with a minimal + `m: 1` cookie to force one at the next navigation. + +**Deletion.** The IAB data-deletion framework, deferred today, would +hard-delete rows (`ec/kv.rs:1551-1563`). When that framework lands, it writes +a tombstone with `TOMBSTONE_TTL` (24 hours, `ec/kv.rs:38-39`) instead, using +its own unconditional writer and today's completion marker. That replaces +one delete with one write, the TTL removes the tombstone with no further +write, and every TS request that reads the row in those 24 hours treats the +identity as withdrawn. A browser that visits within those 24 hours +revalidates (section 6.3) and expires its cookie. One that returns later +finds the row missing, and section 5.6 makes a missing row expire the cookie +and rotate without carrying any positive IDs, so a deletion is never undone +through the cookie. + +### 5.6 Rules that keep removals winning and markers answered + +Round 4 found paths where a cookie could undo a removal or a marker could +loop. These rules close them, and take precedence over any earlier sentence +that seems to allow otherwise: + +- **Removals survive every filter.** Removals and staged `ps.o` and `ps.iv` + live in `ts-eids-r` (section 6.2). They are exempt from the consent + re-gate, the writer-still-owns check and `staged_max_age_sec` in the flush + checks (section 5.3), and keep only the EC binding and, when they carry + one, the revision check. Withdrawing consent, disabling a provider or a + week in an SPA can delay a positive ID, never a removal. +- **A config change keeps removals.** When the `cf` of `ts-eids-s` does not + match, its positive entries are discarded and it is rebuilt from KV. + Pending markers are kept. `ts-eids-r` carries no `cf`, so its removals + apply against the current config unchanged. +- **A missing row empties the cookie.** Any KV read that finds the row + missing (outside a recovery in the same request) treats it like a + tombstone for the server set: no positive entries or markers survive, and + `ts-eids-s` is replaced by a minimal cookie holding only `m: 1`, so the + next navigation reads KV and recovers. `ts-eids-r` is kept. `/auction` + saves no KV gap when the row is missing. Orphan rotation carries its + removals and sticky `ps.o`, `ps.oc` and `ps.iv` into the new row, never + positive staged IDs. +- **Markers are answered without busy writes.** A KV read answers a marker + when the live row makes that provider ineligible (eligibility steps 4 to + 6), as well as by the rules of section 6.2. Phase 4 adds + `ps.n = max(ps.n, now + 60 s)` only for a marked provider it did not dispatch for a + reason the live row does not show, such as the burst cap or the breaker. A + rebuilt `request` that returns `None` already sets `ps.n` (section 8). If + the flush fails, the marker expires at its deadline as before. +- **Unowned dry runs need KV.** A dry run for a provider whose sources are + unowned returns `Some` only when the phase 1 read showed the source missing + and the pull-complete marker is absent. Otherwise the cookie cannot know, + and no marker is set. +- **Staged entries trigger a reconciling read.** A cookie holding `k: 0` + entries makes the writer read KV in phase 1. That read flips flushed + entries to `k: 1` on this response (phase 3's "the phase 1 read changed the + cookie" rule). Phase 4 reuses that generation only when no after-send + provider is marked. Otherwise it re-reads the live row before any + dispatch, so a withdrawal that lands after send is always seen. +- **Withdrawn consent never refills the cookie.** A stager whose decoded + consent denies EID persistence never writes positive entries into + `ts-eids-s`, so a concurrent `/auction` that started before a withdrawal + cannot bring IDs back. It may still set removals in a new `ts-eids-r` + cookie. +- **The cookie is rewritten only when it changes.** A request that read KV + re-emits `ts-eids-s` only when its entries, markers or `ps` changed, or + when the inbound `kc` is older than half of `kv_revalidate_sec`. Phase 3 + stamps `kc` only when this request read KV. This keeps `Set-Cookie` off + most responses and narrows the window for concurrent overwrites. +- **Cluster size rides on a write.** `/_ts/api/v1/identify` is a + cross-site, bearer-token endpoint (`ec/identify.rs:23-60`), so it cannot + rely on TS cookies. When it reads a row without `cluster_size`, it + computes the value for its response and does not store it. A writer whose + phase 1 or phase 4 read finds a live row without `cluster_size` adds the + evaluation to its mutation, run in phase 4 after send, so its KV prefix + list (`ec/kv.rs:1460-1469`) never delays a page. That is one list and at + most one write per EC, once, as today (`ec/kv.rs:1497-1529`); the write + is the page load's one write. +- **The size cap always holds.** If dropping every droppable entry still + leaves `ts-eids-s` over its cap (section 6.2), it is replaced by a minimal + cookie with `kc = 0` and the markers only, which forces a read at the next + navigation. A source that is stubbed on every rewrite is left as a stub, + and the auction reads it from KV only when the navigation's auction needs + that source. +- **EC minting on non-writer navigations.** Today a real browser gets an EC + on any navigation `is_navigation_request` accepts, including prefetch and + the `Accept: text/html` fallback (`adapter-fastly/src/app.rs:846-857`). + That stays: such a request creates the row before routing, as a new + visitor's writer does, and that create is its page load's one write. It + runs no providers and writes nothing else. +- **Proxy capture plumbing.** `ProxyDispatchInput` gains `is_real_browser`, + the writer decision and `IdentityDeps`. To capture, `handle_proxy` buffers + the response body up to `max_response_bytes` (a larger body is passed + through uncaptured), decodes a copy for `capture`, and re-wraps the + original bytes with `EdgeBody::from(bytes)`, so the browser receives the + response unchanged. + +## 6. Storage + +### 6.1 KV record + +`KvPartnerId` (`ec/kv_types.rs:102`) gains optional fields. `KvEntry` gains +`ps`. All new fields use `#[serde(default, skip_serializing_if = ...)]`, so +`SCHEMA_VERSION` stays 1 and old rows read unchanged. KV item metadata, which +Fastly caps at 2,048 bytes, holds no IDs (`ec/kv_types.rs:199-219`) and is +unaffected. + +`KvPartnerId` and `KvEntry` are public structs without `Default`, and about +30 `KvPartnerId { .. }` literals in 8 files would stop compiling (for example +`ec/kv.rs:160`, `:208`), as would the constructors at `ec/kv_types.rs:282`, +`:312` and `:344`. So both derive `Default` and gain constructors, and the +literals move to them. `PartnerIdUpdate` (`ec/kv.rs:75-90`) gains +`acquired_at`, so browser ingest records `at`. + +```json +{ + "ids": { + "id5-sync.com": { + "uid": "ID5*example", + "w": "id5", + "at": 1760000000, + "exp": 1767776000, + "ra": 1760086400, + "atype": 1, + "ext": { "linkType": 2 } + }, + "sharedid.org": { "uid": "example-shared-id", "at": 1759990000 } + }, + "ps": { + "id5": { "n": 1760003600, "f": 1, "st": "example-opaque-state" } + } +} +``` + +| Field | Meaning | +| ------- | ------------------------------------------------------------------------------------------------------------------- | +| `w` | Writer: a provider ID. Absent for browser ingest and legacy rows | +| `at` | Unix seconds when the ID was acquired or last refreshed | +| `exp` | Unix seconds after which the ID must not be sent | +| `ra` | Unix seconds after which the ID is due for refresh | +| `atype` | Provider-supplied atype. Absent: use `[[ec.partners]].openrtb_atype` | +| `ext` | UID-level ext, at most 256 serialized bytes | +| `ps.n` | Unix seconds before which the provider must not be called | +| `ps.f` | Consecutive failures, for backoff | +| `ps.o` | Sticky opt-out: `{at, fp}`, when it was reported and the consent fingerprint | +| `ps.oc` | When the provider's opt-out was last cleared. An opt-out reported at or before it is stale and never re-applied | +| `ps.iv` | Keyed hashes of UIDs the owner invalidated, at most 4, so they are not re-seeded | +| `ps.st` | The provider's private state, as embedded JSON (a raw value, not an escaped string) of at most 1,024 bytes. KV only | + +`KvEntry` validation (`ec/kv_types.rs:375`) adds: `ext` at most 256 bytes, +`st` at most 1,024 bytes of embedded JSON, `ps` at most 32 providers. +Mutations prune the `ps` entries of providers no longer configured, and more +than 32 enabled providers is a config error (section 10.2), so the cap is +never reached in practice. With every cap reached, +eight owned sources come to about 7 KB, plus 1 KB for each provider with +state. A row normally carries one or two stateful providers. That is above the ~5 KB estimate in +`ec/kv_types.rs:40`, which is a documented estimate rather than an enforced +limit. Fastly's value limit is far larger. + +The admin EC lookup returns the stored JSON verbatim (`ec/admin.rs:394-397`). +It redacts `ps.st`. + +**Rollback rule.** The current reader drops unknown fields when it writes the +row again. If TS is rolled back after providers have written metadata, the +next write strips `w`, `exp` and the rest. So, for an owned source whose +provider marks the source as expiring, only records with `w` set to the owner and an +unexpired `exp` are sent to bidders from the server. Any other record for +that source (browser-seeded, or stripped by an older writer) is used only as +the input to the owner's refresh. This fails closed: an ID is re-fetched, +never sent past its real expiry. + +### 6.2 The `ts-eids-s` and `ts-eids-r` cookies + +Two server-owned cookies: + +- **`ts-eids-s`** (`__Host-ts-eids-s`) caches provider-collected IDs for owned + sources, holds positive entries stagers saved, and carries pending markers + and mirrored `ps`. +- **`ts-eids-r`** is a family of up to four cookies, + `__Host-ts-eids-r-`, that hold removals and opt-outs not yet in KV. + Each response that produces a removal or opt-out sets a new one, with a + fresh random 8-hex-digit ``, holding only its own batch. No response + rewrites another's cookie, so neither a concurrent `/auction` rewriting + `ts-eids-s` nor another removal in flight can lose a batch. Stagers never + delete one. They are deleted, by name, in three cases only: by the writer + navigation, after its phase 1 read shows a cookie's entries all applied + or the cookie fails to open; by any request whose KV read confirms a + tombstone; and by an orphan rotation whose create was `Written`. That keeps a capture's revocation, which + nothing can re-derive, safe (section 5.3). + +**Attributes.** Both use `Path=/; Secure; SameSite=Lax; HttpOnly`, with +`Max-Age=31536000`. The `ts-eids-r` cookies get the same lifetime so that a +batch never expires before a writer applies it; they are deleted once +applied, and their count and size caps bound them. The +`__Host-` prefix makes them host-only, with no `Domain`. They are caches of +KV, so per-host copies are fine. The repo's other server-set cookie, the +pull-complete marker, is host-only for the same reason: a `Domain` attribute +would let sibling hosts that share the passphrase replay it +(`pull_sync_marker.rs:198-201`). Host-only also stops a sibling subdomain from +reading the IDs or planting a competing cookie. A request carrying two +cookies of the same name is treated as carrying none. + +**Sealed, not just signed.** These cookies are stripped before the origin +and Prebid Server (section 6.2, outbound requests), but they still travel in +every request from the browser, through any intermediary that logs headers. +A signed cookie would show server-only IDs, such as UID2 tokens and LiveRamp +envelopes, in plain text there, and to the origin if the strip ever +regressed. So each value is sealed with AES-256-GCM: +`v1..`, base64url without padding. The key is +derived from `ec.passphrase` with a namespaced label per cookie +(`trusted-server/ts-eids-s/key/v1`, `trusted-server/ts-eids-r/key/v1`), +following the pull-complete marker's pattern. The nonce is 12 random bytes +from the platform generator. The full EC ID is the associated data, so it is +authenticated but not stored: + +- A cookie copied from another visitor, or left over after an EC rotation, + fails to open and is ignored. +- The payload of `ts-eids-s` carries `kc` and `cf`. A `cf` that does not match + the current identity config makes its positive entries be ignored and + rebuilt from KV (section 5.6). `ts-eids-r` carries no `cf`. +- An old `ts-eids-s` replayed by its own visitor carries an old `kc`, which + forces a KV read (section 6.3), and the read overrides it. Replay can delay + a removal by at most `kv_revalidate_sec`. + +`ec_hash` is not used for binding, because it is the HMAC of the client IP +and is shared by every browser behind one address (`ec/generation.rs:83-93`). +A cookie that fails to open or decode is ignored and rebuilt by the next +request that reads KV. The failure is logged without the value. Sealing +needs an AES-GCM implementation, a new dependency (section 14), which the +UID2 provider needs anyway (section 12.3). + +**Why it is sealed.** Staged entries become KV writes, and sealing proves TS +wrote them. Without it, a visitor could plant an arbitrary ID in KV, where it +would also be sent from the server on routes without the cookie. Encryption +also keeps the IDs out of origin logs and browser extensions. + +**`ts-eids-s` payload.** Compact JSON. A source is written as its index into +the sorted list of configured sources, which `cf` fingerprints, so a config +change that reorders sources also invalidates the indexes. + +```json +{ + "v": 1, + "kc": 1760000000, + "cf": "1a2b3c4d", + "ids": [ + { + "s": 2, + "u": "ID5*example", + "w": "id5", + "t": 1760000000, + "e": 1767776000, + "r": 1760086400, + "a": 1, + "x": { "linkType": 2 }, + "pv": "3f2a9c1d04b7e6a8", + "sa": 1760000000, + "k": 0 + }, + { + "s": 4, + "u": "example-envelope", + "t": 1760000000, + "sa": 1760000000, + "b": 1, + "k": 0 + }, + { "s": 0, "d": 1 } + ], + "ps": { + "id5": { "n": 1760003600, "f": 1, "sg": 1 }, + "pull-ssp-example-com": { "p": 1760000000 } + } +} +``` + +| Field | Meaning | +| ------------- | ------------------------------------------------------------------------------------------ | +| `kc` | When a request last confirmed this cookie against the KV row | +| `cf` | Fingerprint of the identity config: providers, sources and owners | +| `m` | A request saw this EC's row missing. The next writer reads KV in phase 1 | +| `s` | The source, as an index into the configured source list | +| `u`…`x` | Mirror the KV fields `uid`, `w`, `at`, `exp`, `ra`, `atype`, `ext` | +| `pv` | Keyed hash of the revision this entry replaces. Absent: none | +| `sa` | When the entry was staged | +| `b` | `1` for a browser KV-gap entry (`FillMissing`) | +| `k` | `1` when the entry is known to be in KV, `0` when staged and not yet written | +| `d` | A stub: the entry was dropped for size and is still in KV | +| `ps.n/f/o/iv` | Mirror KV `ps`. `sg: 1` marks a staged change not yet written | +| `ps.oc` | A staged opt-out clear from a capture (section 8), with its time | +| `ps.p` | Pending marker: when an after-send dispatch happened whose result is not in the cookie yet | + +**`ts-eids-r` payload.** Removals as `{s, w, pv?, sa}`, where `s` is the +source domain itself, never an index: `ts-eids-r` carries no `cf`, so an +index could point at another source after a config change. An entry +without `pv` removes the source's record if `w` wrote it and acquired it +before `sa`. Opt-outs as `o: {: }`, and invalidated UIDs as +`iv: {: []}`. + +An entry is **applied**, and is skipped if the mutation meets it again, +when the live row shows: + +- a removal with `pv`: the record is gone or at another revision +- a removal without `pv`: the record is gone, written by another writer, or + acquired at or after `sa` +- an opt-out: KV's `ps.o.at` is at or after `at`, or KV's `ps.oc` is after + `at`, so a cleared opt-out is never re-applied + +`iv` entries never keep a cookie alive: they are applied with the cookie's +other entries and then dropped with it. The writer's phase 1 read deletes +each cookie whose entries are all applied, and any `__Host-ts-eids-r-` +cookie that fails to open, for example one sealed under an old key. A +cookie that is only partly applied is kept whole, and its applied entries +are skipped when re-applied. + +Each cookie's sealed value is at most 512 bytes. A batch over that +coalesces a provider's removals into one provider-wide removal for each +source it claims, then drops its `iv` hashes (losing them allows one +re-seed, section 7.2), and then splits into a second cookie. When four +openable `ts-eids-r` cookies already exist, a response merges its batch +into the openable one with the lowest `` instead of adding a fifth. If +the merge does not fit, the response rewrites all four as one +provider-wide removal and opt-out per provider, which at 32 providers fits +the 2 KB of four cookies. Those rewrites are the only remaining race: two +responses rewriting the same cookies at once, while four batches wait for +a writer. + +**Staged `ps` changes** merge at flush: `n` takes the later value, `f` the +cookie's, `iv` is the union, and an opt-out from `ts-eids-r` is kept unless +it is applied or stale under the rules above. A staged clear `ps.oc` is +dropped from `ts-eids-s` once KV holds an equal or later `ps.oc`. +A mirrored `o` in `ts-eids-s` yields to KV (section 4.7). + +**Pending markers.** While a marker is unanswered, the provider is not +dispatched again, and the next writer request reads KV in phase 1 (section +6.3). Until that read, the provider's cookie entries are not used in +auctions, because its after-send result may have removed them. A KV read +answers a marker when any of these hold, and the reader then clears it and +copies the result into the cookie, removals included: + +- `ps.n` for the provider is at or after `p`. +- `ps.o` for the provider is set. +- A source the provider claims has `at` at or after `p`, or lost the value + the cookie holds. +- The live row makes the provider ineligible (section 5.6). + +A marker that a read does not answer by `p + timeout_ms + pending_grace_sec` +is dropped: the dispatch was cancelled or its flush failed, and the provider +is eligible again. A marker cannot block a provider for good. + +**Size cap.** At most 2,048 bytes for the sealed value of `ts-eids-s`. When it +is over, entries are dropped in this order: + +1. Entries with `k: 1` become stubs `{s, d: 1}`. They are still in KV. A + stub makes the next request that needs the source read KV. +2. `ps` entries with no marker and no staged change. They are still in KV. +3. Provider entries with `k: 0`, in ascending `cookie_priority` (section + 10.1). Each one lost costs a later provider call. +4. Browser KV-gap entries (`b: 1`), last, because nothing re-derives one + unless the browser sends it on another `/auction` (section 5.3). + +Pending markers are never dropped. Each drop is logged with the provider and +source. + +**Who writes them.** + +- **`ts-eids-s`:** the writer request's HTML response only when needed + (section 5.2, phase 3), because any HTML response with `Set-Cookie` becomes + `private, max-age=0` (section 3.4); and stagers, whenever they save an + entry or their KV read changes the cookie (section 5.6). Their JSON + responses are already private. +- **`ts-eids-r`:** any response that produces a removal or opt-out, on any + route, including the unusable-consent path (section 5.3), adds a new + cookie for its batch; and the writer navigation's HTML response deletes + the cookies its phase 1 read showed applied. +- **A consent withdrawal, or any KV read that finds a tombstone,** sets + `Max-Age=0` on `ts-eids-s`. Once the tombstone is confirmed (written by + this request, or read), the `ts-eids-r` cookies are pointless and are + deleted too, because the tombstone removes everything. If the + withdrawal's tombstone write failed, they are kept. + +The browser shim never reads or writes either cookie. Both join `ts-ec`, +`ts-eids` and sharedId in the list of identity cookies the template cache +refuses as cache keys (`cookies/template_cache_policy.rs:35`), so no operator +can split the shared template cache per visitor with them. + +**Outbound requests.** TS's identity cookies never reach third parties. One +list, `TS_IDENTITY_COOKIE_NAMES` (`ts-ec`, `ts-eids`, `__Host-ts-eids-s`, +every `__Host-ts-eids-r-` cookie by prefix, `ts-ec-pull-complete`), is stripped from Prebid Server +requests and never added to proxy or provider requests. The publisher origin +is not a third party, but it gets only the small ones: + +- **Publisher origin. ID cookies stripped.** The publisher proxy removes + `ts-eids`, `__Host-ts-eids-s` and every `__Host-ts-eids-r-` cookie + (`TS_ORIGIN_STRIPPED_COOKIE_NAMES`) from the `Cookie` header it forwards + (`publisher.rs:3124-3147`), and drops the header when nothing is left. + `ts-ec` and `ts-ec-pull-complete` still reach the origin, as today, + because publishers' servers may read `ts-ec`. Without the strip the ID cookies + add up to about 5.5 KB to every origin request, and up to 2 KB more while + removal batches wait, and common origin + servers reject a header line over about 8 KB with 400 or 431, which would + fail the page itself. Stripping `ts-eids` from the origin is a behaviour + change: an origin that read it must read `x-ts-eids` or ask TS instead. A + metric still reports the inbound `Cookie` header size (section 14). +- **Prebid Server.** Stripped in the Prebid transport before the + consent-forwarding mode is checked (`integrations/prebid.rs:2091-2108`), so + it applies in every mode, including the early return that forwards the raw + header. A `Cookie` header that is not valid UTF-8 is dropped, not + forwarded. The call site passes every `Cookie` value, not one. +- **Integration proxies.** They already use header allowlists without + `Cookie` (`integrations/lockr.rs:245`, `didomi.rs:258`, `permutive.rs:259`). +- **Identity providers.** The coordinator rejects any request with a `Cookie` + header (section 4.7). + +The strip iterates every `Cookie` field, as `gpt_diagnostics.rs:394-416` does, +and drops the header when nothing is left. Only the routes of section 6.3 +open the cookies, lazily, at most once per request, keeping the result on +`EcContext`. Assets and other routes never decrypt them: the origin and +Prebid Server strips match cookie names only. The derived keys, `cf` and +the source index are computed once per settings load, not per request. + +### 6.3 Read order + +For a request with a valid EC that uses identity (the writer request, +`/auction`, `/_ts/page-bids`, a proxy capture, or any request dispatching a +hold-mode provider): + +1. If `ts-eids-s` opens, use its IDs for owned sources (skipping expired + entries, stubs and the entries of providers with an unanswered marker), + its staged entries, and its `ps` for eligibility (section 4.7). Whether + or not it opens, apply `ts-eids-r` on top: its removals hide the records + they name, from the cookie view and from KV, and its opt-outs block their + providers. +2. **The writer's phase 1** reads the KV snapshot when any of these hold: + - `ts-eids-s` is missing, fails to open, or has a `cf` that does not + match + - `now - kc` exceeds `kv_revalidate_sec` + - `m` is set + - any `ts-eids-r` cookie is present, so the writer can delete the + applied ones + - any pending marker is unanswered + - the navigation's auction needs a source that is only a stub + - a hold-mode provider is due, so the read is its pre-disclosure + revalidation + - an after-send provider without private state is eligible on the + cookie view and has a candidate source that is unowned or only a stub, + so its dry run sees what KV holds (section 5.2). A valid pull-complete + marker removes a generic-pull provider's sources from the cookie-view + candidates, as today, and step 3's cookie-only `ps.n` keeps this from + firing on every navigation. + - the navigation runs an auction that needs the row, as today's preload + decides (`auction_needs_row`, `publisher.rs:3110-3117`). Today that is + every auction navigation with unowned bidstream sources, because the + cookie caches no unowned values + - consent is withdrawn and the tombstone path needs the row, as today + + `/auction` and `/_ts/page-bids` keep their existing reads and gain no new + triggers. + +3. **Any request that read KV** merges the two as below and sets `kc = now`. + It re-emits the cookie only when the merge changed it or the inbound `kc` + is older than half of `kv_revalidate_sec` (section 5.6): + - A tombstone, or `consent.ok = false`, empties the server set and expires + the cookie. + - A missing row, unless this request is about to recover it, replaces + `ts-eids-s` with a minimal cookie holding only `m: 1`: no positive + entries and no markers (section 5.6). + - For each owned source, the KV record wins, with one exception: a staged + provider entry that `OwnerWrite` would still apply (its `pv` matches, or + it has no `pv` and KV holds only a value the provider did not write) + stays `k: 0`. + - A `k: 1` entry whose source KV no longer holds, or holds at a different + revision, is dropped or replaced. + - A staged entry KV already holds becomes `k: 1`. A `b: 1` entry is dropped + once KV holds any value for its source. + - A stub is replaced by the KV record. + - Answered pending markers are cleared, and `m` is cleared when the row is + present. + - A provider the row shows ineligible because all its unowned candidate + sources are present gets `ps.n = now + refresh_after_sec` in the + cookie only, stored with the KV `n` it saw. A later read that shows a + different KV `n` (an `invalidate` sets `n = now`, for example) drops + the cookie-only value. Owned sources need no such value: the cookie + caches them with their `ra` and `exp`, so the cookie view already knows + when they are due. + - On the writer only, each `ts-eids-r` cookie whose entries the row shows + all applied is deleted. Stagers leave the `ts-eids-r` cookies alone. + + The writer's HTML response carries the result only under phase 3's rules, + which include "the phase 1 read changed the cookie". + +Missing cookie, KV present: the writer request reads KV and rebuilds the +cookie. The visitor loses nothing. + +## 7. Auction merge and ownership + +### 7.1 Merge + +`merge_auction_eids` (`auction/endpoints.rs:505`), and the navigation and +page-bids merge (`publisher.rs:5547-5568`), take a third input: the server +set. It holds IDs from `ts-eids-s` (including staged entries) and KV, with +expired entries, stubs and removed entries left out. The server set requires +what today's KV path requires: `ec_allowed()`, and a live consenting row +whenever the request read KV (`endpoints.rs:446-460`). Each server-side ID +must also pass its owning provider's consent policy for this request, so the +sharing and targeted-advertising opt-outs and GPC stop it even when the +generic gates pass. + +For each source, the value sent to bidders is: + +| Source kind | Browser sent the source | Browser did not send it | +| ----------- | ------------------------------------------- | ----------------------------------------------------------------------------- | +| Owned | the owner's valid value, else the browser's | the owner's valid value, else a browser-seeded value if the IDs do not expire | +| Unowned | the browser's value (today's rule) | the server value, if any (today's rule) | + +"The owner's valid value" means a record with `w` set to the owner that is +not past its `exp`. A browser-seeded record never beats the browser's +current value. It may be stale, which is the problem the EID fidelity work +fixed (D9). For a provider whose IDs expire, a browser-seeded record is never +sent from the server at all (section 6.1). + +A server value uses its stored `atype`, else `[[ec.partners]].openrtb_atype`, +and its stored UID-level `ext`. A value the provider wrote also gets the +EID-level `inserter`, `matcher` and `mm` the provider declared for that +source (`with_provenance`, section 4.3). A browser value keeps the browser's +own EID-level fields, as the EID fidelity work requires. Sources with `bidstream_enabled = false` are +left out, as `resolve_partner_ids` does today (`ec/eids.rs:36`). + +The merged set is consent-gated (`gate_eids_by_consent`) and reported in +`x-ts-eids` exactly as today, with the same 4 KB cap. `x-ts-eids` therefore +shows server-only IDs. The calling script can read it, and so can any page +script that wraps `fetch`, so `HttpOnly` on `ts-eids-s` does not hide server +IDs from the page. The header stays for validation and troubleshooting, and +this exposure is accepted. + +One shared servability check covers expiry, the rollback rule, the owner's +consent policy, and opt-outs: a record whose writer has `ps.o` set, in KV or +in `ts-eids-r`, is never served, even before the removal reaches KV. `resolve_partner_ids` (`ec/eids.rs:36`), the admin view +(`ec/admin.rs:485`) and `/_ts/api/v1/identify` all use it. `/identify` reads +`entry.ids` directly today (`ec/identify.rs:99-103`), so it switches to the +check. The stale module comment at `ec/eids.rs:1-5` is corrected. So expired +IDs are never returned by any of them (G7). + +### 7.2 Ownership semantics + +- **Owned source.** The owner is authoritative. It refreshes when due + (section 4.7, step 4), and its valid value wins in the auction. Browser + ingest and KV-gap entries may only fill the source when it is absent or + its record is expired. + That seeds it: the owner treats a value it did not write as due, and + refreshes it at once. No other provider may claim the source (section + 10.2). +- **Invalidated values are not re-seeded.** When the owner invalidates a + value, its keyed hash goes into `ps.iv`. Browser ingest and KV-gap entries + skip a UID in `ps.iv`, so the browser cannot re-seed the value the owner + just rejected, and no invalidate-reseed-refresh loop starts. +- **Unowned source.** Any provider claiming it, and the browser, write with + `FillMissing` only. Unowned sources are never refreshed. This is exactly + today's pull-sync contract. The auction is browser-first. +- **Batch sync.** A batch-sync update to an owned source applies only when + the source is absent (`ec/batch_sync.rs:247`). It never replaces the + owner's value. Unowned sources keep today's batch-sync behaviour. + +## 8. Errors and outcomes + +Every effect below goes into the mutation (writer request), or a stager's +cookies: positive changes into `ts-eids-s`, removals and opt-outs into +`ts-eids-r`. + +| Result or error | Stored IDs | `ps` | Log | +| ------------------------------------------------------------------------------ | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ----- | +| `Resolution::found` | `OwnerWrite` or `FillMissing` per source | `f = 0`; `st` replaced or cleared when the resolution says so | debug | +| `Resolution::no_match` | unchanged | `n = now + retry_in` (default 24 h, max 7 d) | debug | +| `Resolution::invalidate`, `also_invalidate` | `OwnerRemove` with the revision the request read | UID hash added to `iv`; `n = now`: eligible for `Initial` | info | +| `Resolution::opted_out` | `WriterRemove` for every record this provider wrote, owned or unowned | `o = {at: now, fp}`, `st` cleared | info | +| `with_retry_in` on any resolution | as above | `n` is at least `now + retry_in` | debug | +| `request` returns `None`, or `InputUnavailable` | unchanged | `n = now + refresh_after_sec` | debug | +| A recorded `InvalidState`, whatever the provider returned | as the result says | `st` cleared, `f` unchanged, `n = now + refresh_after_sec` unless the result sets `n` | warn | +| Hold-mode budget spent before the answer | unchanged | `n = now + hold_retry_sec` | debug | +| `Rejected` with a 4xx other than 429, `Config`, `UndeclaredInput`, `Malformed` | unchanged | `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)` | warn | +| `Rejected` with 429 or 5xx, `Transport`, timeout | unchanged | `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`; breaker `record` | info | +| Breaker open | not dispatched | unchanged | debug | +| Capture: any result or error | IDs and removals as above | only `o`; never `n`, `f` or the breaker | debug | + +**Sticky opt-outs.** `ps.o = {at, fp}` records a provider-reported opt-out +and the consent fingerprint at that time. Server-side calls do not carry the +vendor's own opt-out cookie, so `o` is TS's only record of it. Applying it +removes every record the provider wrote (section 5.4), and while it is set +no record that provider wrote is served (section 7.1). It is cleared only by +an affirmative re-grant for this provider: + +- **A provider with a GVL ID:** a TC string whose `LastUpdated` is after `at` + and that grants the provider's vendor and purposes. +- **Any provider, under GPP:** the applicable US section showing no sale, + sharing or targeted-advertising opt-out for the signals the provider's + policy reads, where the fingerprint at `at` showed one. GPP carries no + creation time, so the clear is by section values. +- **A capture-only provider without a GVL ID, such as Lockr:** a later 2xx + capture that returns IDs, under consenting cookie and body consent. That + is the vendor issuing again after the visitor re-consented through it. + The capture's proxy request is a stager, so it stages `ps.oc` beside the + IDs in `ts-eids-s`, and the next writer applies the clear before the IDs + (section 5.4). If a concurrent `/auction` overwrites that cookie, both + are lost together, and the next capture stages them again. +- **An operator,** through the admin endpoints. + +A change of jurisdiction, a newly present GPP string, or a privacy signal the +provider relaxes never clears it. A clear is written to KV, and the +browser's mirrored copy yields to it (section 4.7). A consent withdrawal's +tombstone removes everything, provider opt-outs included: after it, the +visitor starts again with a new EC. + +The **consent fingerprint** is the first 16 hex digits of an HMAC, keyed from +`ec.passphrase`, over the consent signals the provider's policy reads: + +- its TCF purpose bits and vendor bit +- the US sale, sharing and targeted-advertising opt-outs it does not relax +- GPC and DNT, unless relaxed + +It is not computed over the raw consent string, which changes whenever a CMP +re-saves it. + +Rules: + +- No error ever deletes a still-valid ID. Only `Resolution::invalidate` and + `also_invalidate` (by revision) and `opted_out` (everything the provider + wrote) delete. +- An expired ID is not permission to call. Eligibility still applies. +- Errors are logged with provider ID, outcome and status only (section 9.3). +- A provider whose IDs expire and whose refresh keeps failing loses its ID + at `exp`, not before. + +## 9. Security and privacy + +### 9.1 Consent + +- **Before dispatch:** global EC consent, EID-persistence consent, and the + descriptor's `ConsentPolicy` (section 4.3). "Where TCF applies" means + `gdpr_applies` or a jurisdiction of `Gdpr` or `Unknown`, matching + `consent_allows_server_side_auction` (`consent/mod.rs:298-308`). Each check + maps to a signal TS decodes: + - TCF purposes and vendor consents (`consent/tcf.rs:145`). A small + `has_vendor_consent` helper is new. + - The US sale opt-out, from GPP US sections (`consent/gpp.rs:74`, `:125`) + or the US Privacy string (`consent/us_privacy.rs:50-63`). + - The US sharing and targeted-advertising opt-outs. This work extends the + GPP decoder to extract them next to `decode_us_sale_opt_out` + (`consent/gpp.rs:125`). The `iab_gpp` crate already exposes + `sharing_opt_out` in the US National and California sections, and + `targeted_advertising_opt_out` in the US National section and the state + sections that define it. + - GPP section 6, the US Privacy string inside GPP, which the decoder skips + today (`consent/gpp.rs:112`), and the US National section's GPC and + known-child fields. This work decodes them too. A known child without + consent blocks every provider. + - `Sec-GPC` (`consent/extraction.rs:25-50`). + - DNT, captured in phase 1 (section 4.4). TS reads it today only for the + auction (`auction/routing.rs:336`). +- **Fail closed.** `strict()` blocks when a raw TC, GPP or US Privacy string + is present but has no decoded form, for example in proxy mode or after a + decode failure (`consent/mod.rs:104-127`, `:197-210`). +- **Before disclosure:** the live-row read and eligibility recomputation + described in section 4.7. +- **At write:** the CAS on that read's generation. +- **At flush and on every cookie write:** staged positive entries, and IDs + written into `ts-eids-s`, pass the generic gate and the writing provider's + policy against the current consent (section 5.3). Removals and opt-outs + never need consent to apply (section 5.6). +- **In the bidstream and `/identify`:** server-side IDs pass the owner's + policy (section 7.1). +- **On withdrawal:** the tombstone is written synchronously on whichever + request carries it, and the response expires `ts-eids-s` (section 5.1). + +### 9.2 Outbound requests + +- HTTPS only. Redirects are not followed. +- The host must be one of the descriptor's `resolves` hosts. +- Providers build headers from scratch. The coordinator rejects any request + with a `Cookie` header, and TS's identity cookies are stripped from Prebid + Server requests. The publisher origin receives `ts-ec` but not the ID + cookies (section 6.2). +- Providers get only the inputs their descriptor declares (section 4.4). + Only generic pull receives the EC ID. Location is coarse (country and + subdivision), page URLs carry no query string, and the consent view + carries no decoded jurisdiction, though the raw GPP string still shows the + applicable state section (section 4.4). A URL path can still carry + identifiers, so most providers should want `PageUrl` rather than need it. +- The client IP comes only from the platform's client-IP API. A provider that + needs it in `X-Forwarded-For` sets it from `ResolveInput::client_ip`. + Client-supplied forwarding headers are never copied. +- Response bytes are capped before and after decompression. + +### 9.3 Logging and data handling + +- Never log UIDs, envelopes, HEMs, provider state, consent strings, cookie + values, or provider URLs with a query string. EC IDs go through `log_id` + (`ec/mod.rs:63`). +- Provider errors are logged with `current_context()` only, never with + `{:?}` of the report, whose attachments could carry a URL or body. +- `ProviderRequest`, `ProviderResponse`, `ResolveInput`, `CaptureInput`, + `StoredId`, `ResolvedId`, `ProviderConsent` and `HashedEmail` have + redacting `Debug` implementations. `ConsentContext` gets one too, because + it derives `Debug` over the raw consent strings today + (`consent/types.rs:109`). +- The Lockr proxy logs only the path of the upstream URL, not its query + (`integrations/lockr.rs:202-204`). +- Hashes of identifiers (the rate-limit visitor key, `pv`, `ps.iv`, the + consent fingerprint) are HMACs with keys derived from `ec.passphrase`, so + they cannot be linked to an EC ID without the key. +- `CaptureInput` never exposes request bodies or request headers. Proxied + requests can contain an email or HEM. TS forwards them, and reads from a + body only the consent pointers a descriptor declares (section 4.7). It + never reads anything else for identity. +- Provider state stays in KV. It is never sent to bidders, put in a cookie or + returned by the admin lookup. + +### 9.4 Caching + +Responses that set `ts-eids-s` follow the same cookie-privacy rule as every +cookie-bearing response: `private, max-age=0`, with edge-cache headers +removed (`response_privacy.rs:122-145`). The Fastly terminal hook re-applies +it after late mutations (`adapter-fastly/src/main.rs:745`). Section 5.2 keeps +`ts-eids-s` off returning-visitor HTML except when something is due, changed +or stale. + +## 10. Configuration + +### 10.1 Layers + +```toml +# Coordinator policy: vendor-neutral, one place for every timing choice. +[ec.identity] +# Replaces ec.pull_sync_concurrency. The old key is accepted and deprecated. +max_concurrency = 4 +# Shared wall-clock budget for all hold-mode providers, in milliseconds. +# Default 100 (section 4.7 explains why). +hold_budget_ms = 100 +# After a call is cut off by the hold budget, wait this long before the +# next. Default 300. +hold_retry_sec = 300 +# Per provider response, before and after decoding. +max_response_bytes = 65536 +# Seconds a pending marker outlives its provider's timeout. Default 300. +pending_grace_sec = 300 +# Staged cookie entries older than this are not written. Default 604800. +staged_max_age_sec = 604800 +# A cookie not confirmed against KV for this long forces a KV read. +# Must be below the 24-hour tombstone TTL. Default 3600. +kv_revalidate_sec = 3600 + +[ec.identity.providers.id5] +# "hold" or "after_send". Default "after_send". ID5 keeps private state, so +# it runs only on the writer navigation: before the auction when that +# navigation runs one, after send when it does not (section 4.7, step 8). +mode = "hold" +# Hold mode: min(timeout_ms, remaining hold budget). Default: hold_budget_ms +# in hold mode, 1000 after send. +timeout_ms = 80 +# Narrows the sources this provider may write. Default: every source its +# descriptor declares. Narrowed here to the one source registered below. +sources = ["id5-sync.com"] +# Refresh interval when the provider gives no refresh hint, and the wait +# after a skipped call. Default 86400. +refresh_after_sec = 86400 +# Burst cap per visitor, enforced in 60-second windows. Default 1. +max_calls_per_minute = 1 +# Skip the provider for every visitor once this many transport failures, +# timeouts, 429s or 5xx responses happen within one 60-second window. +# Default 50. +breaker_failures_per_minute = 50 +# Lower is dropped first when ts-eids-s exceeds its cap. Default 0. +cookie_priority = 10 + +# Source policy: the existing table, with one new field. +[[ec.partners]] +name = "ID5" +source_domain = "id5-sync.com" +openrtb_atype = 1 +bidstream_enabled = true +# New. A provider ID that owns this source. Absent: unowned. +owner = "id5" + +# Vendor protocol: the integration's own table, defined by its own spec. +[integrations.id5] +enabled = true +partner_id = 1234 +``` + +`[ec.identity]` and each `[ec.identity.providers.]` table are optional. +A provider with no table uses the defaults. + +### 10.2 Validation + +Deploy-time validation (`validate_settings_for_deploy`, `config.rs:249-262`) +parses integration configs but never builds providers. Each identity +integration gets a line in a new `validate_identity_integration::` list +next to today's per-type list (`config.rs:303-332`). That function parses the +config as `T: IdentityProviderConfig` and calls `build_descriptor`, which +needs no secrets and returns the same descriptor the running provider +stores. A separate pass builds the descriptors of `[[ec.partners]]` pull +providers. The identity check runs next to +`PartnerRegistry::validate_config_for_deploy` (`config.rs:260`) in +`ts config validate` and `ts config push`, and in +`validate_settings_for_runtime` at startup (`config.rs:270-280`). It applies +each provider's `sources` narrowing before checking ownership. + +Each of these is an error: + +- `owner` names a provider that is not enabled. +- `owner` names a provider whose descriptor does not include that source. +- A descriptor fails to build (section 4.3). +- An enabled provider claims a source with no `[[ec.partners]]` entry. +- An enabled provider claims a source owned by a different provider. Use + `sources` to narrow one of them. For example, when native ID5 owns + `id5-sync.com`, Lockr is narrowed to the sources no native provider owns. +- `sources` names a source the provider's descriptor does not declare, or + narrows the provider to no source at all. +- Two providers have the same ID (section 10.4). +- A capture rule lies outside the registering integration's proxy routes. + `IntegrationRegistry::with_plan` reports it at startup (section 4.3). +- `[ec.identity.providers.]` names an unknown provider. +- `mode` is not `hold` or `after_send`. A provider with state runs only on + the writer request in either mode: in hold mode on writer navigations that + run an auction, and after send on those that do not (section 4.7, step 8). +- More than 32 providers are enabled, generic pull partners included. A KV + row holds `ps` for at most 32 (section 6.1). +- `timeout_ms` is greater than `hold_budget_ms` for a hold-mode provider. +- `hold_budget_ms` is over 1,000, `max_concurrency` is 0, + `refresh_after_sec` is 0, or `kv_revalidate_sec` is not below 86,400. + +Validation never calls a provider. Live approvals, such as a LiveRamp +placement or an approved origin, stay on the rollout checklist of each vendor +spec. + +**Secrets.** A provider's secret fields, such as UID2's API key and client +secret, are `Redacted` config fields. At rest the config holds their +secret-store key names, and they are resolved in memory before runtime +settings are built (`secret_resolution.rs`). Each identity integration adds +its secret paths to `TrustedServerAppConfig::secret_fields()` +(`config.rs:136-175`). `build_descriptor` must never read a secret field, +because deploy validation runs before resolution and would see key names. +The harness's `check_descriptor_ignores_secrets` builds the descriptor from +a config whose secrets are placeholder key names and from one with resolved +secrets, so a descriptor that depends on a secret fails the test (section +13). + +### 10.3 Registration + +`IntegrationRegistration::builder(ID)` gets +`.with_identity_provider(Arc)`, next to `.with_proxy()` +(`integrations/registry.rs:663`), and +`.try_with_identity_provider(Result>)`, +which wraps a constructor's result for the common case (section 4.6). An +integration can register more than one provider. The registry exposes `identity_providers()` to build the +coordinator. `handle_proxy` offers a proxied response to every provider whose +descriptor has a capture rule matching it (section 4.7). A compile-time guard, +`const _: Option<&dyn IdentityProvider> = None;`, keeps the trait +object-safe. + +### 10.4 Pull sync on the trait + +Each `[[ec.partners]]` entry with `pull_sync_enabled = true` registers a +built-in `GenericPullProvider`. Its ID is `pull-` plus the source domain with +dots replaced by `-` (for example `pull-ssp-example-com`). An ID that would +exceed 63 characters is cut and given an 8-character hash suffix of the full +domain, so IDs stay unique and valid. + +Unchanged: + +- The protocol: `GET ?ec_id=`, bearer `ts_pull_token`, + response `{"uid": "..."}`, the 64 KiB cap and the + `pull_sync_allowed_domains` allowlist. +- Fill-missing only, unless the operator sets `owner`. +- The hourly rotation of partner order, which keeps dispatch fair under + `max_concurrency` (`pull_sync.rs:143-156`). +- The pull-complete marker, which stays the signal that every pull partner + already has an ID (section 6.3). + +Its descriptor, built from the `[[ec.partners]]` entry: + +- One source, the entry's `source_domain`. Its IDs do not expire. +- `resolves` with the `pull_sync_allowed_domains` as hosts and the + crate-private `with_ec_id()`, because the protocol is keyed by the EC ID. +- A consent policy that blocks exactly what pull sync blocks today. The + migration parity tests (section 13) pin it. +- Default `after_send`, and unowned. +- Optional `owner = "pull-"` makes it authoritative, and the legacy + `pull_sync_ttl_sec` becomes its `refresh_after_sec`. +- `pull_sync_rate_limit` (per hour) maps to + `max_calls_per_minute = max(1, ceil(pull_sync_rate_limit / 60))`, which is + what the limiter enforces today. A `[ec.identity.providers.]` table + overrides it. + +What changes: + +- It is dispatched only on the writer request for a returning EC. Today it + runs after any response whose request loaded a consenting row + (`pull_sync.rs:86`, `:166-174`), including a new visitor's first + navigation. New visitors now reach pull partners from their second page + load. +- Its results go into the single flush. Today it makes its own write. +- Errors back off through `ps` (section 8). Today a failing partner is + retried on every eligible request, within the rate limit. +- The rate-limit key moves from `pull::` + (`pull_sync.rs:373`) to the per-visitor key of section 4.7. Counters reset + at deploy. + +`dispatch_pull_sync` and `build_pull_sync_context` are removed. +`run_edgezero_pull_sync_after_send` (`adapter-fastly/src/main.rs:618`) +becomes the coordinator's phase 4 entry point. + +### 10.5 Overlap with #777 + +The module stack in #777 replaces `[integrations.]` with +`[.]` (for example `[identity.lockr]`) and adds `ts config migrate`. +Under that stack only the vendor protocol layer moves. +`[ec.identity.providers.]` is keyed by the same provider ID and is +unaffected. Whichever lands second adapts its config paths. This spec is +written against `main`. + +## 11. Adapters + +KV identity runs only on Fastly. Axum, Cloudflare and Spin build an +`EcContext` for consent-gated routes, but have no `KvIdentityGraph`, no EC +finalize and no EC generation (`publisher.rs:6896-6899`). + +This design changes nothing on them except the Prebid Server and origin cookie strips, which +apply on every adapter. Providers are never dispatched there, and their +`/auction` and `/_ts/page-bids` keep resolving client EIDs without KV. When +EC identity is ported to them (#777 rows 52 to 54), phase 4 runs after send +where the platform allows it (Cloudflare's `waitUntil`), and before the +response otherwise. + +## 12. Vendor examples + +These examples were written against the API of sections 4.2 to 4.6 to test +it. They are illustrations of the API, not the vendors' real code, and the +framework does not depend on any detail in them. Endpoints, payloads, +consent parameters, size limits and semantics such as what Lockr's +revocation covers are decided in each vendor's implementation plan, which +replaces any shape marked "assumed" below. `IntegrationConfig` impls are +omitted; they match section 4.6. Endpoints come from operator config, shown with +`example.com` hosts. Section 12.5 lists what writing them changed in the +API. + +### 12.1 LiveRamp ATS (active resolution, expiring IDs) + +LiveRamp exercises refresh of an existing ID, expiring IDs, region-dependent +lifetimes, initial resolution that needs a HEM, and headers built from +inputs. The request parameters come from #1245 §9; the response shape and the +consent parameter names are assumed. + +```rust +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct LiveRampConfig { + pub enabled: bool, + /// For example `https://ats.example.com/api/identity/v2/envelope`. + #[validate(url)] + pub endpoint: String, + /// LiveRamp placement ID. An operator identifier, not a secret. + pub placement_id: String, + /// Countries the placement is approved for, as ISO 3166-1 alpha-2. + pub approved_countries: Vec, +} + +impl IdentityProviderConfig for LiveRampConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(LIVERAMP_INTEGRATION_ID.parse()?) + .with_source("liveramp.com".parse()?) + .with_expiring_ids() + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::ClientIp, Input::Region]) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(LIVERAMP_GVL_ID)) + .build() + } +} + +pub struct LiveRampProvider { + descriptor: ProviderDescriptor, + placement_id: String, + approved_countries: Vec, +} + +/// Whether the call was an initial lookup or a refresh. Carried from +/// `request` to `response`. +#[derive(Serialize, Deserialize)] +enum LiveRampCall { + Initial, + Refresh, +} + +/// Assumed shape: V2 returns several envelope products. +#[derive(Deserialize)] +struct EnvelopeReply { + envelopes: Vec, +} + +#[derive(Deserialize)] +struct EnvelopeProduct { + #[serde(rename = "type")] + product_type: u32, + source: String, + value: String, +} + +/// The regular ATS envelope (#1245 §9). +const ATS_ENVELOPE_TYPE: u32 = 19; +const ATS_ENVELOPE_SOURCE: &str = "envelopeLiveramp"; +/// Identifier types (#1245 §2): 4 is a hashed email, 19 an envelope refresh. +const IT_HASHED_EMAIL: &str = "4"; +const IT_ENVELOPE: &str = "19"; + +impl IdentityProvider for LiveRampProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let region = input.region()?; + if !self.approved_countries.iter().any(|country| country == region.country()) { + return Ok(None); + } + + let source = input.primary_source(); + let (call, identifier_type, identifier_value) = match input.stored(source) { + Some(stored) if input.is_refresh(source) => { + (LiveRampCall::Refresh, IT_ENVELOPE, stored.uid().to_owned()) + } + _ => (LiveRampCall::Initial, IT_HASHED_EMAIL, input.hashed_email()?.to_hex()), + }; + + let consent = input.consent(); + let mut request = ProviderRequest::get(input.endpoint()?) + .with_query("pid", &self.placement_id) + .with_query("it", identifier_type) + .with_query("iv", &identifier_value) + .with_query("atype", "3") + .try_with_call_context(&call)? + .try_with_header("origin", &format!("https://{}", input.publisher_domain()))? + .try_with_header("x-forwarded-for", &input.client_ip()?.to_string())?; + if let Some(tc_string) = consent.raw_tc_string() { + request = request.with_query("gdpr_consent", tc_string); + } + if let Some(gpp) = consent.raw_gpp_string() { + request = request.with_query("gpp", gpp); + } + Ok(Some(request)) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let source = input.primary_source(); + let call: LiveRampCall = response + .call_context()? + .ok_or_else(|| IdentityProviderError::config("missing call context"))?; + + if response.status() == StatusCode::NO_CONTENT { + return Ok(match call { + LiveRampCall::Refresh => Resolution::invalidate(source), + LiveRampCall::Initial => Resolution::no_match(), + }); + } + + let reply: EnvelopeReply = response.success_json()?; + let Some(envelope) = reply.envelopes.into_iter().find(|product| { + product.product_type == ATS_ENVELOPE_TYPE && product.source == ATS_ENVELOPE_SOURCE + }) else { + return Ok(Resolution::no_match()); + }; + + let lifetime_days = if input.region()?.is("US", "CA") { 15 } else { 30 }; + let id = ResolvedId::new(source, envelope.value) + .with_atype(3) + .with_expires_in(Duration::from_secs(lifetime_days * 86_400)); + Ok(Resolution::found([id])) + } +} +``` + +What this shows: + +- `request` and `response` both need to know whether the call was an + initial lookup or a refresh, because a 204 means different things. The + call context carries it (section 12.5). +- A `Refresh` is attempted for an expired envelope too (section 4.4, + `ResolveKind`). LiveRamp answers 204 for one it no longer accepts, and the + record is invalidated. +- `Input::HashedEmail` is declared only for `Initial`, so in v1 an owned + `liveramp.com` source is resolved only by `Refresh`, of an envelope that + browser ingest seeded (section 7.2). For an `Initial`, eligibility step 4 + leaves the source out because its needs are missing, so LiveRamp is not + called at all, and `input.hashed_email()?` is never reached. +- A 451 or 4xx goes through `success_json` as `Rejected` and backs off that + visitor only (section 8). +- The example sets no refresh hint, so the operator's `refresh_after_sec` + (default one day) paces refreshes. Each refresh costs a provider call and + a KV write, so a vendor hint belongs here only when LiveRamp documents + one. + +### 12.2 Lockr (passive capture) + +Lockr exercises capture from several exact paths, a revocation path that +must work after consent is denied, consent signals carried in the request +body, vendor key names mapped to sources, expiry in milliseconds, and +EID-level provenance. The paths and payloads come from the earlier Lockr +capture design (`spec/lockr-identity-capture`, its section 2), measured on a +live page. + +```rust +const LOCKR_API: &str = "/integrations/lockr/api/publisher/app/v2/identityLockr"; + +/// Lockr's token key names and the sources they carry. Only confirmed +/// mappings are listed. +const KEY_SOURCES: &[(&str, &str)] = &[ + ("id5id", "id5-sync.com"), + ("_lr_env", "liveramp.com"), + ("__uid2_advertising_token", "uidapi.com"), +]; + +impl IdentityProviderConfig for LockrConfig { + fn build_descriptor(&self) -> Result> { + let mut captures = Captures::new() + .with_request_consent(["/consentString", "/gppString", "/ccpaString"]); + for endpoint in ["page-view", "generate-tokens", "refresh-tokens", "sync-no-hem-ids", "revoke-consent"] { + captures = captures.with_path(Method::POST, &format!("{LOCKR_API}/{endpoint}")); + } + + let mut builder = ProviderDescriptor::builder(LOCKR_INTEGRATION_ID.parse()?) + .with_captures(captures); + for (_, domain) in KEY_SOURCES { + builder = builder.with_source(domain.parse()?); + } + builder + .with_provenance( + &"id5-sync.com".parse()?, + EidProvenance::new().with_inserter("Lockr-For-Publishers").with_matcher("").with_mm(2), + ) + .build() + } +} + +#[derive(Deserialize)] +struct TokenReply { + #[serde(rename = "aimTokens", default)] + aim_tokens: Vec, + #[serde(default)] + ids: BTreeMap, +} + +#[derive(Deserialize)] +struct AimToken { + key_name: String, + /// Milliseconds since the epoch. + identity_expires: Option, +} + +#[derive(Deserialize)] +struct LockrId { + eid: LockrEid, +} + +/// Lockr's own EID shape. Not `openrtb::Eid`, which denies unknown fields +/// (`openrtb.rs:72`, `:124`), so one new Lockr field would fail the whole +/// reply. +#[derive(Deserialize)] +struct LockrEid { + source: String, + #[serde(default)] + uids: Vec, +} + +#[derive(Deserialize)] +struct LockrUid { + id: String, + atype: Option, + ext: Option, +} + +impl IdentityProvider for LockrProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + if input.path().ends_with("/revoke-consent") { + return Ok(Resolution::opted_out()); + } + + let Ok(reply) = input.response().json::() else { + return Ok(Resolution::no_match()); + }; + Ok(Resolution::found(reply.ids.iter().filter_map(|(key, entry)| { + let (_, domain) = KEY_SOURCES.iter().find(|(name, _)| name == key)?; + let source = input.source_named(domain)?; + if !source.matches(&entry.eid.source) { + return None; + } + let expires = reply + .aim_tokens + .iter() + .find(|token| token.key_name == *key) + .and_then(|token| token.identity_expires) + .map(UnixTime::from_millis); + if expires.is_some_and(|at| at <= input.now()) { + return None; + } + + let uid = entry.eid.uids.first()?; + let mut id = ResolvedId::new(source, uid.id.clone()); + if let Some(atype) = uid.atype { + id = id.with_atype(atype); + } + if let Some(ext) = &uid.ext { + id = id.with_ext(ext.clone()); + } + if let Some(at) = expires { + id = id.with_expires_at(at); + } + Some(id) + }))) + } +} +``` + +What this shows: + +- **Exact paths.** A prefix would also capture `settings` and any future + endpoint. `Captures::with_path` matches one method and path. +- **Revocation outlives consent.** `revoke-consent` arrives after the visitor + has denied consent, so the request's consent gate would block it. A + capture's removals must apply whatever the consent (section 12.5). It is + provider-scoped here: it removes Lockr-written records only, never the EC. + Lockr's plan settles the real scope. +- **Consent in the request body.** Lockr's SDK sends its own consent strings + in the body. The descriptor names them, the coordinator extracts and + decodes only those fields, and capture is gated by the stricter of the + cookie consent and the body consent. The provider never sees the body. +- **Provenance.** Lockr's ID5 EID carries `inserter`, `matcher` and `mm`, + which bidders read. The descriptor declares them once per source, and the + coordinator adds them to every server-side value Lockr wrote. +- **Milliseconds.** `UnixTime::from_millis` converts Lockr's expiry, and a + token already expired at capture time is skipped. +- **Optional expiry.** Lockr gives an expiry only for tokens listed in + `aimTokens`, so it marks no source as expiring. An expiry it does give is + still enforced: expired IDs are never sent (section 7.1). A provider whose + sources differ marks only the ones that always expire, with + `with_expiring_source`. +- **Vendor types.** Lockr's payload is parsed into local, permissive types. + The repo's `openrtb::Eid` denies unknown fields, so it suits TS's own + output, not a vendor's input. +- **The opt-out lasts until Lockr issues again.** Lockr has no GVL ID for a + TC string to re-grant, so its `ps.o` is cleared by a later successful + token capture under consenting cookie and body consent (section 8). +- **Ownership.** Lockr claims `liveramp.com` and `uidapi.com` too. Where native + providers own those, the operator narrows Lockr with `sources` (section + 10.2). + +### 12.3 UID2 (secrets, crypto, private state) + +UID2 exercises resolved secrets, encrypted request envelopes that need +random nonces, a nonce that `response` must check against `request`, a large +private state, and a provider-reported opt-out. The API names follow the +UID2 server-side token endpoints. The envelope crypto is elided. + +```rust +#[derive(Debug, Clone, Deserialize, Validate)] +pub struct Uid2Config { + pub enabled: bool, + /// For example `https://uid2.example.com`. + #[validate(url)] + pub base_url: String, + /// Secret-store key names at rest, resolved values at runtime. Listed in + /// `secret_fields()` (section 10.2), so `build_descriptor` never reads them. + pub api_key: Redacted, + pub client_secret: Redacted, +} + +#[derive(Serialize, Deserialize)] +struct Uid2State { + refresh_token: String, + refresh_response_key: String, + /// Milliseconds since the epoch. + refresh_expires: u64, +} + +#[derive(Serialize, Deserialize)] +struct GenerateCall { + request_nonce: [u8; 8], +} + +#[derive(Deserialize)] +struct Uid2Reply { + status: String, + body: Option, +} + +#[derive(Deserialize)] +struct Uid2Identity { + advertising_token: String, + refresh_token: String, + refresh_response_key: String, + /// Milliseconds since the epoch, like the two below. + identity_expires: u64, + refresh_from: u64, + refresh_expires: u64, +} + +fn join(base: &Url, path: &str) -> Result> { + base.join(path) + .map_err(|_| IdentityProviderError::config("`base_url` cannot take a path")) +} + +// Envelope crypto, elided: AES-256-GCM as UID2's API documentation describes. +// fn encrypt_envelope(secret: &Redacted, iv: &[u8; 12], now: UnixTime, +// nonce: &[u8; 8], payload: &serde_json::Value) -> Result>; +// fn decrypt_generate(secret: &Redacted, body: &str, nonce: &[u8; 8]) +// -> Result>; +// fn decrypt_refresh(key: &str, body: &str) -> Result>; + +impl IdentityProviderConfig for Uid2Config { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(UID2_INTEGRATION_ID.parse()?) + .with_source("uidapi.com".parse()?) + .with_expiring_ids() + .with_resolves( + Resolves::new() + .with_endpoint(&self.base_url) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]) + .with_private_state(), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(UID2_GVL_ID)) + .build() + } +} + +impl IdentityProvider for Uid2Provider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let base = input.endpoint()?; + if input.is_refresh(input.primary_source()) { + let Some(state) = input.state::()? else { + return Ok(None); + }; + if UnixTime::from_millis(state.refresh_expires) <= input.now() { + return Ok(None); + } + let url = join(base, "v2/token/refresh")?; + return Ok(Some( + ProviderRequest::post_text(&url, &state.refresh_token).with_bearer(&self.api_key), + )); + } + + let url = join(base, "v2/token/generate")?; + let request_nonce = input.random_bytes::<8>(); + let envelope = encrypt_envelope( + &self.client_secret, + &input.random_bytes::<12>(), + input.now(), + &request_nonce, + &json!({ "email_hash": input.hashed_email()?.to_base64() }), + )?; + Ok(Some( + ProviderRequest::post_text(&url, &envelope) + .with_bearer(&self.api_key) + .try_with_call_context(&GenerateCall { request_nonce })?, + )) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let body = response.success_text()?; + let reply: Uid2Reply = match response.call_context::()? { + Some(call) => decrypt_generate(&self.client_secret, body, &call.request_nonce)?, + None => { + let state = input + .state::()? + .ok_or_else(|| IdentityProviderError::config("refresh without state"))?; + decrypt_refresh(&state.refresh_response_key, body)? + } + }; + + match reply.status.as_str() { + "optout" => Ok(Resolution::opted_out()), + "success" => { + let identity = reply.body.ok_or_else(IdentityProviderError::malformed)?; + let id = ResolvedId::new(input.primary_source(), identity.advertising_token) + .with_atype(3) + .with_expires_at(UnixTime::from_millis(identity.identity_expires)) + .with_refresh_at(UnixTime::from_millis(identity.refresh_from)); + Ok(Resolution::found([id]).replacing_state(&Uid2State { + refresh_token: identity.refresh_token, + refresh_response_key: identity.refresh_response_key, + refresh_expires: identity.refresh_expires, + })) + } + _ => Err(IdentityProviderError::malformed()), + } + } +} +``` + +What this shows: + +- **Secrets.** `api_key` and `client_secret` are `Redacted` fields + that hold secret-store key names at rest and resolved values at runtime. + Their paths must be added to `TrustedServerAppConfig::secret_fields()` + (`config.rs:136-175`), and `build_descriptor` must not read them, because + deploy validation runs before resolution (section 10.2). +- **Expired tokens refresh.** A `Refresh` runs for an expired advertising + token too, so a visitor who misses the refresh window still recovers while + the refresh token lives. The provider checks `refresh_expires` itself. +- **No dry run.** UID2 keeps private state, so it gets no dry run (section + 5.2), and its `request` always builds the real, encrypted call. +- **Randomness.** The envelope needs a random IV and nonce. A provider has no + safe source of its own in a pure method, so `ResolveInput::random_bytes` + supplies them, and the harness makes them deterministic. +- **Call context.** `response` must check the nonce `request` generated. The + call context carries it, as it carries LiveRamp's call kind. +- **Text bodies.** Both endpoints send and return base64 text, not JSON, so + `ProviderRequest::post_text` and `ProviderResponse::success_text` are + needed. +- **State size.** A UID2 refresh token is several hundred bytes, so the state + cap rises from 512 to 1,024 bytes (sections 4.5, 6.1). +- **Opt-out.** `"optout"` maps to `Resolution::opted_out()`, which is sticky + (section 8). +- The crypto needs an AES-GCM implementation, a new dependency that the UID2 + spec justifies. + +### 12.4 ID5, generic pull, and a provider not in v1 + +- **ID5** has the shape of the section 4.6 example: `ClientIp` and + `UserAgent` needed, `PageUrl` wanted, the `signature` as private state, and + `ext.linkType` per UID. It declares three sources (`id5-sync.com`, the + EUID source its responses name, and `true-link-id5-sync.com`, + `id5IdSystem.js:40-41`, `:130-170`) and uses `also_invalidate` for a + declared source a response no longer returns. +- **Generic pull** is built in (section 10.4). +- **A local provider**, one that needs no outbound call, does not fit v1. An + example is a publisher login ID read from a first-party cookie and + emitted as an EID. It would need a third capability, say `derive`, and an + input that reads a named first-party cookie. Both are additive under + section 4.2, which is the point of the check. This spec does not add them. + +### 12.5 What the examples changed + +Writing the examples found these gaps. Each is now in the API of sections 4 +to 6: + +| Gap | Found by | Change | +| ------------------------------------------------------------ | ----------------- | -------------------------------------------------------------------------------- | +| `response` cannot know what `request` sent | LiveRamp, UID2 | `ProviderRequest::try_with_call_context`, `ProviderResponse::call_context` (4.5) | +| No safe randomness in a pure method | UID2 | `ResolveInput::random_bytes`, deterministic in the harness (4.2, 4.4, 13) | +| Secrets have no stated path | UID2 | Secret fields in `secret_fields()`; `build_descriptor` never reads them (10.2) | +| Text bodies | UID2 | `post_text`, `success_text` (4.5) | +| State cap too small for refresh tokens | UID2 | 1,024 bytes (4.5, 6.1) | +| Expiry in milliseconds | Lockr, UID2 | `UnixTime::from_millis` (4.4) | +| Prefix capture is too broad | Lockr | `Captures::with_path(method, path)` (4.3) | +| Revocation arrives after consent is denied | Lockr | Capture removals bypass the consent gate (4.7) | +| Consent signals in the request body | Lockr | `Captures::with_request_consent`; the stricter consent wins (4.3, 4.7) | +| EID-level `inserter`, `matcher` and `mm` cannot be expressed | Lockr | `ProviderDescriptorBuilder::with_provenance`, applied on output (4.3, 7.1) | +| A source string from a response must match a declared source | Lockr | `SourceDomain::matches` normalizes before comparing (4.3) | +| An oversized UID-level `ext` discarded the whole ID | Lockr (ID5 `pba`) | The `ext` is dropped and the ID kept (4.5) | +| Region tests need country and subdivision | LiveRamp | `Region::country`, `Region::is` (4.4) | +| An expired record stranded refreshable providers | UID2, LiveRamp | `Refresh` covers expired records; the provider decides (4.4, 4.7) | +| The dry run repeated expensive work and could not see state | UID2 | No dry run for providers with private state (5.2) | +| Every example parsed its endpoint twice | all | `Resolves::with_endpoint(&str)`, checked by `build()` (4.3) | +| Capture had two ways to say "nothing" | Lockr | `capture` returns `Resolution`; `no_match` means nothing (4.2) | + +## 13. Testing + +**Provider harness** (`ec/identity/test_support.rs`). `ProviderHarness` +drives a provider the way the coordinator does, so vendor tests stay short: + +```rust +let provider = ExampleIdProvider::new(&example_config()).expect("should build the provider"); +let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_user_agent("ExampleBrowser/1.0"); + +let request = harness + .request() + .expect("should build a request") + .expect("should not skip the request"); +assert_eq!( + request.url().host_str(), + Some("api.exampleid.example.com"), + "should call the declared host" +); + +let resolution = harness + .respond(StatusCode::OK, &json!({ "uid": "example-uid", "signature": "example-signature" })) + .expect("should parse the response"); +assert_eq!(resolution.ids()[0].uid(), "example-uid", "should return the uid"); +``` + +`harness.respond_raw(status, headers, bytes)` covers empty bodies such as a +204, and headers. `harness.request_json::()` decodes the built body. + +The harness runs the coordinator's steps around the provider, so a test +sees what production would: + +- **Kind.** A source with a `with_stored` record is `Refresh` (valid or + expired), any other `Initial`. +- **Eligibility.** It applies the consent policy (eligibility step 1) and + step 4: sources whose kind's needs are missing are left out, and with none + left `request()` returns `Ok(None)` without calling the provider. +- **Provenance.** `ProviderDescriptor::provenance` gives the EID-level + fields; the harness adds nothing of its own. + +The harness API, in the test-support module, gated by +`#[cfg(any(test, feature = "test-support"))]`. A dependent crate never sees +`cfg(test)`, so a provider in its own crate enables `test-support` as a +dev-dependency feature of `trusted-server-core`: + +```rust +impl<'p> ProviderHarness<'p> { + pub fn new(provider: &'p dyn IdentityProvider) -> Self; + /// Replaces every input below at once, for conformance cases. + pub fn with_setup(self, setup: HarnessSetup) -> Self; + pub fn with_client_ip(self, ip: &str) -> Self; + pub fn with_user_agent(self, user_agent: &str) -> Self; + pub fn with_page_url(self, url: &str) -> Self; + pub fn with_region(self, country: &str, subdivision: Option<&str>) -> Self; + pub fn with_hashed_email(self, email: HashedEmail) -> Self; + /// Raw consent strings. Default: GDPR does not apply and no opt-outs. + pub fn with_consent(self, consent: TestConsent) -> Self; + pub fn with_stored(self, stored: StoredId) -> Self; + pub fn with_state(self, state: &impl Serialize) -> Self; + /// The fixed clock. Default: a constant test time. + pub fn with_now(self, now: UnixTime) -> Self; + pub fn with_random_seed(self, seed: u64) -> Self; + pub fn as_dry_run(self) -> Self; + + /// Calls `request` and keeps the built request and its call context. + pub fn request(&mut self) -> Result, Report>; + /// Decodes the last built request's JSON body. + /// + /// # Panics + /// + /// When no request was built or its body is not `T`. + pub fn request_json(&self) -> T; + /// Calls `response` with the kept call context. + pub fn respond( + &mut self, + status: StatusCode, + body: &serde_json::Value, + ) -> Result>; + pub fn respond_raw( + &mut self, + status: StatusCode, + headers: &[(&str, &str)], + body: &[u8], + ) -> Result>; + /// Runs a capture as the coordinator does, consent extraction included. + pub fn capture(&self, fixture: &CaptureFixture) -> Result>; + /// Resolves an expiry or refresh against the fixed clock. + pub fn resolve(&self, when: When) -> UnixTime; + /// The replacement state in a result, decoded, for assertions. + pub fn state_of(&self, resolution: &Resolution) -> Option; + + /// Runs only the cases passed in, never the harness's own inputs. A + /// provider declared `with_private_state()` fails here with a pointer to + /// `check_conformance_with_state`. + pub fn check_conformance(&self, cases: &ConformanceCases); + pub fn check_conformance_with_state(&self, cases: &ConformanceCases); +} + +/// Inputs for one harness run. The harness's own `with_*` methods delegate +/// to an inner `HarnessSetup`. +#[derive(Debug, Clone, Default)] +pub struct HarnessSetup { /* private */ } + +impl HarnessSetup { + pub fn with_client_ip(self, ip: &str) -> Self; + pub fn with_user_agent(self, user_agent: &str) -> Self; + pub fn with_page_url(self, url: &str) -> Self; + pub fn with_region(self, country: &str, subdivision: Option<&str>) -> Self; + pub fn with_hashed_email(self, email: HashedEmail) -> Self; + pub fn with_consent(self, consent: TestConsent) -> Self; + pub fn with_stored(self, stored: StoredId) -> Self; + pub fn with_state(self, state: &impl Serialize) -> Self; + pub fn with_now(self, now: UnixTime) -> Self; + pub fn with_random_seed(self, seed: u64) -> Self; +} + +/// Raw TCF, GPP and US Privacy strings, as a request would carry them. +#[derive(Debug, Clone, Default)] +pub struct TestConsent { /* private */ } + +impl TestConsent { + pub fn with_gdpr_applies(self, applies: bool) -> Self; + pub fn with_tc_string(self, tc: &str) -> Self; + pub fn with_gpp_string(self, gpp: &str, section_ids: &[u16]) -> Self; + pub fn with_us_privacy(self, us_privacy: &str) -> Self; + pub fn with_gpc(self) -> Self; + pub fn with_dnt(self) -> Self; +} + +/// One proxied call: the request a capture rule matches, and its response. +#[derive(Debug, Clone)] +pub struct CaptureFixture { /* private */ } + +impl CaptureFixture { + pub fn new(method: Method, path: &str) -> Self; + pub fn with_request_body(self, body: &serde_json::Value) -> Self; + pub fn with_response(self, status: StatusCode, body: &serde_json::Value) -> Self; +} + +#[derive(Debug, Clone, Default)] +pub struct ConformanceCases { /* private */ } + +impl ConformanceCases { + /// A setup to run `request` under, and the response to answer it with. + pub fn with_resolve_case( + self, + setup: HarnessSetup, + status: StatusCode, + response: &serde_json::Value, + ) -> Self; + /// For text, binary or empty replies, such as UID2's base64 body or a + /// LiveRamp 204. + pub fn with_resolve_case_raw( + self, + setup: HarnessSetup, + status: StatusCode, + headers: &[(&str, &str)], + body: &[u8], + ) -> Self; + pub fn with_capture_case(self, fixture: CaptureFixture) -> Self; +} + +/// Test-only constructors, behind the same gate. +impl StoredId { + pub fn for_test(source: &SourceDomain, uid: &str) -> Self; + pub fn written_by(self, writer: &IdentityProviderId) -> Self; + pub fn expiring_at(self, at: UnixTime) -> Self; +} + +impl HashedEmail { + /// From the SHA-256 of a normalized address, as 64 hex digits. + pub fn for_test(sha256_hex: &str) -> Self; +} + +pub fn check_descriptor_ignores_secrets(placeholder: &C, resolved: &C); +``` + +`harness.check_conformance(&cases)` runs the checks every provider must +pass. `ConformanceCases` lists the setups to try (inputs, stored IDs, state +and a test-only `HashedEmail`) and one vendor fixture per capture rule, so the +harness never guesses which setup makes a provider call out: + +- The harness builds inputs that hold only what the descriptor declares, so + reading an undeclared input fails the test with `UndeclaredInput`. +- A provider that declares `resolves` returns `Some` from `request` for at + least one listed setup, and its `response` on a vendor fixture does not + return the `not_implemented` marker. +- A provider that declares `captures` returns a `Resolution` other than + `no_match` for at least one listed fixture, and never the + `not_implemented` marker. +- No undeclared input or state read is recorded. +- A provider without `with_private_state()` never returns state. +- Requests go only to declared hosts, over HTTPS, with no `Cookie` header and + with valid header names and values. +- Returned IDs are only for declared sources, at most one per source. +- `ResolvedId::expiry()` and `refresh()` report what the provider set, and a + fixture ID of a source marked expiring carries an expiry. +- `capture` never returns a state change. +- For every case with state, the harness also injects a stored state that + is a bare JSON string, which no struct decodes, and checks that the + provider neither panics nor treats it as valid state. A + `#[serde(default)]` state type that decodes anything fails here. That the + coordinator then clears it is a coordinator test. +- During the dry run, `is_dry_run()` is true and the built request is + discarded unsent. `as_dry_run()` fails the test for a provider declared + `with_private_state()`, which never gets one. + +`harness.check_conformance_with_state::(&cases)` also checks that the +state type round-trips through `replacing_state` and `input.state()`. +`check_descriptor_ignores_secrets::(&placeholder, &resolved)` builds the +descriptor from a config with placeholder secret names and from one with +resolved secrets, and fails if the two differ (section 10.2). Vendor tests add a +fixture response for each outcome the vendor can return, and assert it +through the `Resolution` getters. + +The harness also: + +- fixes `random_bytes` with `with_random_seed`, so encrypted requests are + reproducible +- carries the built request's call context into `respond`, as the + coordinator does +- takes capture fixtures with a request body, and checks that only the + declared consent pointers are read from it and that a stricter body + consent blocks positive IDs but not removals +- applies a provider's provenance, so a test can assert the EID-level fields + bidders will see + +The examples in section 12 are the first harness cases: LiveRamp's initial +and refresh 204s, Lockr's page-view and revoke-consent, and UID2's success, +refresh and opt-out. + +**Coordinator.** + +- An eligibility matrix: consent × ownership × per-source kind × due state × + inputs × mode × writer or stager × new or returning EC. +- The merged eligibility view: a cookie-only `no_match` or result stops a + re-call on the next `/auction`; KV's `o` blocks dispatch even when the + cookie lacks it. +- `sources` narrowing, owner takeover of a browser-seeded record, sticky + `ps.o` surviving a jurisdiction change and cleared only by a newer grant, + and `ps.iv` stopping a re-seed. +- Hold budget: a fake HTTP client with controlled latency shows the auction + starts at the budget, an answer is saved with `k: 0`, and a cut-off call + sets `hold_retry_sec` without counting a failure. +- A dry-run `None` sets no pending marker. A `None` or `InputUnavailable` + waits `refresh_after_sec`. +- Pending markers block re-dispatch, hide that provider's cookie entries + until a read, are answered by each of the three rules (including a + removal), and are dropped past their deadline. +- A navigation answered with a status other than 2xx or 304 dispatches no + after-send provider and keeps hold-mode answers in its cookie. +- An `UndeclaredInput` is logged as `Config` and backs off. +- Only transport failures, timeouts, 429 and 5xx feed the breaker. 4xx and + `Malformed` from many visitors never open it. +- The burst cap is checked after every other eligibility step. +- Passive capture runs from `handle_proxy` only for providers with a capture + rule matching the request, decodes gzip and brotli bodies under the caps, + reads only the declared consent pointers from a request body, and never + touches `n`, `f` or the breaker. +- A Lockr `revoke-consent` captured on the proxy removes every Lockr-written + record, owned or unowned, including sources the cookie holds only as + stubs, and survives a concurrent `/auction` rewriting `ts-eids-s`. +- Providers declared `with_private_state()` never run on stagers. A due + stateful provider gets a marker without a dry run, phase 4 rebuilds its + request with the live state, and a writer navigation without an auction + runs it after send. +- A provider's `ResolveKind` is `Refresh` for a stored record whether it is + still valid or expired, and an expired record counts as absent for + `FillMissing`. + +**Consent.** + +- The GPP sharing and targeted-advertising opt-outs, GPP section 6, and the + US National GPC and known-child fields decode from each section that + defines them. Each one blocks a strict policy. +- A raw consent string that does not decode blocks a strict policy. +- A targeted-advertising opt-out without a sale opt-out produces no + server-side IDs in `user.eids`, no flush of that provider's entries, and no + IDs written into the cookie. + +**One-write invariant.** A new test store counts writes by mode (Add, +`IfGenerationMatch`, Overwrite) and is shared through a test-support feature +so adapter tests can use it. `KvIdentityGraph::counting` counts reads only +(`ec/kv.rs:1596-1605`, `ec/kv_backend.rs:205-221`). + +| Scenario | KV writes | +| ------------------------------------------------------------------ | ---------------------------------------------------- | +| New visitor, navigation | 1 (create, before send) | +| New visitor whose first ID collides | 1 (create with a re-minted ID) | +| Returning visitor, changes pending | 1 | +| Returning visitor, nothing changed | 0 | +| 304 revalidation with changes pending | 1 | +| Returning EC, navigation answered with 3xx, 4xx or 5xx | 0 | +| New visitor, prefetch or prerender navigation | 1 (create, before send) | +| Returning visitor, prefetch or prerender navigation | 0 | +| New visitor, client without `Sec-Fetch-Dest` | 1 (create, before send) | +| Returning visitor, iframe HTML or no `Sec-Fetch-Dest` | 0 | +| Integration-proxy navigation that mints an EC | 1 (create) | +| `/auction`, `/_ts/page-bids`, `/identify`, Lockr API proxy, assets | 0 | +| Consent withdrawal, on any request | CAS tombstone (up to 5 attempts) plus marker, per ID | +| CAS conflict or per-key limit on the flush | 1 attempt, not retried | + +**Withdrawal and removal.** + +- A withdrawal on `/auction` after the navigation's send leaves a tombstone, + and the navigation's phase 4 drops its mutation and calls no provider. +- After `opted_out` or `invalidate`, the removed ID stops reaching bidders + from the cookie by the next request that reads KV, and within + `kv_revalidate_sec` at most. +- A KV read that finds a tombstone expires `ts-eids-s`. +- A deletion tombstone makes every browser drop its cookie copy within + `kv_revalidate_sec`. + +**Failed flush.** After a refused write, staged entries and hold-mode answers +are still `k: 0` in the cookie, and the next navigation writes them. + +**Reads.** A returning visitor with a valid, recently confirmed `ts-eids-s`, +nothing due, staged or pending, and no registered source in `ts-eids` pays +zero KV reads on the navigation. `/auction` and `/_ts/page-bids` pay their +existing one read and no more. + +**No lost EIDs (G10).** + +- A registered source present only in the `/auction` body (beyond the + cookie trim) is saved as a KV gap and written by the next navigation. +- A gap too large for the cookie is saved again by the next `/auction` and + eventually written. +- Every source today's `/auction` ingest would write reaches KV. + +**Orphans and collisions (section 5.5).** + +- A row deleted out of band is found within `kv_revalidate_sec` and recovered + by rotation before send. +- A stager that sees the row missing sets `m`, and the next writer recovers. +- A collision on the first create re-mints before any cookie is set. + +**Storage.** + +- `KvPartnerId` and `ps` round-trip, with and without the new fields. +- The rollback rule: a stripped or browser-seeded record of an expiring + owned source is never sent from the server. +- Revision-checked `OwnerRemove` and `OwnerWrite` lose to a newer write. + Re-applying changes on a later writer keeps the other tab's change. +- `ts-eids-s`: + - sealing and opening with the EC ID as associated data; a cookie sealed + for another EC, or tampered with, fails to open and is ignored without + logging its value + - rejection of a duplicate `__Host-ts-eids-s` + - a `cf` that does not match the identity config discards the cookie's + positive entries and keeps its removals + - a stale `kc` forcing a read + - the size cap's drop order, stubs forcing a read, and removals and + markers never dropped + - a staged refresh that `OwnerWrite` would apply survives the merge, and + one it would not is dropped + - the flush checks (freshness, writer still enabled and owning, consent + re-gate, opt-out, revision) + - a replayed old cookie cannot re-insert an opted-out ID, and is overridden + at its next revalidation +- `ts-eids-r`: + - an entry written by a removal-producing response survives a concurrent + `/auction` that rewrites `ts-eids-s` from an older copy + - two removal-producing responses in flight together both keep their + batches, each in its own cookie + - a fifth batch merges into the lowest openable ``; a partly applied + cookie is kept whole; an over-cap batch coalesces, drops `iv`, then + splits; a merge that does not fit rewrites all four per provider + - a Lockr revoke followed by a re-consent capture, before and across + navigations, ends with Lockr's new IDs in KV and the old opt-out never + re-applied (`ps.oc` in KV), including after a failed flush + - a removal names its source by domain, so a config change that reorders + sources removes the right one + - a cookie that fails to open is deleted by the writer and never counted + - a removal without `pv` skips a record its writer re-acquired after `sa` + - it is honoured on the unusable-consent path, where `ts-eids-s` is + expired, until the next writer applies it + - the 512-byte sealed cap per cookie, the one-year `Max-Age`, and that + stagers never delete one +- The admin lookup redacts `ps.st`. + +**Outbound requests.** Requests to Prebid Server, in every +consent-forwarding mode, carry none of the `TS_IDENTITY_COOKIE_NAMES` +cookies, keep every other cookie, and drop the header when nothing is left. +Requests to the publisher origin carry none of `ts-eids`, +`__Host-ts-eids-s` or any `__Host-ts-eids-r-` cookie, keep `ts-ec`, +`ts-ec-pull-complete` and every other cookie, and drop the header when +nothing is left. + +**Merge.** + +- Owned versus unowned precedence. +- A browser-seeded record never beats the browser's current value. +- Expired IDs are never sent. +- `x-ts-eids` matches what was sent. +- `/_ts/api/v1/identify` omits expired IDs and IDs the owner's policy blocks. + +**Batch sync.** An update to an owned source applies only when absent. + +**Migration parity.** Today's `ec/pull_sync.rs` tests are ported against +`GenericPullProvider`. Each behaviour change in section 10.4 gets its own +test. + +**Config.** Each validation error in section 10.2, `build_descriptor` +running in `ts config validate` without secrets, and acceptance of a config +with no `[ec.identity]` at all. + +**Adapters.** Phase 4 runs after send on Fastly. Axum, Cloudflare and Spin +behave as before apart from the Prebid Server and origin cookie strips. The parity test crate +passes +(`cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`). + +## 14. Rollout + +Three steps, each shippable on its own: + +0. **Strips first.** One release before any new cookie is set, ship only + the cookie strips (origin and Prebid Server, by name and by the + `__Host-ts-eids-r-` prefix) and the template cache's identity cookie + list. Stripping `ts-eids` from origin requests is a behaviour change for + any origin that read it. A rollback after step 1 then lands on code that + still keeps the new one-year cookies away from the origin and its header + limit. This release is the rollback floor. +1. **One write, no new vendors.** This step includes: + - the writer-request rule and the `IdentityMutation` single flush, with + no retry; the new visitor's create stays before routing, as today + - `/identify` and `/_ts/page-bids` stopping their writes + - `ts-eids-s`: the host-only cookie sealed with AES-256-GCM (a new + `aes-gcm` dependency, which the UID2 provider later reuses), `kc` + revalidation, KV-gap staging on `/auction`, pending and `m` markers, + the size cap + - the `ts-eids-r` removal cookies + - a metric for the inbound `Cookie` header size (section 6.2) + - metadata-preserving `KvPartnerId` and `ps`, with `Default` and + constructors + - `GenericPullProvider` replacing pull sync, which brings the trait, the + descriptor, the harness and the write-counting test store + - the shared servability check for `/identify` and the bidstream + + It changes behaviour on its own: fewer KV writes, `/auction`, + `/_ts/page-bids` and `/identify` stop writing, and new visitors reach + pull partners from their second page load. Write and read counters per + page load show the effect. + +2. **Providers.** `.with_identity_provider` and + `.try_with_identity_provider` for vendors, ownership and its + validation, hold mode, passive capture, provider state, the rate-limiter + `record` and `count` methods and the circuit breaker, and the consent + decoder extensions of section 9.1. Vendor modules follow in their own + specs. + +After providers write metadata, do not roll back past step 1. Step 1's reader +preserves the new fields. A rollback past it strips them: + +- `w`, `exp`, `ra` and `at`. The rollback rule of section 6.1 turns these into + re-fetches, not stale IDs, once rolled forward again. +- `ps.o`. Opted-out providers would be called again after rolling forward, + except where the cookie still holds the sticky `o`. +- `ps.st`. Providers with state start over. +- `ps.iv`. Rejected values could be re-seeded once. + +While rolled back, `exp` is not enforced at all. Rolling back past step 1 +also restores today's `/auction` ingest. Step 0's code ignores `ts-eids-s` +and the `ts-eids-r` cookies but still strips them, and the browser keeps +sending what it staged. + +## 15. Open questions + +1. **How long single-page apps wait.** Some sites are single-page apps: + after the first page loads, they change pages with JavaScript instead of + loading a new document from TS. TS hears about those page changes only + through `/_ts/page-bids`, which never writes KV (section 5.1). So on those + sites, entries saved in `ts-eids-s` wait until the visitor's next full + page load. After launch, measure how long saved entries wait before they + are written, and revisit if the wait is long. +2. **Tombstone failures.** Today's withdrawal tombstone is a CAS write with + up to five attempts (`ec/kv.rs:1349-1430`). It can fail on CAS exhaustion, + on Fastly's per-key limit when another write to the same key landed + within about a second, on a read failure, or on a KV error. Should a + failed tombstone be retried after send, with a delay? + +## 16. Decisions log + +Decisions made while writing this spec, with the user: + +| Decision | Choice | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Scope | Framework and trait. Vendors get their own specs. | +| HEM input | Typed slot, always empty in v1. Intake is a later spec. | +| Browser vs server precedence | Configured per source through `owner`. The owner's valid value wins. | +| Cookie for server IDs | Separate `ts-eids-s`, server-owned and HttpOnly, sealed with AES-256-GCM so the origin and its logs never see server-only IDs. | +| Structure | Trait plus coordinator, with an optional passive-capture capability. | +| Trait shape | One required method, `descriptor`. Capabilities are default methods. Synchronous. Additive by construction. | +| Inputs | Declared in the descriptor and enforced: a provider can read only what it declared. | +| Page load for the one-write rule | A navigation plus every request it causes. Only the navigation writes. Strict. | +| New-visitor write | One create before routing, as today, carrying browser EIDs. No provider runs on the first navigation. Changed in rounds 3 and 4. | +| New-visitor recovery | Today's rotation. The cookie-proof recreation of revision 3 is removed with the deferred create. | +| Failed write | Not retried. Changed from one merge-retry once Fastly's per-key limit was confirmed. | +| Browser EIDs beyond the cookie | `/auction` saves the KV gap in `ts-eids-s`. The next navigation writes it. | +| Delay on body-less routes | Accepted: a browser ID first seen in an `/auction` body reaches KV one page view later. | +| SPA views (`/_ts/page-bids`) | Stagers. They never write KV. | +| Hold budget | `hold_budget_ms` defaults to 100 ms, reduced from 300 ms (section 4.7 gives the reasons). | +| Outbound cookies | TS's identity cookies are stripped from Prebid Server requests. The origin keeps `ts-ec` but loses `ts-eids`, `ts-eids-s` and `ts-eids-r`, to stay under header limits. | +| Deletion | A tombstone with the 24-hour `TOMBSTONE_TTL` replaces the hard delete. No extra writes. | +| GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | +| `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | +| Consent withdrawal | Synchronous tombstone on any request, the only G4 exception. Proposed in review; not yet confirmed. | + +Round 7 proposed these, and they are applied in revision 8 pending the +user's review of the spec: + +| Decision | Choice | +| ----------------------- | ----------------------------------------------------------------------------------------------------------- | +| Removals and opt-outs | One `__Host-ts-eids-r-` cookie per removal batch, up to four, so no concurrent response can lose one | +| Provider opt-out | Removes every record the provider wrote, with no revision check. Opted-out writers are never served | +| Expired records | `Refresh` covers a stored record, valid or expired. An expired record counts as absent for `FillMissing` | +| Stateful hold providers | Run only on the writer: before the auction when it runs one, after send when it does not | +| `capture` result | Returns `Resolution`; `no_match` means nothing was captured | +| Registration | `try_with_identity_provider` converts a constructor's error once; `build()` stays infallible | +| KV read cost | An auction navigation reads once, as today. Phase 4 reuses that snapshot unless an after-send provider runs | +| Provider count | At most 32 enabled providers, so a row's `ps` never fills up | + +## 17. Review log + +**Round 1** (2026-10-21) found 5 blocking issues, 16 significant gaps and 17 +accuracy problems. Revision 2 resolved them. Its findings covered the +deferred create's rotation risk, staged refreshes, pending-marker liveness, +provider state, the writer definition, hold-mode latency, concurrent cookie +writes, HTML cacheability, multi-source providers, inputs, `ext` level, +owner takeover, `/identify` and batch sync, the size cap, flush checks, rate +limits, the breaker, tombstones, `/auction` ingest and deploy-time +validation. + +**Round 2** (2026-10-21) answered the open questions and reworked the trait +for ergonomics and additivity (revision 3). + +**Round 3** (2026-10-21) ran four independent reviews: Rust API, write +protocol, code references, and privacy and consent. They found 7 blocking +issues and about 30 significant ones. Revision 4 resolves them as follows: + +| Finding | Resolution | +| ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------ | +| Deferred create loses withdrawals, collision checks and orphan detection (B1, B2, B3, C1, D1) | Create before send, sections 5.2, 5.5 | +| Cookie IDs outlive KV removals (D2, B5) | `kc` revalidation, merge rules, G12, 6.2, 6.3 | +| `ts-eids-s` reaches Prebid Server (D3, C11) | Strip from Prebid Server; origin unchanged (revision 5), 6.2 | +| Stagers cannot save removals, state or `ps` (B4) | Removal entries, staged `ps`, state writer-only, 5.3 | +| Eligibility on stagers ignored the cookie (B6) | Merged eligibility view, 4.7 | +| Invalidate and re-seed loop (B7) | `ps.iv`, 7.2 | +| Writer could not clear markers or flip `k` (B8, C16) | Phase 3 cookie rule, 5.2 | +| Late hold results impossible; overruns tripped the breaker (B9, C2) | Cut-off calls, `hold_retry_sec`, 4.7, 8 | +| `request` returning `None` left providers due forever (A1) | `None` waits `refresh_after_sec`, dry run, 4.7, 8 | +| Capture backoff on unrecognised bodies (A2) | Capture never backs off, 4.7, 8 | +| `MissingInput` conflated a bug and a skip; state could drop IDs (A3) | `UndeclaredInput`, `InputUnavailable`, `with_state(&T)`, 4.4, 4.5 | +| EC ID and capture prefix unenforced (A4) | Crate-private EC ID, prefix check, 4.3 | +| Multi-source invalidation (A5) | `also_invalidate`, 4.5 | +| Registration snippet and config trait (A6) | `change_context`, `IdentityProviderConfig: IntegrationConfig`, 4.6, 10.2 | +| `#[non_exhaustive]` ineffective in-crate (A7) | Additivity by construction, opt-in kinds, 4.2 | +| Getters, setters and naming (A8, A13) | `with_*` setters, getters, renames, 4.3 to 4.5 | +| Second `ProviderId` (A9) | `IdentityProviderId`, 4.3 | +| Error type and header validation (A10, A11) | 4.5 | +| Lint and doc conventions; silent mis-wiring (A12, A14) | Conventions note, conformance checks, 4.2, 13 | +| Phase 4 lacked UA, page URL, DNT and a usable IP (C3) | Captured in phase 1, 4.4 | +| `/identify` bypassed the expiry filter (C4, D6) | Shared servability check, 7.1 | +| Missing writes; withdrawal path misdescribed (C5, B10, D13) | P3 table, 5.1, 13, 15 | +| Writer decision placement and prefetch visibility (C6) | 5.1 | +| Rate-limiter API and breaker (C7, B11, D12) | `record`/`count`, cap last, transport-only breaker, 4.7 | +| Capture buffering, encoding and cookie site (C8, C18) | 4.7, 5.3 | +| Unowned completeness signal (C9) | Pull-complete marker kept, 6.3, 10.4 | +| Domain-wide cookie and MAC binding (C10, D5) | `__Host-`, delimited MAC, `cf`, 6.2 | +| Clock failure, schema literals, write-counting tests (C12, C13, C14) | 4.4, 6.1, 13 | +| Policies gated only dispatch (D4) | Re-applied at flush, cookie, merge, `/identify`, 9.1 | +| Opt-out fingerprint lapsed (D8) | Sticky `o`, 8 | +| Strict policy failed open (D9) | Fail closed, more decoding, 9.1 | +| Minimization leaks: geo, jurisdiction, page URL, `x-ts-eids` (D10) | `Region`, `ProviderConsent`, sanitized URL, documented header, 4.4, 7.1 | +| Logging holes (D11) | 9.3 | +| `Sec-Fetch-Dest` fallback, undefined transitions, rollback losses (B12, B13, B14, B15, C15, C17) | 3.4, 5.1, 5.2, 6.2, 6.3, 14 | +| Reference corrections (C19) | Throughout | + +**Round 4** (2026-10-21) re-reviewed revision 4 with two independent +reviewers. They found 6 blocking issues, all with contained fixes, and +confirmed most round-3 fixes. Revision 5 resolves them: + +| Finding | Resolution | +| --------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| A minted ID was used in routing before its deferred create, on publisher and proxy routes | Create stays before routing; no provider on the first navigation, 4.7, 5.2 | +| A config change discarded staged removals and opt-outs | 5.6 | +| Flush checks could veto removals | Removals exempt, 5.6 | +| Orphan rotation re-bound positive IDs, undoing deletion | Missing row expires the cookie, 5.5, 5.6 | +| Markers could not be answered when phase 4 wrote nothing | Phase 4 always answers; unowned dry runs need KV, 5.6 | +| Markers set on non-2xx responses | 5.2 | +| Unowned hold answers became `OwnerWrite` | 5.3 | +| Writer never flipped its flushed entries to `k: 1` | Reconciling read, 5.6 | +| EC minting on non-writer navigations undefined | 5.6 | +| Withdrawal windows and `kc` stamping | 5.6 | +| G4 named withdrawal the only exception | G4 scoped to page loads | +| `cluster_size` had no trigger; size cap could fail | 5.6 | +| `pv` hash compared with a raw revision | `RevisionHash`, 5.4 | +| Capture plumbing, Prebid strip paths, default `capture`, `ProviderConsent`, `type Err`, `with_private_state()` naming | 4.2, 4.3, 5.6, 6.2 | + +**Round 5** (2026-10-21), a final review focused on trait ergonomics, found +3 blocking and 6 significant issues, all fixed in revision 6: + +- Phase 4 re-reads before dispatching whenever after-send providers are + marked, so a withdrawal after send is always seen (5.6). +- Conformance checks a provider against at least one working harness setup + and a fixture, so UID2- and LiveRamp-style providers pass (13). +- The capture-prefix check moves to `IntegrationRegistration::build()` (4.3). +- Undeclared input and state reads are recorded and caught even through + `.ok()` (4.4). +- Every capability default, including `request`, returns a + `not_implemented` marker (4.2, 4.5). +- State is enforced: `with_state` and `state()` need `with_private_state()` + (4.4, 4.5). +- Less boilerplate: `Resolves::with_endpoint`, `primary_endpoint`, + `ResolveInput::primary_source`; the example parses its endpoint once (4.6). +- Consistent names: `with_private_state`, `with_expiring_ids`, + `also_invalidate`; `found([])` is `no_match` (4.3, 4.5). +- Stale references, outbound-cookie wording, capture state, the additivity + claim for consent, mixed-kind inputs and the step 8 wording are corrected. + +**Round 6** (2026-10-21) was an example-driven review. LiveRamp ATS, Lockr +and UID2 were written as full providers against the API (section 12), and +everything they could not express was added. Section 12.5 lists the gaps: + +- a call context from `request` to `response` +- deterministic randomness +- a stated path for secrets +- text bodies +- a larger state cap +- millisecond times +- exact capture paths +- capture removals that bypass the consent gate +- consent from the request body +- EID-level provenance +- source matching +- keeping an ID whose `ext` is oversized +- region tests + +**Round 7** (2026-10-21) ran two final reviews in parallel. One covered the +API, the examples and feasibility against the code. The other covered the +protocol, privacy, consistency and efficiency. Together they found 2 +blocking, 13 significant and 11 minor issues, three of them found by both. +All are fixed in revision 8: + +- A provider opt-out removes every record that provider wrote, with no + revision check, and servability excludes records whose writer is opted + out. `OwnerRemove` is never skipped because of `ps.o` (5.4, 7.1). +- Capture removals and opt-outs go into a new append-only cookie, + `ts-eids-r`, which a concurrent `/auction` cannot overwrite. It is also + written on the unusable-consent path (5.3, 6.2). +- A due stateful provider, or one with an unowned or stubbed candidate + source, triggers the phase 1 KV read, so its dry run sees KV. A dry run + without loaded state never sets `ps.n` (5.2, 6.3). +- `Refresh` covers expired records, and an expired record counts as absent + for `FillMissing`. An expiring ID without an expiry keeps the stored `exp` + when its UID is unchanged, and is discarded otherwise (4.4, 4.5, 5.4). +- `ps` is pruned on every mutation, more than 32 providers is a config + error, and a mutation that fails validation falls back to its removals + (5.4, 6.1, 10.2). +- `ts-eids-s` is sealed with AES-256-GCM, compactly encoded and capped at + 2,048 bytes. Rollout adds a `Cookie` header size metric and an origin + header-limit checklist item (6.2, 14). +- A stateful hold-mode provider runs after send on writer navigations + without an auction (4.7, 10.1). +- Opt-out clearing: a mirrored `o` yields to KV, GPP clears by section + values, a capture-only provider clears on a later positive capture, and a + withdrawal resets every opt-out (8). +- Efficiency: the read-cost table is corrected, phase 4 reuses the phase 1 + snapshot, cluster evaluation runs only for `/identify` and after send, the + cookie is rewritten only when it changed, and `is_dry_run`, cached + `state()`, embedded-JSON `st` and one cookie decode per request remove + repeated work (4.2, 4.4, 5.2, 5.6, 6.1). +- Request-body consent is buffered in `handle_proxy` with + `collect_body_bounded` and merged by `consent::stricter_of` (4.7, 9.3). +- The capture-path check runs in `IntegrationRegistry::with_plan`, so + `build()` stays infallible (4.3, 10.3). +- API: `Resolves::with_endpoint(&str)`, an `Access` enum in + `UndeclaredInput`, a fallible `try_with_call_context` and `call_context`, + `expiry()` and `refresh()` getters, `StateChange`, `capture` returning + `Resolution`, `with_retry_in`, `replacing_state`, `CaptureInput` + accessors, and full listings for `Region`, `HashedEmail`, `UnixTime`, + `SourceDomain::matches` and `try_with_identity_provider` (4.2 to 4.6). +- Harness: `check_conformance(&ConformanceCases)` and + `check_descriptor_ignores_secrets` (13). +- Lockr parses into permissive local types and skips expired tokens (12.2). +- Minimization claims and stale wording are corrected (9.2, and + throughout). + +After round 7, at the user's request, `ts-eids`, `ts-eids-s` and `ts-eids-r` +are also stripped from publisher origin requests, which keep `ts-ec`. This +replaces the origin header-limit checklist item (6.2, 14). + +**Round 8** (2026-10-21) was a last overall review: two parallel reviews, +one on the protocol and design and one on the API, examples and +efficiency, plus an outside review. They found 1 blocking, 9 significant +and 22 minor issues, several found more than once. All are fixed in +revision 9: + +- Removal batches no longer share a cookie. Each removal-producing + response sets its own `__Host-ts-eids-r-` (up to four), stagers never + delete one, and only the writer deletes those its phase 1 read shows + applied. The cookies last a year, so a removal never expires before a + writer applies it (6.2, 6.3). +- Orphan rotation drops the rest of its mutation and skips phase 4, and + phase 4 applies the tombstone, missing and failed rules to a reused + snapshot too (5.2). +- A failed flush drops a stateful provider's staged IDs with its state, + and the loss of an after-send removal is bounded by the marker deadline + (5.2). +- Pull sync no longer reads KV on every navigation: the pull-complete + marker and a cookie-only `ps.n` stop the trigger (6.3). +- `cluster_size` is no longer driven by a `/identify` cookie flag. A writer + that is already writing adds it after send (5.6). +- A missing row has one outcome, a minimal `m: 1` cookie, and saves no KV + gap. The KV gap also fills expired sources (5.3, 5.6, 6.3). +- Lockr's opt-out can be cleared: a positive capture stages `ps.oc`, which + the writer applies before the IDs (5.4, 8). +- Expiry is declared per source (`with_expiring_source`, `is_expiring`). + Lockr marks none, because its expiry is optional (4.3, 12.2). +- `state()` returns `Result>` with a new `InvalidState` error, and + `capture` never changes state (4.4, 4.5, 4.7). +- The harness has a full listing, including `HarnessSetup`, `TestConsent`, + `CaptureFixture`, `ConformanceCases` and test-only constructors (13). +- Stateful providers skip the dry run, saving a phase 1 read per refresh. + Cookies open lazily, only on identity routes. Browser KV-gap entries are + dropped last at the size cap (5.2, 6.2). +- Smaller fixes: provenance getters, descriptor derives, the LiveRamp + refresh pacing, Lockr narrowing, the G10 and G12 bounds, and stale + wording. + +**Round 9** (2026-10-21) ran two parallel reviews, protocol and design, and +API, examples and harness, plus an outside review of developer experience. +They found 2 blocking, 11 significant and 16 minor issues. All are fixed in +revision 10: + +- A re-consent clear is stored in KV as `ps.oc`, so an old opt-out batch is + never re-applied, and "applied" is defined for every removal kind (5.4, + 6.1, 6.2). +- `ts-eids-r` names sources by domain, deletes cookies that fail to open, + and has a bounded fallback when four cookies are full (6.2). +- A stateful hold provider's answer stays out of the cookie, so its ID + never flushes without its state (5.2). +- The cookie-only `ps.n` applies to unowned sources and yields to a changed + KV `n` (6.3). Undispatched markers back off by at least 60 seconds (5.6). +- `cluster_size` is evaluated once per EC from any writer read, as today + (5.6). +- A strips-only release ships first and is the rollback floor (14). +- UID2's dead dry-run branch is gone; `InvalidState` always clears state and + has a section 8 row (4.4, 8, 12.3). +- The harness gains a raw resolve case, `HarnessSetup` methods, more + `TestConsent` setters, `state_of`, and stated kind and eligibility rules. + It is gated by `cfg(any(test, feature = "test-support"))`, so provider + crates can use it (13). +- `ProviderRequest::new` and `with_body` cover any method and body; both + header helpers follow one `Cookie` rule; `found` takes at most one ID per + source (4.5). +- Derives are corrected (`When` has no `Display`; `Hash` and `Ord` added), + and stale wording is fixed. From 573e011cc2d53ed3cc3fd8ccf1eefc629e563d41 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 22 Oct 2026 12:46:28 +0530 Subject: [PATCH 02/89] Resolve round 10 review of the identity providers spec Filter browser candidates by the owner's invalidations and opt-outs, check the writer's consent policy for every provider-written ID, apply a stateful provider's IDs and state atomically, keep expired records as refresh anchors while refresh credentials last, replace the removal-cookie merge with a synchronous overflow write, give records a random revision with removal-wins ties, and define the consent /identify checks. Signed-off-by: dhruv8sh --- ...0-server-side-identity-providers-design.md | 320 ++++++++++++++---- 1 file changed, 255 insertions(+), 65 deletions(-) diff --git a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md index 66581d2662..d46bc42af6 100644 --- a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md +++ b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md @@ -1,6 +1,6 @@ # Server-side identity providers -Status: Draft, revision 10. Revised after review rounds 1 to 9. Section 17 +Status: Draft, revision 11. Revised after review rounds 1 to 10. Section 17 maps each finding to its fix. Issue: none filed yet. Related: [#1245](https://github.com/IABTechLab/trusted-server/issues/1245) @@ -154,8 +154,9 @@ can be written, and each extra write is another CAS race between tabs. - **G3.** Operators choose per provider between hold mode and after-send mode. - **G4.** At most one browser-driven KV write per page load, and a write - that fails is not retried. This is strict. The only page-load exception is - a consent withdrawal's tombstone (section 5.1). Writers outside page loads + that fails is not retried. This is strict. The only page-load exceptions + are a consent withdrawal's tombstone (section 5.1) and a removal batch that + the removal cookies cannot hold (section 6.2). Writers outside page loads keep their own writes: batch sync, the admin endpoints and the deferred deletion framework (sections 2.2, 5.5). - **G5.** Provider-collected IDs are stored in both a first-party cookie @@ -951,8 +952,10 @@ impl Resolution { /// Replaces the provider's private state. /// /// The state is stored as embedded JSON of at most 1,024 bytes. A larger - /// value is not stored: the old state stays, the IDs are kept and a - /// state-size warning is logged. From a provider not declared + /// value rejects the result's positive part whole (section 5.4): no ID, + /// state or scheduling change is stored, the old state stays, and the + /// call is handled as `Malformed`. The result's removals still apply. + /// From a provider not declared /// `with_private_state()`, the whole resolution is a `Config` error. /// `capture` never changes state: a state change in a capture result is /// ignored with a warning, and the harness fails the test. @@ -984,6 +987,11 @@ impl ResolvedId { pub fn with_expires_at(self, at: UnixTime) -> Self; pub fn with_refresh_in(self, delay: Duration) -> Self; pub fn with_refresh_at(self, at: UnixTime) -> Self; + /// When this ID stops being refreshable, such as a refresh token's + /// expiry. Only for a provider declared `with_private_state()`; from any + /// other it is ignored with a warning. Until then an expired record is + /// kept as a refresh anchor (section 5.4). + pub fn with_refresh_until(self, at: UnixTime) -> Self; pub fn source(&self) -> &SourceDomain; pub fn uid(&self) -> &str; @@ -991,6 +999,7 @@ impl ResolvedId { pub fn ext(&self) -> Option<&serde_json::Value>; pub fn expiry(&self) -> Option; pub fn refresh(&self) -> Option; + pub fn refresh_until(&self) -> Option; } /// A time as a provider gave it: relative to `now`, or absolute. @@ -1053,8 +1062,8 @@ otherwise, because an owned expiring ID without `exp` is never served runtime; the harness fails a fixture that does. `replacing_state` with a value that does not serialize is handled like an -oversized one: the old state stays, the IDs are kept and a warning is -logged. +oversized one: the positive part of the result is rejected whole, so a new ID +is never stored beside refresh state that does not belong to it. ### 4.6 A complete provider @@ -1280,10 +1289,13 @@ request when all of these hold: `acquired_at + refresh_after_sec` has passed or `acquired_at` is missing. A browser-seeded value whose UID matches `ps.iv` is never due (section 7.2). An expired record is never served (section 7.1), so the - provider either refreshes it or returns `None`. + provider either refreshes it or returns `None`. An expired record kept + as a refresh anchor (section 5.4) is a stored record, so the kind stays + `Refresh` while the provider's refresh credentials last. - **Unowned.** `Initial` only, and only when KV has no unexpired ID for the - source. An expired record counts as absent. Unowned sources are never - refreshed (section 7.2). + source. An expired record, or one whose writer is disabled or no longer + configured, counts as absent. Unowned sources are never refreshed + (section 7.2). - Sources whose kind's needs are missing are left out. If none remain, the provider is skipped and the skip is logged at debug. 5. `ps.n` (next eligible time) has passed, and no sticky `ps.o` opt-out is @@ -1504,8 +1516,9 @@ browsers (`adapter-fastly/src/app.rs:446-452`). Revocation cannot wait for a navigation that may never come. The exception covers the whole of today's withdrawal path: a CAS tombstone with up to five attempts, then its completion-marker write, for each withdrawn EC ID (one or two; -`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). It is the only -page-load exception to G4. The tombstone is not part of any `IdentityMutation`. A +`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). It and the +removal overflow write (section 6.2) are the only page-load exceptions to +G4. The tombstone is not part of any `IdentityMutation`. A writer whose phase 4 re-read finds the tombstone drops its mutation (section 5.4, step 1). @@ -1752,10 +1765,24 @@ only the binding and revision checks (section 5.6). ```rust pub(crate) struct IdentityMutation { partner_ids: Vec, + provider_results: Vec, provider_state: BTreeMap, cluster_size: Option, } +/// The positive part of one stateful provider's result, applied whole or not +/// at all. Its removals, opt-out and `iv` hashes are not part of it: they go +/// into `partner_ids` and `provider_state` and apply on their own. +pub(crate) struct ProviderResult { + provider: IdentityProviderId, + /// `OwnerWrite` and `FillMissing` changes for the sources it returned. + ids: Vec, + /// The state replacement or clear, and the new `n` and `f`. + state: ProviderStateChange, + /// Keyed hash of the `ps.st` the call read. `None`: it read no state. + expected_state: Option, +} + pub(crate) enum PartnerIdChange { /// From browser ingest, a KV-gap entry or an unowned provider. /// Applies only if the source is absent or its record is expired. @@ -1774,11 +1801,13 @@ pub(crate) enum PartnerIdChange { WriterRemove { source: SourceDomain, writer: IdentityProviderId }, } -/// A keyed hash of the `(uid, acquired_at)` pair a request read. Comparing -/// the UID alone is not enough: a refresh can extend expiry without changing -/// the UID. The cookie's `pv` stores the same hash, so staged entries compare -/// against the live record's hash. -pub(crate) struct Revision(RevisionHash); +/// A record's opaque revision: its KV `rv`, 64 random bits from the +/// platform generator, drawn fresh on every write of the record, including a +/// refresh that keeps the UID. Neither the UID nor `acquired_at` can serve: +/// two refreshes of an unchanged UID in one second share both. A legacy +/// record without `rv` uses the keyed hash of its `(uid, at)` pair until its +/// next write. The cookie's `pv` stores the revision a staged entry replaces. +pub(crate) struct Revision(RecordRevision); ``` A new visitor's create is not a mutation: `generate_if_needed` builds the @@ -1790,21 +1819,38 @@ Applying the mutation to a live entry, in order: 2. Apply removals first: every `OwnerRemove` and `WriterRemove`. Applying a new `ps.o` for a provider also removes every live record that provider wrote, owned or unowned, at any revision. Removals are never skipped. - An opt-out applies only when its `at` is after both the live `ps.o.at` - and the live `ps.oc`. Then apply each staged clear `ps.oc` that is later + An opt-out applies when its `at` is after the live `ps.o.at` and at or + after the live `ps.oc`, so an opt-out and a clear in the same second + leave the provider opted out. Then apply each staged clear `ps.oc` that is + strictly later than the provider's opt-out `at`, live or staged, and store it as the live `ps.oc`, so the next step sees the provider as not opted out and a later replay of the old opt-out is stale. A clear that is not later is dropped. 3. Apply the positive changes by their own rules. Every change written by a provider the live `ps.o` marks opted out is skipped, `OwnerWrite` and - `FillMissing` alike, and so is a staged entry whose `sa` is before that - provider's latest opt-out `at`, even after a later clear. A `FillMissing` whose - UID matches the live `ps.iv` is skipped. Changes that no longer apply are - dropped silently: a newer write won. + `FillMissing` alike, and so is a staged entry whose `sa` is at or before + that provider's latest opt-out `at`, even after a later clear. A + `FillMissing` whose UID matches the live `ps.iv` is skipped. Changes that + no longer apply are dropped silently: a newer write won. + Each `ProviderResult` applies atomically. Its IDs, its state change and + its `n` and `f` apply together, and only when every ID change in it + would apply on its own and the live `ps.st` still hashes to + `expected_state`. Otherwise the whole group is dropped. A writer whose + phase 4 re-read finds that another writer committed a newer ID or state + since its call read the row therefore stores neither its ID nor its + state, and no row pairs one call's ID with another call's state. 4. Apply `provider_state` changes (`n`, `f`, `o`, `iv`, `st`) and - `cluster_size`. Drop expired records and the `ps` entries of providers no - longer configured. + `cluster_size`. Drop the `ps` entries of providers no longer configured. + Drop expired records, except **refresh anchors**: an expired record of an + owned source whose owner still holds `ps.st` and whose `ru` has not + passed. An anchor is never served (section 7.1), but it keeps the + owner's kind `Refresh` (section 4.7), so an unrelated mutation cannot + strand refresh credentials that are still valid. It is dropped when its + `ru` passes, when the owner's state is cleared (an `InvalidState` + included) or missing, or when the owner is no longer configured. A + record without `ru` is dropped at expiry, as before. After a drop the + owner sees no record and selects `Initial`. 5. Validate the result with `KvEntry::validate`. If it fails, keep only the removals and opt-outs and validate again, so a full row can always shrink. If that also fails, drop the mutation and log. An invalid row is @@ -1946,6 +1992,7 @@ literals move to them. `PartnerIdUpdate` (`ec/kv.rs:75-90`) gains "id5-sync.com": { "uid": "ID5*example", "w": "id5", + "rv": "9c1d04b7e6a83f2a", "at": 1760000000, "exp": 1767776000, "ra": 1760086400, @@ -1963,15 +2010,17 @@ literals move to them. `PartnerIdUpdate` (`ec/kv.rs:75-90`) gains | Field | Meaning | | ------- | ------------------------------------------------------------------------------------------------------------------- | | `w` | Writer: a provider ID. Absent for browser ingest and legacy rows | +| `rv` | The record's revision: 16 hex digits, drawn at random on every write of the record (section 5.4) | | `at` | Unix seconds when the ID was acquired or last refreshed | | `exp` | Unix seconds after which the ID must not be sent | | `ra` | Unix seconds after which the ID is due for refresh | +| `ru` | Unix seconds after which the ID can no longer be refreshed. Keeps an expired record as a refresh anchor until then | | `atype` | Provider-supplied atype. Absent: use `[[ec.partners]].openrtb_atype` | | `ext` | UID-level ext, at most 256 serialized bytes | | `ps.n` | Unix seconds before which the provider must not be called | | `ps.f` | Consecutive failures, for backoff | | `ps.o` | Sticky opt-out: `{at, fp}`, when it was reported and the consent fingerprint | -| `ps.oc` | When the provider's opt-out was last cleared. An opt-out reported at or before it is stale and never re-applied | +| `ps.oc` | When the provider's opt-out was last cleared. An opt-out reported before it is stale and never re-applied | | `ps.iv` | Keyed hashes of UIDs the owner invalidated, at most 4, so they are not re-seeded | | `ps.st` | The provider's private state, as embedded JSON (a raw value, not an escaped string) of at most 1,024 bytes. KV only | @@ -2004,7 +2053,7 @@ Two server-owned cookies: - **`ts-eids-s`** (`__Host-ts-eids-s`) caches provider-collected IDs for owned sources, holds positive entries stagers saved, and carries pending markers and mirrored `ps`. -- **`ts-eids-r`** is a family of up to four cookies, +- **`ts-eids-r`** is a family of cookies, normally at most four, `__Host-ts-eids-r-`, that hold removals and opt-outs not yet in KV. Each response that produces a removal or opt-out sets a new one, with a fresh random 8-hex-digit ``, holding only its own batch. No response @@ -2108,8 +2157,8 @@ change that reorders sources also invalidates the indexes. | `cf` | Fingerprint of the identity config: providers, sources and owners | | `m` | A request saw this EC's row missing. The next writer reads KV in phase 1 | | `s` | The source, as an index into the configured source list | -| `u`…`x` | Mirror the KV fields `uid`, `w`, `at`, `exp`, `ra`, `atype`, `ext` | -| `pv` | Keyed hash of the revision this entry replaces. Absent: none | +| `u`…`x`, `rv` | Mirror the KV fields `uid`, `w`, `at`, `exp`, `ra`, `atype`, `ext`, `rv`. Staged: no `rv` | +| `pv` | The revision (`rv`) of the record this entry replaces. Absent: none | | `sa` | When the entry was staged | | `b` | `1` for a browser KV-gap entry (`FillMissing`) | | `k` | `1` when the entry is known to be in KV, `0` when staged and not yet written | @@ -2121,8 +2170,10 @@ change that reorders sources also invalidates the indexes. **`ts-eids-r` payload.** Removals as `{s, w, pv?, sa}`, where `s` is the source domain itself, never an index: `ts-eids-r` carries no `cf`, so an index could point at another source after a config change. An entry -without `pv` removes the source's record if `w` wrote it and acquired it -before `sa`. Opt-outs as `o: {: }`, and invalidated UIDs as +without `pv` removes the source's record if `w` wrote it and acquired it at +or before `sa`. A tie goes to the removal, so an acquisition in the same +second never dismisses a revocation; an ID re-acquired in that second is +removed too and resolved again later. Opt-outs as `o: {: }`, and invalidated UIDs as `iv: {: []}`. An entry is **applied**, and is skipped if the mutation meets it again, @@ -2130,9 +2181,10 @@ when the live row shows: - a removal with `pv`: the record is gone or at another revision - a removal without `pv`: the record is gone, written by another writer, or - acquired at or after `sa` + acquired after `sa` - an opt-out: KV's `ps.o.at` is at or after `at`, or KV's `ps.oc` is after - `at`, so a cleared opt-out is never re-applied + `at`, so a cleared opt-out is never re-applied. A clear in the same second + as the opt-out does not count: the opt-out still applies `iv` entries never keep a cookie alive: they are applied with the cookie's other entries and then dropped with it. The writer's phase 1 read deletes @@ -2141,17 +2193,35 @@ cookie that fails to open, for example one sealed under an old key. A cookie that is only partly applied is kept whole, and its applied entries are skipped when re-applied. -Each cookie's sealed value is at most 512 bytes. A batch over that -coalesces a provider's removals into one provider-wide removal for each -source it claims, then drops its `iv` hashes (losing them allows one -re-seed, section 7.2), and then splits into a second cookie. When four -openable `ts-eids-r` cookies already exist, a response merges its batch -into the openable one with the lowest `` instead of adding a fifth. If -the merge does not fit, the response rewrites all four as one -provider-wide removal and opt-out per provider, which at 32 providers fits -the 2 KB of four cookies. Those rewrites are the only remaining race: two -responses rewriting the same cookies at once, while four batches wait for -a writer. +**Size.** Each cookie's sealed value is at most 512 bytes. The `v1.` +prefix, the base64url nonce and its separator take 20 bytes, base64url +turns every 3 bytes into 4, and the tag takes 16, so a cookie holds at most +353 bytes of payload JSON. A removal costs 56 bytes plus its source domain +and provider ID: about four fit with typical names (a 15-byte source and an +8-byte provider), and one with a 253-byte source domain. An `iv` hash costs +19 bytes, and an opt-out about 15 bytes plus the provider ID. + +A batch is never coarsened to fit. It is not merged into another +response's cookie, its removals are not widened, its `iv` hashes are not +dropped, and an invalidation never becomes an opt-out. + +**Overflow.** A response writes its batch to KV before send, instead of +setting a cookie, when the batch does not fit one cookie or the request +already carries four openable `ts-eids-r` cookies. This removal overflow +write is the second page-load exception to G4 (section 5.1). It reads the +row and applies only the batch's removals, opt-outs and `iv` hashes under +the rules of section 5.4, with a CAS and the withdrawal path's five +attempts. When it lands, or the read shows the batch already applied, no +cookie is set. When it fails, the response sets the batch in new cookies of +its own, split across as many 512-byte cookies as it needs, so the batch is +never lost. + +Four is where TS stops relying on cookies, not a hard cap. Concurrent +responses each count from their own inbound snapshot, so five responses +that start with no removal cookie can set five. The count passes four only +while removal batches race or overflow writes fail, and the next writer +navigation deletes every applied cookie. A metric reports the number and +total size of `ts-eids-r` cookies per request (section 14). **Staged `ps` changes** merge at flush: `n` takes the later value, `f` the cookie's, `iv` is the union, and an opt-out from `ts-eids-r` is kept unless @@ -2199,7 +2269,8 @@ source. responses are already private. - **`ts-eids-r`:** any response that produces a removal or opt-out, on any route, including the unusable-consent path (section 5.3), adds a new - cookie for its batch; and the writer navigation's HTML response deletes + cookie for its batch, unless its overflow write landed (above); and the + writer navigation's HTML response deletes the cookies its phase 1 read showed applied. - **A consent withdrawal, or any KV read that finds a tombstone,** sets `Max-Age=0` on `ts-eids-s`. Once the tombstone is confirmed (written by @@ -2330,16 +2401,23 @@ set. It holds IDs from `ts-eids-s` (including staged entries) and KV, with expired entries, stubs and removed entries left out. The server set requires what today's KV path requires: `ec_allowed()`, and a live consenting row whenever the request read KV (`endpoints.rs:446-460`). Each server-side ID -must also pass its owning provider's consent policy for this request, so the +must also pass, for this request, the consent policy of the provider that +wrote it (`w`) and, for an owned source, the owner's policy as well. So the sharing and targeted-advertising opt-outs and GPC stop it even when the -generic gates pass. +generic gates pass, and an unowned source that a strict provider wrote is +not disclosed once that provider's policy fails. A record whose writer is +disabled or no longer configured has no policy left to check, so it is +never served, and it counts as absent for `FillMissing` and for unowned +`Initial`, so the browser or another provider can replace it. A record +without `w` (browser ingest or legacy) needs the owner's policy for an owned +source, and only the generic gates for an unowned one, as today. For each source, the value sent to bidders is: -| Source kind | Browser sent the source | Browser did not send it | -| ----------- | ------------------------------------------- | ----------------------------------------------------------------------------- | -| Owned | the owner's valid value, else the browser's | the owner's valid value, else a browser-seeded value if the IDs do not expire | -| Unowned | the browser's value (today's rule) | the server value, if any (today's rule) | +| Source kind | Browser sent the source | Browser did not send it | +| ----------- | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Owned | the owner's valid value, else the browser's unless the owner denies it | the owner's valid value, else a browser-seeded value if the IDs do not expire | +| Unowned | the browser's value (today's rule) | the server value, if any (today's rule) | "The owner's valid value" means a record with `w` set to the owner that is not past its `exp`. A browser-seeded record never beats the browser's @@ -2351,7 +2429,25 @@ A server value uses its stored `atype`, else `[[ec.partners]].openrtb_atype`, and its stored UID-level `ext`. A value the provider wrote also gets the EID-level `inserter`, `matcher` and `mm` the provider declared for that source (`with_provenance`, section 4.3). A browser value keeps the browser's -own EID-level fields, as the EID fidelity work requires. Sources with `bidstream_enabled = false` are +own EID-level fields, as the EID fidelity work requires. + +**Owner denials apply to browser candidates.** Before the precedence above, +the browser's candidate for an owned source passes the owner's denials, read +from KV and from `ts-eids-r` alike, so they hold before the removal reaches +KV: + +- A browser UID whose keyed hash is in the owner's `ps.iv` is dropped. Once + the owner invalidates UID A, an A still in the `/auction` body or in + `ts-eids` is not sent. +- While the owner has a sticky `ps.o`, the browser's value for every source + that provider owns is dropped, until the opt-out clears (section 8). + +These run before `gate_eids_by_consent`, which knows nothing of +provider-specific rejections. `ps.iv` keeps the four most recent hashes, so +an older rejected UID can be sent again once evicted. Unowned sources keep +today's browser-first rule: no provider can veto them. + +Sources with `bidstream_enabled = false` are left out, as `resolve_partner_ids` does today (`ec/eids.rs:36`). The merged set is consent-gated (`gate_eids_by_consent`) and reported in @@ -2361,14 +2457,36 @@ script that wraps `fetch`, so `HttpOnly` on `ts-eids-s` does not hide server IDs from the page. The header stays for validation and troubleshooting, and this exposure is accepted. -One shared servability check covers expiry, the rollback rule, the owner's -consent policy, and opt-outs: a record whose writer has `ps.o` set, in KV or +One shared servability check covers expiry, the rollback rule, the +writer's and owner's consent policies, and opt-outs: a record whose writer has `ps.o` set, in KV or in `ts-eids-r`, is never served, even before the removal reaches KV. `resolve_partner_ids` (`ec/eids.rs:36`), the admin view (`ec/admin.rs:485`) and `/_ts/api/v1/identify` all use it. `/identify` reads `entry.ids` directly today (`ec/identify.rs:99-103`), so it switches to the check. The stale module comment at `ec/eids.rs:1-5` is corrected. So expired IDs are never returned by any of them (G7). +**Consent for `/identify`.** The policies are checked against the visitor's +consent on this request, never against the row's stored `consent.tcf` and +`consent.gpp`, which are snapshots from creation and authorize nothing. +`/identify` has the visitor's context by construction. Its EC comes only +from the `ts-ec` cookie, never from a header (`ec/mod.rs:325`), and its +consent is built from the same request's cookies, `Sec-GPC` and +geolocation (`ec/mod.rs:338-345`). A call that does not carry the visitor's +own cookies has no EC and gets 204 (`ec/identify.rs:73-83`), so a partner's +server cannot ask on a visitor's behalf. Within such a request: + +- An absent consent string is treated as the auction treats it. Where TCF + applies, a strict policy finds no vendor consent and blocks. Elsewhere + only the signals present (GPC, GPP, US Privacy) apply. +- The geolocation is the requester's, which for a browser call is the + visitor's. +- A later vendor denial blocks that provider's IDs even while global EC + consent still passes, because each ID needs its writer's and owner's + policy, not only `ec_consent_granted`. + +No persisted visitor authorization is added, so `IdentityMutation` carries +no consent update. + ### 7.2 Ownership semantics - **Owned source.** The owner is authoritative. It refreshes when due @@ -2398,6 +2516,8 @@ cookies: positive changes into `ts-eids-s`, removals and opt-outs into | Result or error | Stored IDs | `ps` | Log | | ------------------------------------------------------------------------------ | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ----- | | `Resolution::found` | `OwnerWrite` or `FillMissing` per source | `f = 0`; `st` replaced or cleared when the resolution says so | debug | +| A stateful result whose IDs or state lost to a newer write (section 5.4) | unchanged; the result's removals still apply | unchanged | debug | +| `replacing_state` oversized or unserializable | no IDs stored; the result's removals still apply | `st` unchanged; `f` and `n` as for `Malformed` | warn | | `Resolution::no_match` | unchanged | `n = now + retry_in` (default 24 h, max 7 d) | debug | | `Resolution::invalidate`, `also_invalidate` | `OwnerRemove` with the revision the request read | UID hash added to `iv`; `n = now`: eligible for `Initial` | info | | `Resolution::opted_out` | `WriterRemove` for every record this provider wrote, owned or unowned | `o = {at: now, fp}`, `st` cleared | info | @@ -2494,8 +2614,10 @@ Rules: written into `ts-eids-s`, pass the generic gate and the writing provider's policy against the current consent (section 5.3). Removals and opt-outs never need consent to apply (section 5.6). -- **In the bidstream and `/identify`:** server-side IDs pass the owner's - policy (section 7.1). +- **In the bidstream and `/identify`:** server-side IDs pass their + writer's policy and, for an owned source, the owner's, against the + request's visitor consent (section 7.1). Owner denials also filter the + browser's candidates. - **On withdrawal:** the tombstone is written synchronously on whichever request carries it, and the response expires `ts-eids-s` (section 5.1). @@ -3236,7 +3358,8 @@ impl IdentityProvider for Uid2Provider { let id = ResolvedId::new(input.primary_source(), identity.advertising_token) .with_atype(3) .with_expires_at(UnixTime::from_millis(identity.identity_expires)) - .with_refresh_at(UnixTime::from_millis(identity.refresh_from)); + .with_refresh_at(UnixTime::from_millis(identity.refresh_from)) + .with_refresh_until(UnixTime::from_millis(identity.refresh_expires)); Ok(Resolution::found([id]).replacing_state(&Uid2State { refresh_token: identity.refresh_token, refresh_response_key: identity.refresh_response_key, @@ -3258,7 +3381,9 @@ What this shows: deploy validation runs before resolution (section 10.2). - **Expired tokens refresh.** A `Refresh` runs for an expired advertising token too, so a visitor who misses the refresh window still recovers while - the refresh token lives. The provider checks `refresh_expires` itself. + the refresh token lives. `with_refresh_until` keeps the expired record as + a refresh anchor until `refresh_expires` (section 5.4), and the provider + still checks `refresh_expires` itself. - **No dry run.** UID2 keeps private state, so it gets no dry run (section 5.2), and its `request` always builds the real, encrypted call. - **Randomness.** The envelope needs a random IV and nonce. A provider has no @@ -3586,6 +3711,15 @@ refresh and opt-out. - A provider's `ResolveKind` is `Refresh` for a stored record whether it is still valid or expired, and an expired record counts as absent for `FillMissing`. +- An expired UID2-style ID with valid refresh state survives an unrelated + writer's mutation as a refresh anchor, and the next eligibility pass + selects `Refresh`. Past `ru`, or once the state is cleared, missing or + fails to decode, the anchor is dropped and the kind is `Initial`. +- A stateful result applies whole. When a phase 4 re-read finds another + writer's newer ID and state, the call's IDs, state, `n` and `f` are all + dropped and the row keeps the other writer's ID with that writer's state. + An oversized or unserializable replacement state stores no ID, and the + result's removals still apply. **Consent.** @@ -3617,6 +3751,7 @@ so adapter tests can use it. `KvIdentityGraph::counting` counts reads only | Integration-proxy navigation that mints an EC | 1 (create) | | `/auction`, `/_ts/page-bids`, `/identify`, Lockr API proxy, assets | 0 | | Consent withdrawal, on any request | CAS tombstone (up to 5 attempts) plus marker, per ID | +| Removal batch that overflows the removal cookies, on any request | CAS removal write before send (up to 5 attempts) | | CAS conflict or per-key limit on the flush | 1 attempt, not retried | **Withdrawal and removal.** @@ -3660,6 +3795,9 @@ existing one read and no more. owned source is never sent from the server. - Revision-checked `OwnerRemove` and `OwnerWrite` lose to a newer write. Re-applying changes on a later writer keeps the other tab's change. +- Two refreshes of an unchanged UID in the same second get different `rv`, + so an invalidation read before the second refresh does not remove it. +- An opt-out and a clear in the same second leave the provider opted out. - `ts-eids-s`: - sealing and opening with the EC ID as associated data; a cookie sealed for another EC, or tampered with, fails to open and is ignored without @@ -3681,16 +3819,23 @@ existing one read and no more. `/auction` that rewrites `ts-eids-s` from an older copy - two removal-producing responses in flight together both keep their batches, each in its own cookie - - a fifth batch merges into the lowest openable ``; a partly applied - cookie is kept whole; an over-cap batch coalesces, drops `iv`, then - splits; a merge that does not fit rewrites all four per provider + - a batch that does not fit one cookie, or meets four openable cookies, + is written to KV before send; when that write fails it is set in new + cookies of its own, and no other cookie is rewritten + - concurrent responses that each create a first cookie, or each + overflow, from the same inbound snapshot all keep their batches + - a 253-byte source domain and a long provider ID fit the per-cookie + budget, and no batch is widened, stripped of `iv` or turned into an + opt-out to fit + - a partly applied cookie is kept whole - a Lockr revoke followed by a re-consent capture, before and across navigations, ends with Lockr's new IDs in KV and the old opt-out never re-applied (`ps.oc` in KV), including after a failed flush - a removal names its source by domain, so a config change that reorders sources removes the right one - a cookie that fails to open is deleted by the writer and never counted - - a removal without `pv` skips a record its writer re-acquired after `sa` + - a removal without `pv` skips a record its writer re-acquired after `sa`, + and removes one acquired in the same second as `sa` - it is honoured on the unusable-consent path, where `ts-eids-s` is expired, until the next writer applies it - the 512-byte sealed cap per cookie, the one-year `Max-Age`, and that @@ -3711,7 +3856,18 @@ nothing is left. - A browser-seeded record never beats the browser's current value. - Expired IDs are never sent. - `x-ts-eids` matches what was sent. -- `/_ts/api/v1/identify` omits expired IDs and IDs the owner's policy blocks. +- After the owner invalidates UID A, or reports an opt-out, a stale A in the + `/auction` body or in `ts-eids` is not sent, before and after the removal + reaches KV. +- An unowned source a strict provider wrote stops reaching bidders and + `/identify` once that provider's vendor consent is denied or a + targeted-advertising opt-out applies. A record whose writer is disabled + is not served, and the browser can refill it. +- `/_ts/api/v1/identify` omits expired IDs and IDs the writer's or owner's + policy blocks under the request's consent. It never uses the stored + consent snapshot. A request from another jurisdiction, one with no + consent string, and a later vendor denial while global EC consent is + still granted each get that policy's answer for the request. **Batch sync.** An update to an owned source applies only when absent. @@ -3748,7 +3904,8 @@ Three steps, each shippable on its own: revalidation, KV-gap staging on `/auction`, pending and `m` markers, the size cap - the `ts-eids-r` removal cookies - - a metric for the inbound `Cookie` header size (section 6.2) + - a metric for the inbound `Cookie` header size, and for the number and + total size of `ts-eids-r` cookies (section 6.2) - metadata-preserving `KvPartnerId` and `ps`, with `Default` and constructors - `GenericPullProvider` replacing pull sync, which brings the trait, the @@ -3822,7 +3979,7 @@ Decisions made while writing this spec, with the user: | Deletion | A tombstone with the 24-hour `TOMBSTONE_TTL` replaces the hard delete. No extra writes. | | GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | | `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | -| Consent withdrawal | Synchronous tombstone on any request, the only G4 exception. Proposed in review; not yet confirmed. | +| Consent withdrawal | Synchronous tombstone on any request, one of two G4 exceptions with the removal overflow write. Proposed in review; not yet confirmed. | Round 7 proposed these, and they are applied in revision 8 pending the user's review of the spec: @@ -3838,6 +3995,19 @@ user's review of the spec: | KV read cost | An auction navigation reads once, as today. Phase 4 reuses that snapshot unless an after-send provider runs | | Provider count | At most 32 enabled providers, so a row's `ps` never fills up | +Round 10 proposed these, and they are applied in revision 11 pending the +user's review of the spec: + +| Decision | Choice | +| ------------------- | --------------------------------------------------------------------------------------------------------------- | +| Removal overflow | A synchronous, removal-only KV write before send, the second G4 exception. Batches are never merged or widened | +| Record revision | A random `rv` per record write. Ties between a removal or opt-out and an acquisition or clear go to the removal | +| Stateful results | IDs, state and scheduling apply atomically. An invalid replacement state rejects the positive result | +| Refresh anchors | An expired owned record is kept, unserved, while its owner holds state, until its `ru` | +| Disclosure policy | The writer's policy always, plus the owner's. Records of disabled writers are not served | +| Browser candidates | The owner's `ps.iv` and `ps.o` filter the browser's value for owned sources | +| `/identify` consent | The request's visitor consent. No persisted visitor authorization | + ## 17. Review log **Round 1** (2026-10-21) found 5 blocking issues, 16 significant gaps and 17 @@ -4074,3 +4244,23 @@ revision 10: source (4.5). - Derives are corrected (`When` has no `Display`; `Hash` and `Ord` added), and stale wording is fixed. + +**Round 10** (2026-10-21), an outside review of revision 10, found 6 blocking +issues and 1 question. Small executable models confirmed the ordering and +concurrency counterexamples. Revision 11 resolves them: + +| Finding | Resolution | +| -------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| The browser fallback could resend an ID the owner invalidated or opted out | Owner denials filter browser candidates for owned sources, 7.1 | +| Unowned provider-written IDs skipped the writer's consent policy | Writer's policy always, owner's too; disabled writers unserved, 4.7, 7.1, 9.1 | +| A re-read could pair one call's ID with another's state | `ProviderResult` applies atomically; invalid state rejects it, 4.5, 5.4, 8 | +| Pruning an expired ID stranded valid refresh state | Refresh anchors bounded by `ru`, `with_refresh_until`, 4.5, 4.7, 5.4, 6.1, 12.3 | +| Removal-cookie overflow could lose a batch under concurrency | Removal overflow write before send; measured byte budget; no coarsening, 6.2 | +| `hash(uid, at)` and equal-second ties could dismiss a revocation | Random `rv`; ties go to the removal or opt-out, 5.4, 6.1, 6.2 | +| The consent `/identify` checks against was undefined | The request's visitor consent; the stored snapshot authorizes nothing, 7.1 | + +The review's non-blocking notes are not applied here: the contract covers +the declared capabilities rather than every provider shape, HEM intake and +multi-step exchanges stay out of v1 (section 2.2), and the vendor examples +are illustrations, to be checked against protocol fixtures in their own +specs. From fbae25f41e0b0c041cf2c7952111cb26442f0065 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 22 Oct 2026 13:11:29 +0530 Subject: [PATCH 03/89] Confirm the removal overflow write and equal-second tie rule State that the overflow write, like the withdrawal tombstone, retries only a CAS generation conflict and stops at any other failure, Fastly's per-key limit included, so it never repeats a rate-limited write. Move both round 10 decisions out of the pending table. Signed-off-by: dhruv8sh --- ...0-server-side-identity-providers-design.md | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md index d46bc42af6..e6fa017e64 100644 --- a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md +++ b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md @@ -2211,10 +2211,12 @@ already carries four openable `ts-eids-r` cookies. This removal overflow write is the second page-load exception to G4 (section 5.1). It reads the row and applies only the batch's removals, opt-outs and `iv` hashes under the rules of section 5.4, with a CAS and the withdrawal path's five -attempts. When it lands, or the read shows the batch already applied, no -cookie is set. When it fails, the response sets the batch in new cookies of -its own, split across as many 512-byte cookies as it needs, so the batch is -never lost. +attempts. Like that path, it re-reads and tries again only when the CAS +generation no longer matches. Any other failure, Fastly's per-key limit +included, ends the write at once. When it lands, or the read shows the batch +already applied, no cookie is set. When it fails, the response sets the +batch in new cookies of its own, split across as many 512-byte cookies as it +needs, so the batch is never lost. Four is where TS stops relying on cookies, not a hard cap. Concurrent responses each count from their own inbound snapshot, so five responses @@ -3980,6 +3982,8 @@ Decisions made while writing this spec, with the user: | GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | | `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | | Consent withdrawal | Synchronous tombstone on any request, one of two G4 exceptions with the removal overflow write. Proposed in review; not yet confirmed. | +| Removal overflow | A synchronous, removal-only KV write before send, the second G4 exception. It retries only a CAS conflict and falls back to new cookies. Confirmed after round 10. | +| Equal-second ties | A removal or opt-out wins a tie with an acquisition or clear. An ID re-acquired in that second is resolved again later. Confirmed after round 10. | Round 7 proposed these, and they are applied in revision 8 pending the user's review of the spec: @@ -3998,15 +4002,14 @@ user's review of the spec: Round 10 proposed these, and they are applied in revision 11 pending the user's review of the spec: -| Decision | Choice | -| ------------------- | --------------------------------------------------------------------------------------------------------------- | -| Removal overflow | A synchronous, removal-only KV write before send, the second G4 exception. Batches are never merged or widened | -| Record revision | A random `rv` per record write. Ties between a removal or opt-out and an acquisition or clear go to the removal | -| Stateful results | IDs, state and scheduling apply atomically. An invalid replacement state rejects the positive result | -| Refresh anchors | An expired owned record is kept, unserved, while its owner holds state, until its `ru` | -| Disclosure policy | The writer's policy always, plus the owner's. Records of disabled writers are not served | -| Browser candidates | The owner's `ps.iv` and `ps.o` filter the browser's value for owned sources | -| `/identify` consent | The request's visitor consent. No persisted visitor authorization | +| Decision | Choice | +| ------------------- | ---------------------------------------------------------------------------------------------------- | +| Record revision | A random `rv` per record write | +| Stateful results | IDs, state and scheduling apply atomically. An invalid replacement state rejects the positive result | +| Refresh anchors | An expired owned record is kept, unserved, while its owner holds state, until its `ru` | +| Disclosure policy | The writer's policy always, plus the owner's. Records of disabled writers are not served | +| Browser candidates | The owner's `ps.iv` and `ps.o` filter the browser's value for owned sources | +| `/identify` consent | The request's visitor consent. No persisted visitor authorization | ## 17. Review log From 21fd17269d63717e25804c3a24d39466f61b01eb Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 22 Oct 2026 16:25:06 +0530 Subject: [PATCH 04/89] Add implementation plan for server-side identity providers Signed-off-by: dhruv8sh --- ...26-10-22-server-side-identity-providers.md | 7442 +++++++++++++++++ 1 file changed, 7442 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-22-server-side-identity-providers.md diff --git a/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md b/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md new file mode 100644 index 0000000000..4ce4338879 --- /dev/null +++ b/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md @@ -0,0 +1,7442 @@ +# Server-side Identity Providers Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let Trusted Server call identity providers (ID5, LiveRamp ATS, Lockr, UID2 and generic pull partners) from the edge through one `IdentityProvider` trait and one coordinator. Every page load makes at most one browser-driven KV write. Provider IDs are cached first in a sealed first-party cookie (`__Host-ts-eids-s`) and then in KV, and removals always win. + +**Architecture:** New code lives in `crates/trusted-server-core/src/ec/identity/`. Providers are pure. They describe themselves with a `ProviderDescriptor`, build `ProviderRequest`s and parse buffered `ProviderResponse`s into `Resolution`s. The `IdentityCoordinator` owns eligibility, consent, transport limits, backoff and turning results into changes. Each request is either the **writer** (a real-browser top-level document navigation) or a **stager**. The writer adds every KV change to one request-scoped `IdentityMutation` and applies it with one conditional write after the response is sent. Stagers never write KV. They save positive changes in `__Host-ts-eids-s` and removals in per-batch `__Host-ts-eids-r-` cookies, and the next writer flushes them. Today's pull sync becomes `GenericPullProvider`. + +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `trusted-server-adapter-fastly`, `error-stack`, `derive_more`, `serde`/`serde_json`, `hmac`/`sha2`, `base64`, `rand`, `flate2`/`brotli` (already dependencies), and a new `aes-gcm = "0.10"`. Fastly Compute (Viceroy for tests). + +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md` (revision 11, branch `spec/1246-server-side-identity-providers`). Read it before starting. "Spec 5.4" means section 5.4 of that file. Where this plan and the spec disagree, the spec wins. Stop and report the disagreement instead of choosing one. The spec cites `file:line` at commit `c84a73da7`. Line numbers have drifted slightly since then, so find code by the function names given here, not by line number. + +## How this plan is split + +The spec's rollout (spec 14) has three steps, and each step can ship on its own. This plan has one **Part** per step, and each Part is one pull request. Each PR's branch is stacked on the previous one: + +| Part | Spec step | Branch | Base | Tasks | +| ---- | ------------------------ | ---------------------------------- | ------------------------------------------ | -------- | +| 0 | 0. Strips first | `feat/1246-identity-cookie-strips` | `spec/1246-server-side-identity-providers` | 1 to 4 | +| 1 | 1. One write, no vendors | `feat/1246-identity-single-write` | `feat/1246-identity-cookie-strips` | 5 to 31 | +| 2 | 2. Providers | `feat/1246-identity-providers` | `feat/1246-identity-single-write` | 32 to 41 | + +Part 0 is the rollback floor (spec 14). Do not start Part 1 until Part 0's PR is open. Do not start Part 2 until Part 1's one-write invariant tests (Task 30) pass. + +## Global Constraints + +Copied from the spec and `AGENTS.md`. Every task implicitly includes these. + +- Follow `AGENTS.md` for commands, style, error handling and CI gates. Bare `cargo test`, `cargo build` and `cargo check` fail at the workspace root by design. Use the aliases (`cargo test-fastly`, `cargo clippy-fastly`, and the others). +- Errors use `error-stack` `Report` with `derive_more::Display` enums and `impl core::error::Error`. No `anyhow` and no `thiserror`. Use `expect("should ...")`, never `unwrap()`, in production and in tests. +- No local imports inside functions. No wildcard imports except `use super::*` in test modules. Comments go on their own line, above the code. +- Every public item has a doc comment, and every fallible public function has `# Errors`. Every builder method and getter is `#[must_use]`. Spec 4.2, "Conventions for the listings", gives the derive list for each type. Copy it exactly. +- Never log UIDs, envelopes, HEMs, provider state, consent strings, cookie values, or provider URLs with a query string. EC IDs go through `log_id`. Log provider errors with `current_context()` only, never `{:?}` of the report (spec 9.3). +- Use only example or fictional data in code, tests, comments and docs (`example.com` hosts). Public protocol source domains (`id5-sync.com`, `liveramp.com`, `uidapi.com`, `sharedid.org`) may appear where the spec uses them. +- **Fixed values from the spec. Do not change them:** + - `__Host-ts-eids-s`: sealed value at most **2,048** bytes. `__Host-ts-eids-r-<8 hex>`: sealed value at most **512** bytes, normally at most **4** cookies. Both: `Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=31536000`, with no `Domain`. + - Seal: AES-256-GCM, value `v1..`, base64url without padding, 12-byte nonce, the full EC ID as associated data. Key labels: `trusted-server/ts-eids-s/key/v1` and `trusted-server/ts-eids-r/key/v1`, each derived with HMAC-SHA256 from `ec.passphrase`. + - KV: `ext` at most **256** serialized bytes, `ps.st` at most **1,024** bytes of embedded JSON, at most **32** `ps` entries, at most **4** `ps.iv` hashes, `rv` is **16** hex digits (64 random bits). `SCHEMA_VERSION` stays **1**. + - `ProviderRequest` call context: at most **1,024** bytes of JSON. `IdentityProviderId`: `^[a-z][a-z0-9_-]{0,62}$`. + - `[ec.identity]` defaults: `max_concurrency = 4`, `hold_budget_ms = 100`, `hold_retry_sec = 300`, `max_response_bytes = 65536`, `pending_grace_sec = 300`, `staged_max_age_sec = 604800`, `kv_revalidate_sec = 3600`. Per-provider defaults: `mode = "after_send"`, `timeout_ms` = `hold_budget_ms` in hold mode and `1000` after send, `refresh_after_sec = 86400`, `max_calls_per_minute = 1`, `breaker_failures_per_minute = 50`, `cookie_priority = 0`. + - Backoff (spec 8): a `no_match` waits `n = now + retry_in`, default 24 h and at most 7 d. An input-caused failure waits `min(1 h × 2^(f-1), 24 h)`. A transport, 429 or 5xx failure waits `min(5 min × 2^(f-1), 1 h)`. A cut-off hold call waits `hold_retry_sec`. A skip waits `refresh_after_sec`. A marked provider that was not dispatched gets `ps.n = max(ps.n, now + 60 s)`. +- **G4 is strict.** A page load makes at most one browser-driven KV write and never retries it. The only exceptions are the consent-withdrawal tombstone path (unchanged) and the removal overflow write (Task 27). +- **Removals win.** No filter (consent re-gate, staleness, ownership, `cf` mismatch, size cap) may drop a removal, an opt-out or an `iv` hash. Only the EC binding and, when present, the revision check apply to them (spec 5.6). +- **Keep `ts-eids` and `x-ts-eids`.** Do not remove either (product requirement). +- **Feature gate for test support.** The spec names a `test-support` feature. `trusted-server-core` already has a `test-utils` feature for the same purpose (`crates/trusted-server-core/Cargo.toml`). Reuse `test-utils` and gate on `#[cfg(any(test, feature = "test-utils"))]`. This is the only intended deviation from the spec. Mention it in the PR description. +- Test discipline: + 1. **Red first.** Run each new test against unchanged code and watch it fail for the expected reason before you implement. + 2. **Pair absence with presence.** Each "X is gone" assertion sits next to a "control Y is still there" assertion on the same value. + 3. Compare structured outputs whole (full `KvEntry`, full `serde_json::Value`). Do not use `..Default::default()` in exhaustiveness tests. List every field. + 4. Use `json!` for JSON fixtures, except where exact byte lengths matter. +- Commits: sentence case, imperative mood, no `fix:`/`feat:` prefixes, no bracketed tags, `git commit --signoff -S`, one `-m` flag per paragraph, no heredoc, no AI attribution lines. + +## Test commands + +| What | Command | +| --------------------------------------- | --------------------------------------------------------------------------------------------- | +| Core and Fastly adapter (Viceroy) | `cargo test-fastly` | +| One core test by name | `cargo test-fastly -- ` | +| Fastly adapter with reusable sandbox | `cargo test-fastly-reuse` | +| Other adapters (Part 0 strips, Task 31) | `cargo test-axum && cargo test-cloudflare && cargo test-spin` | +| Lint, Fastly target | `cargo clippy-fastly` | +| Docs | `cargo doc --no-deps --all-features -p trusted-server-core` | +| Parity crate | `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` | +| CLI (config validation, Task 33) | `./scripts/test-cli.sh` | +| Full gate before each PR | `/check-ci` | + +`cargo test-fastly -- ` runs only the tests whose path contains ``. Every "Run" step below gives the exact filter. + +## File map + +| File | Responsibility | Tasks | +| ------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------- | +| `Cargo.toml` (workspace), `crates/trusted-server-core/Cargo.toml` | `aes-gcm` dependency | 5 | +| `crates/trusted-server-core/src/constants.rs`, `src/cookies.rs` | Cookie names, identity cookie lists, name-and-prefix strip | 1 | +| `crates/trusted-server-core/src/auction/routing.rs`, `src/integrations/prebid.rs` | Strip identity cookies from Prebid Server requests | 2 | +| `crates/trusted-server-core/src/publisher.rs` | Origin strip; `is_prefetch_request` move; page-bids stops ingest; navigation merge; hold mode | 3, 23, 26, 28, 36, 37 | +| `crates/trusted-server-core/src/cookies/template_cache_policy.rs` | New cookies refused as cache keys | 4 | +| `crates/trusted-server-core/src/ec/identity/mod.rs`, `ids.rs`, `resolution.rs` (new) | Module, ID newtypes, error type, results | 5, 10 | +| `crates/trusted-server-core/src/ec/kv_types.rs` | `KvPartnerId` metadata, `KvProviderState`, `KvEntry.ps`, validation | 6 | +| `crates/trusted-server-core/src/ec/kv.rs` | Constructors; `PartnerIdUpdate.acquired_at`; `compute_cluster_size`; removal overflow write; counting store | 6, 7, 22, 24, 27 | +| `crates/trusted-server-core/src/ec/admin.rs` | Redact `ps.st`; servability; operator opt-out clear | 6, 14, 36 | +| `crates/trusted-server-core/src/ec/identity/descriptor.rs` (new) | Descriptor, builder, `Resolves`, `Captures`, `ConsentPolicy`, `EidProvenance`, `Input`, `Access`, `ResolveKind`, `UsOptOut` | 5, 8 | +| `crates/trusted-server-core/src/ec/identity/input.rs` (new) | `ResolveInput`, `CaptureInput`, `StoredId`, `Region`, `HashedEmail`, `ProviderConsent`, randomness | 9 | +| `crates/trusted-server-core/src/ec/identity/exchange.rs` (new) | `ProviderRequest`, `ProviderResponse` | 10 | +| `crates/trusted-server-core/src/ec/identity/provider.rs` (new) | `IdentityProvider`, `IdentityProviderConfig` | 11 | +| `crates/trusted-server-core/src/ec/identity/policy.rs` (new), `src/consent/*` | Policy evaluation, GPP opt-outs, `stricter_of`, fingerprint, redacting `Debug` | 12, 34 | +| `crates/trusted-server-core/src/ec/identity/revision.rs` (new) | `IdentityKeys`, `Revision`, keyed hashes | 13 | +| `crates/trusted-server-core/src/ec/identity/catalog.rs`, `servable.rs` (new) | Provider catalog, ownership, `cf`; the servability check | 14, 22, 32, 33 | +| `crates/trusted-server-core/src/ec/eids.rs`, `ec/identify.rs` | Servability for KV IDs; `/identify` stops writing | 14, 24 | +| `crates/trusted-server-core/src/ec/identity/mutation.rs` (new) | `IdentityMutation` and its apply algorithm | 15 | +| `crates/trusted-server-core/src/ec/identity/cookie.rs` (new) | Seal and open, `ts-eids-s`, size cap, KV merge, `ts-eids-r` | 16, 17, 18 | +| `crates/trusted-server-core/src/ec/identity/config.rs` (new), `src/settings.rs`, `src/config.rs` | `[ec.identity]`, `owner`, deploy and runtime validation | 19, 33 | +| `crates/trusted-server-core/src/ec/identity/coordinator.rs` (new) | Eligibility, outcomes, dispatch, breaker, hold, capture, re-grants | 20, 21, 35, 36, 37, 38 | +| `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (new), `ec/pull_sync.rs` | `GenericPullProvider`; pull sync dispatcher removed | 22, 29 | +| `crates/trusted-server-core/src/ec/identity/test_support.rs` (new) | Shared fixtures, `ProviderHarness`, fake HTTP client and limiter | 14, 15, 20, 21, 22, 35, 39 | +| `crates/trusted-server-core/src/ec/identity/request.rs` (new), `src/ec/mod.rs`, `src/http_util.rs` | Writer decision, phase-1 captures, request identity state, read triggers | 23, 29 | +| `crates/trusted-server-core/src/ec/finalize.rs` | Mutation instead of writes; stager cookies; `ts-eids-r` on every path; metrics | 25, 26, 27, 29, 31 | +| `crates/trusted-server-core/src/auction/endpoints.rs` | KV-gap staging; server set in the merge; owner denials; hold mode | 26, 28, 36, 37 | +| `crates/trusted-server-core/src/ec/identity/flush.rs` (new) | Phase 4 `run_after_send` | 29 | +| `crates/trusted-server-adapter-fastly/src/app.rs`, `src/main.rs`, `src/rate_limiter.rs`, `tests/identity_writes.rs` | Writer decision, status capture, phase 4, `record`/`count`, invariant tests | 23, 25, 29, 30, 35 | +| `crates/trusted-server-core/src/integrations/registry.rs` | Provider registration, capture-path check, capture plumbing, `is_real_browser` | 23, 32, 38 | +| `crates/trusted-server-core/src/ec/rate_limiter.rs` | `record` and `count` | 35 | +| `crates/trusted-server-core/src/ec/batch_sync.rs` | Owned sources fill only when absent | 40 | +| `crates/trusted-server-core/src/ec/identity/examples.rs` (new, tests only) | Spec 4.6 reference providers | 39 | +| `crates/trusted-server-core/src/migration_guards.rs` | Every new `ec/identity/*.rs` file | 5 onward | +| `docs/guide/*.md`, `CHANGELOG.md`, `trusted-server.example.toml` | Docs | 4, 19, 31, 41 | + +------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------- | +| `Cargo.toml` (workspace), `crates/trusted-server-core/Cargo.toml` | `aes-gcm` dependency | 5 | +| `crates/trusted-server-core/src/constants.rs` | New cookie-name constants | 1 | +| `crates/trusted-server-core/src/cookies.rs` | Identity-cookie lists and the name-and-prefix strip helper | 1 | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Strip identity cookies from Prebid Server requests | 2 | +| `crates/trusted-server-core/src/publisher.rs` | Origin strip; writer-request hooks; page-bids stops ingest; navigation merge and hold mode | 3, 22, 25, 27, 36 | +| `crates/trusted-server-core/src/cookies/template_cache_policy.rs` | New cookies refused as cache keys | 4 | +| `crates/trusted-server-core/src/ec/identity/mod.rs` (new) | Module docs, re-exports | 5 | +| `crates/trusted-server-core/src/ec/identity/ids.rs` (new) | `IdentityProviderId`, `SourceDomain`, `UnixTime` | 5 | +| `crates/trusted-server-core/src/ec/kv_types.rs` | `KvPartnerId` metadata, `KvProviderState`, `KvEntry.ps`, validation | 6 | +| `crates/trusted-server-core/src/ec/kv.rs` | Literals to constructors; `PartnerIdUpdate.acquired_at`; mutation flush; overflow write | 6, 7, 21, 26 | +| `crates/trusted-server-core/src/ec/admin.rs` | Redact `ps.st`; servability in the admin view | 6, 13 | +| `crates/trusted-server-core/src/ec/identity/descriptor.rs` (new) | `ProviderDescriptor`, builder, `Resolves`, `Captures`, `ConsentPolicy`, `EidProvenance`, `Input`, `Access`, `UsOptOut` | 8 | +| `crates/trusted-server-core/src/ec/identity/input.rs` (new) | `ResolveInput`, `CaptureInput`, `StoredId`, `Region`, `HashedEmail`, `ProviderConsent`, `ResolveKind` | 9 | +| `crates/trusted-server-core/src/ec/identity/exchange.rs` (new) | `ProviderRequest`, `ProviderResponse` | 10 | +| `crates/trusted-server-core/src/ec/identity/resolution.rs` (new) | `Resolution`, `ResolvedId`, `When`, `StateChange`, `IdentityProviderError` | 10 | +| `crates/trusted-server-core/src/ec/identity/provider.rs` (new) | `IdentityProvider`, `IdentityProviderConfig` | 11 | +| `crates/trusted-server-core/src/ec/identity/revision.rs` (new) | `Revision`, `rv` generation, keyed hashes (`pv` fallback, `iv`, visitor key, fingerprint) | 12 | +| `crates/trusted-server-core/src/ec/identity/servable.rs` (new) | The shared servability check | 13 | +| `crates/trusted-server-core/src/ec/eids.rs`, `ec/identify.rs` | Use the servability check; `/identify` stops writing | 13, 23 | +| `crates/trusted-server-core/src/ec/identity/mutation.rs` (new) | `IdentityMutation`, `PartnerIdChange`, `ProviderResult`, apply algorithm | 14 | +| `crates/trusted-server-core/src/ec/identity/cookie.rs` (new) | Seal and open, `ts-eids-s` payload, size cap, `ts-eids-r` batches, merge with KV | 15, 16, 17 | +| `crates/trusted-server-core/src/ec/identity/config.rs` (new) | `[ec.identity]` settings and validation | 18, 32 | +| `crates/trusted-server-core/src/settings.rs` | `Ec.identity`; `EcPartner.owner`; deprecate `pull_sync_concurrency` | 18, 32 | +| `crates/trusted-server-core/src/ec/identity/coordinator.rs` (new) | Eligibility, merged view, dispatch, outcomes, markers, breaker | 19, 20, 34, 35 | +| `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (new) | `GenericPullProvider` | 21 | +| `crates/trusted-server-core/src/ec/pull_sync.rs` | Reduced to helpers `GenericPullProvider` reuses; dispatch removed | 21, 28 | +| `crates/trusted-server-core/src/ec/identity/test_support.rs` (new) | `ProviderHarness`, `WriteCountingEcKv`, fake HTTP client | 11, 21, 40 | +| `crates/trusted-server-core/src/ec/mod.rs` | `EcContext` writer flag, phase-1 captures, identity cookie state, mutation | 22 | +| `crates/trusted-server-core/src/http_util.rs` | `is_prefetch_request` moves here; `is_document_navigation` | 22 | +| `crates/trusted-server-core/src/ec/finalize.rs` | Mutation instead of ingest writes; stager cookie saves; `ts-eids-r` on every path | 24 | +| `crates/trusted-server-core/src/auction/endpoints.rs` | KV-gap staging; server set in `merge_auction_eids`; hold mode | 25, 27, 36 | +| `crates/trusted-server-adapter-fastly/src/app.rs`, `src/main.rs` | Writer decision, status capture, phase 4 entry point | 22, 28 | +| `crates/trusted-server-core/src/integrations/registry.rs` | Identity provider registration; capture plumbing; capture-path check | 31, 37 | +| `crates/trusted-server-core/src/config.rs` | `validate_identity_integration`, ownership validation | 32 | +| `crates/trusted-server-core/src/consent/{gpp.rs,tcf.rs,types.rs,mod.rs}` | Sharing and targeted-advertising opt-outs, section 6, GPC and known-child, `has_vendor_consent`, `stricter_of`, redacting `Debug` | 33 | +| `crates/trusted-server-core/src/ec/identity/policy.rs` (new) | `ConsentPolicy` evaluation and the consent fingerprint | 33 | +| `crates/trusted-server-core/src/ec/rate_limiter.rs`, `adapter-fastly/src/rate_limiter.rs` | `record` and `count` | 34 | +| `crates/trusted-server-core/src/ec/batch_sync.rs` | Owned sources fill only when absent | 39 | +| `docs/guide/edge-cookies.md`, `configuration.md`, `api-reference.md`, `ec-setup-guide.md`, `integrations/prebid.md` | Docs | 4, 30, 41 | + +--- + +# Part 0: Strip the identity cookies first (spec 14, step 0) + +This part only removes cookies from outbound requests. It sets no new cookies and changes no KV behaviour. It ships one release before Part 1, so a rollback after Part 1 still keeps the new one-year cookies away from the origin and from Prebid Server. + +Before you start: create a worktree from the spec branch, `git worktree add -b feat/1246-identity-cookie-strips .claude/worktrees/1246-identity-providers spec/1246-server-side-identity-providers`. The implementation stacks on its spec branch, which already carries the Prebid EID-fidelity work this spec depends on. Build with the toolchain that branch pins in `rust-toolchain.toml`. + +### Task 1: Identity cookie names and a name-and-prefix strip helper + +**Files:** + +- Modify: `crates/trusted-server-core/src/constants.rs` (next to `COOKIE_TS_EIDS`, line 8) +- Modify: `crates/trusted-server-core/src/cookies.rs` (after `CONSENT_COOKIE_NAMES`, and after `strip_cookies`) +- Test: `crates/trusted-server-core/src/cookies.rs` (`mod tests`) + +**Interfaces:** + +- Produces: + - `pub const COOKIE_TS_EIDS_S: &str = "__Host-ts-eids-s";` + - `pub const COOKIE_TS_EIDS_R_PREFIX: &str = "__Host-ts-eids-r-";` + - `pub const TS_IDENTITY_COOKIE_NAMES: &[&str]` = `ts-ec`, `ts-eids`, `__Host-ts-eids-s`, `ts-ec-pull-complete` + - `pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str]` = `ts-eids`, `__Host-ts-eids-s` + - `pub const TS_IDENTITY_COOKIE_PREFIXES: &[&str]` = `__Host-ts-eids-r-` + - `pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str], prefixes: &[&str]) -> String` + - `pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str], prefixes: &[&str])` + +- [ ] **Step 1: Write the failing tests** + +Append to `mod tests` in `cookies.rs`: + +```rust + #[test] + fn strip_cookie_pairs_removes_names_and_prefixes() { + let header = "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=v1.x.y; \ + __Host-ts-eids-r-0a1b2c3d=v1.p.q; other=1; ts-eids-extra=keep"; + + let stripped = strip_cookie_pairs( + header, + TS_ORIGIN_STRIPPED_COOKIE_NAMES, + TS_IDENTITY_COOKIE_PREFIXES, + ); + + assert_eq!( + stripped, "ts-ec=abc; other=1; ts-eids-extra=keep", + "should drop the ID cookies and keep ts-ec and unrelated cookies" + ); + } + + #[test] + fn strip_cookie_headers_joins_every_cookie_field_and_drops_non_utf8() { + let mut headers = http::HeaderMap::new(); + headers.append(header::COOKIE, http::HeaderValue::from_static("ts-ec=abc; ts-eids=W10")); + headers.append( + header::COOKIE, + http::HeaderValue::from_bytes(b"bad=\xff").expect("should build an opaque header"), + ); + headers.append(header::COOKIE, http::HeaderValue::from_static("other=1")); + + strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES); + + let values: Vec<_> = headers.get_all(header::COOKIE).iter().collect(); + assert_eq!(values.len(), 1, "should collapse into one Cookie header"); + assert_eq!(values[0], "other=1", "should keep only the unrelated cookie"); + } + + #[test] + fn strip_cookie_headers_removes_header_when_nothing_is_left() { + let mut headers = http::HeaderMap::new(); + headers.insert( + header::COOKIE, + http::HeaderValue::from_static("ts-ec=abc; __Host-ts-eids-r-00000000=v1.a.b"), + ); + headers.insert(header::USER_AGENT, http::HeaderValue::from_static("ExampleBrowser/1.0")); + + strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES); + + assert!(headers.get(header::COOKIE).is_none(), "should remove the empty Cookie header"); + assert!(headers.get(header::USER_AGENT).is_some(), "should leave other headers alone"); + } +``` + +- [ ] **Step 2: Run the tests and see them fail** + +Run: `cargo test-fastly -- cookies::tests::strip_cookie` +Expected: compile errors, because `strip_cookie_pairs`, `strip_cookie_headers` and the constants do not exist. + +- [ ] **Step 3: Implement** + +In `constants.rs`, under `COOKIE_TS_EIDS`: + +```rust +/// Server-owned, sealed cache of provider-collected EIDs (spec 6.2). +pub const COOKIE_TS_EIDS_S: &str = "__Host-ts-eids-s"; +/// Name prefix of the per-batch removal cookies, `__Host-ts-eids-r-<8 hex>`. +pub const COOKIE_TS_EIDS_R_PREFIX: &str = "__Host-ts-eids-r-"; +``` + +In `cookies.rs`, extend the `crate::constants` import with `COOKIE_SHAREDID` (if you use it later), `COOKIE_TS_EC`, `COOKIE_TS_EC_PULL_COMPLETE`, `COOKIE_TS_EIDS`, `COOKIE_TS_EIDS_R_PREFIX` and `COOKIE_TS_EIDS_S`. Then add, after `CONSENT_COOKIE_NAMES`: + +```rust +/// Trusted Server identity cookies that never reach a third party. +/// +/// Stripped from Prebid Server requests and never added to proxy or provider +/// requests. The `__Host-ts-eids-r-` family is matched by +/// [`TS_IDENTITY_COOKIE_PREFIXES`]. +pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_S, + COOKIE_TS_EC_PULL_COMPLETE, +]; + +/// Identity cookies removed from publisher-origin requests. +/// +/// `ts-ec` and `ts-ec-pull-complete` still reach the origin. The ID cookies +/// are removed so that they cannot push the origin's `Cookie` header past +/// common 8 KB header limits. +pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; + +/// Name prefixes of identity cookie families, stripped wherever the named +/// identity cookies are. +pub const TS_IDENTITY_COOKIE_PREFIXES: &[&str] = &[COOKIE_TS_EIDS_R_PREFIX]; +``` + +After `strip_cookies`: + +```rust +/// Strips cookies matching any exact name or any name prefix from a `Cookie` +/// header value. +/// +/// Returns an empty string when nothing is left. +#[must_use] +pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str], prefixes: &[&str]) -> String { + cookie_header + .split(';') + .map(str::trim) + .filter(|pair| !pair.is_empty()) + .filter(|pair| { + let name = pair.split('=').next().unwrap_or_default().trim(); + !names.contains(&name) && !prefixes.iter().any(|prefix| name.starts_with(prefix)) + }) + .collect::>() + .join("; ") +} + +/// Strips matching cookies from every `Cookie` field in `headers`. +/// +/// All `Cookie` fields are read, matching pairs are removed, and the rest is +/// written back as one field. A field that is not valid UTF-8 is dropped +/// rather than forwarded, because it cannot be checked. The header is removed +/// when nothing is left. +pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str], prefixes: &[&str]) { + let retained = headers + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .map(|value| strip_cookie_pairs(value, names, prefixes)) + .filter(|value| !value.is_empty()) + .collect::>() + .join("; "); + + headers.remove(header::COOKIE); + if retained.is_empty() { + return; + } + match http::HeaderValue::from_str(&retained) { + Ok(value) => { + headers.insert(header::COOKIE, value); + } + Err(_) => log::debug!("Dropping Cookie header that is invalid after stripping"), + } +} +``` + +`unwrap_or_default()` on an `Option<&str>` is not `unwrap()`, and `split` always yields at least one item, so it is safe here. + +- [ ] **Step 4: Run the tests and see them pass** + +Run: `cargo test-fastly -- cookies::tests` +Expected: PASS, including the existing `strip_cookies` tests. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/constants.rs crates/trusted-server-core/src/cookies.rs +git commit --signoff -S -m "Add identity cookie lists and a name-and-prefix cookie strip" +``` + +### Task 2: Strip identity cookies from Prebid Server requests in every mode + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/routing.rs` (`PrebidTransportHeaders`, around line 205) +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` (`apply_prebid_header_values` and both callers, around lines 2025-2108) +- Test: `crates/trusted-server-core/src/integrations/prebid.rs` (`mod tests`) + +**Interfaces:** + +- Consumes: `strip_cookie_pairs`, `TS_IDENTITY_COOKIE_NAMES`, `TS_IDENTITY_COOKIE_PREFIXES`, `CONSENT_COOKIE_NAMES` (Task 1) +- Produces: `PrebidTransportHeaders::cookies(&self) -> &[HeaderValue]`, which replaces `cookie()`. `apply_prebid_header_values` takes `cookies: &[HeaderValue]` as its first argument. + +Today `apply_prebid_header_values` forwards the raw `Cookie` header unchanged whenever `consent_forwarding.strips_consent_cookies()` is false, and both callers pass only the first `Cookie` field. After this task, identity cookies are stripped in every mode, every field is read, and a non-UTF-8 field is dropped (spec 6.2, "Prebid Server"). + +- [ ] **Step 1: Write the failing tests** + +Find the existing tests of `apply_prebid_header_values` with `grep -n "apply_prebid_header_values\|strips_consent_cookies" crates/trusted-server-core/src/integrations/prebid.rs`, and add next to them. Build the target request the way those tests do. The `ConsentForwardingMode` variants are listed in `consent_config.rs:137-150`. Use the one that does **not** strip consent cookies for the first test and the one that does for the second. + +```rust + fn cookie_values(request: &http::Request) -> Vec { + request + .headers() + .get_all(header::COOKIE) + .iter() + .map(|value| value.to_str().expect("should be ASCII").to_owned()) + .collect() + } + + #[test] + fn prebid_headers_strip_identity_cookies_when_consent_cookies_are_forwarded() { + let cookies = [ + HeaderValue::from_static("ts-ec=abc; euconsent-v2=tcf; other=1"), + HeaderValue::from_static("__Host-ts-eids-s=v1.a.b; __Host-ts-eids-r-0a1b2c3d=v1.c.d"), + HeaderValue::from_static("ts-eids=W10; ts-ec-pull-complete=v1; second=2"), + ]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::CookiesAndBody, + None, + ); + + assert_eq!( + cookie_values(&to), + vec!["euconsent-v2=tcf; other=1; second=2".to_owned()], + "should drop identity cookies, keep consent and unrelated cookies, and read every field" + ); + } + + #[test] + fn prebid_headers_strip_identity_and_consent_cookies_in_openrtb_only_mode() { + let cookies = [HeaderValue::from_static("ts-ec=abc; euconsent-v2=tcf; other=1")]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::OpenrtbOnly, + None, + ); + + assert_eq!(cookie_values(&to), vec!["other=1".to_owned()], "should keep only other=1"); + } + + #[test] + fn prebid_headers_drop_non_utf8_cookie_fields_and_empty_results() { + let cookies = [ + HeaderValue::from_bytes(b"x=\xff").expect("should build an opaque header"), + HeaderValue::from_static("ts-ec=abc"), + ]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::CookiesAndBody, + None, + ); + + assert!(to.headers().get(header::COOKIE).is_none(), "should send no Cookie header"); + } +``` + +If the mode variants are named differently, use the real names and keep the meaning: one mode that forwards consent cookies and `OpenrtbOnly`. + +- [ ] **Step 2: Run the tests and see them fail** + +Run: `cargo test-fastly -- integrations::prebid::tests::prebid_headers_` +Expected: compile error, because the first argument is still `Option<&HeaderValue>`. + +- [ ] **Step 3: Implement** + +In `auction/routing.rs`, change the field and its accessor: + +```rust +pub(crate) struct PrebidTransportHeaders { + cookies: Vec, + user_agent: Option, + referer: Option, + accept_language: Option, +} + +impl PrebidTransportHeaders { + /// Every inbound `Cookie` field, in order. + pub(crate) fn cookies(&self) -> &[HeaderValue] { + &self.cookies + } + // user_agent, referer and accept_language unchanged +``` + +and in `snapshot`: + +```rust + cookies: request.headers().get_all(header::COOKIE).iter().cloned().collect(), +``` + +Fix every other use of `cookie:` or `.cookie()` in `routing.rs` that the compiler reports, including the tests at about lines 1300-1315 that append two `Cookie` fields. Those tests should now see both fields. + +In `integrations/prebid.rs`, change the import to `use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES, strip_cookie_pairs};`. Then replace the cookie half of `apply_prebid_header_values`: + +```rust +fn apply_prebid_header_values( + cookies: &[HeaderValue], + user_agent: Option<&HeaderValue>, + referer: Option<&HeaderValue>, + accept_language: Option<&HeaderValue>, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + // ... unchanged header loop and x-forwarded-for ... + + // Identity cookies are stripped in every mode, before the consent mode is + // checked, so no mode forwards the raw header (spec 6.2). + let consent_names: &[&str] = if consent_forwarding.strips_consent_cookies() { + CONSENT_COOKIE_NAMES + } else { + &[] + }; + let retained = cookies + .iter() + .filter_map(|value| value.to_str().ok()) + .map(|value| strip_cookie_pairs(value, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES)) + .map(|value| strip_cookie_pairs(&value, consent_names, &[])) + .filter(|value| !value.is_empty()) + .collect::>() + .join("; "); + if !retained.is_empty() + && let Ok(cookie_header) = HeaderValue::from_str(&retained) + { + to.headers_mut().insert(header::COOKIE, cookie_header); + } +} +``` + +Update both callers. `copy_request_headers`-style callers pass a local `let cookies: Vec = from.headers().get_all(header::COOKIE).iter().cloned().collect();` and then `&cookies`. `apply_prebid_transport_headers` passes `from.cookies()`. Remove `strip_cookies` from the import if nothing else uses it. + +- [ ] **Step 4: Run the tests and see them pass** + +Run: `cargo test-fastly -- integrations::prebid` and then `cargo test-fastly -- auction::routing` +Expected: PASS. If an existing test asserted that the raw header is forwarded unchanged, update its expectation to the stripped value and say why in the commit body. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/auction/routing.rs crates/trusted-server-core/src/integrations/prebid.rs +git commit --signoff -S -m "Strip identity cookies from Prebid Server requests in every mode" -m "Every Cookie field is now read and checked, and a field that is not valid UTF-8 is dropped instead of forwarded." +``` + +### Task 3: Strip the ID cookies from publisher-origin requests + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (`rewrite_origin_request`, around line 3124) +- Test: `crates/trusted-server-core/src/publisher.rs` (`mod tests`) + +**Interfaces:** + +- Consumes: `strip_cookie_headers`, `TS_ORIGIN_STRIPPED_COOKIE_NAMES`, `TS_IDENTITY_COOKIE_PREFIXES` (Task 1) + +- [ ] **Step 1: Write the failing test** + +Add to `publisher.rs` tests, next to any existing `rewrite_origin_request` test (`grep -n "fn .*rewrite_origin_request" crates/trusted-server-core/src/publisher.rs`): + +```rust + #[test] + fn rewrite_origin_request_strips_id_cookies_and_keeps_ts_ec() { + let mut req = Request::builder() + .uri("https://www.example.com/article") + .header(header::COOKIE, "ts-ec=abc; ts-eids=W10; session=1") + .header( + header::COOKIE, + "__Host-ts-eids-s=v1.a.b; __Host-ts-eids-r-0a1b2c3d=v1.c.d; ts-ec-pull-complete=v1", + ) + .body(EdgeBody::empty()) + .expect("should build request"); + + rewrite_origin_request( + &mut req, + "https://origin.example.com/article".parse().expect("should parse uri"), + "origin.example.com", + ) + .expect("should rewrite request"); + + let cookies: Vec<_> = req.headers().get_all(header::COOKIE).iter().collect(); + assert_eq!(cookies.len(), 1, "should send one Cookie header"); + assert_eq!( + cookies[0], "ts-ec=abc; session=1; ts-ec-pull-complete=v1", + "should keep ts-ec, the pull marker and unrelated cookies, and drop the ID cookies" + ); + } + + #[test] + fn rewrite_origin_request_drops_cookie_header_when_only_id_cookies() { + let mut req = Request::builder() + .uri("https://www.example.com/") + .header(header::COOKIE, "ts-eids=W10; __Host-ts-eids-s=v1.a.b") + .body(EdgeBody::empty()) + .expect("should build request"); + + rewrite_origin_request( + &mut req, + "https://origin.example.com/".parse().expect("should parse uri"), + "origin.example.com", + ) + .expect("should rewrite request"); + + assert!(req.headers().get(header::COOKIE).is_none(), "should drop the empty Cookie header"); + assert_eq!(req.headers()[header::HOST], "origin.example.com", "should still set Host"); + } +``` + +- [ ] **Step 2: Run and see it fail** + +Run: `cargo test-fastly -- publisher::tests::rewrite_origin_request_` +Expected: FAIL. The first assertion sees two `Cookie` fields that still hold `ts-eids`. + +- [ ] **Step 3: Implement** + +In `rewrite_origin_request`, after `req.headers_mut().remove("fastly-ssl");`: + +```rust + // The publisher origin keeps `ts-ec` but never receives the ID cookies, + // which can push its `Cookie` header past common 8 KB limits (spec 6.2). + strip_cookie_headers( + req.headers_mut(), + TS_ORIGIN_STRIPPED_COOKIE_NAMES, + TS_IDENTITY_COOKIE_PREFIXES, + ); +``` + +Add `use crate::cookies::{TS_IDENTITY_COOKIE_PREFIXES, TS_ORIGIN_STRIPPED_COOKIE_NAMES, strip_cookie_headers};` to the imports at the top of `publisher.rs`. + +- [ ] **Step 4: Run and see it pass, then run the other adapters** + +Run: `cargo test-fastly -- publisher::tests` then `cargo test-axum && cargo test-cloudflare && cargo test-spin` +Expected: PASS. The strip lives in core, so every adapter gets it (spec 11). If an adapter test asserted that `ts-eids` reaches the origin, update it and note the behaviour change in the commit body. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit --signoff -S -m "Strip ID cookies from publisher origin requests" -m "The origin keeps ts-ec and the pull-complete marker. ts-eids, __Host-ts-eids-s and the __Host-ts-eids-r- cookies no longer reach it. An origin that read ts-eids must read x-ts-eids instead." +``` + +### Task 4: Refuse the new cookies as template-cache keys, and document Part 0 + +**Files:** + +- Modify: `crates/trusted-server-core/src/cookies/template_cache_policy.rs` (around line 35) +- Modify: `docs/guide/edge-cookies.md`, `CHANGELOG.md` +- Test: `crates/trusted-server-core/src/cookies/template_cache_policy.rs` (`mod tests`, line 172) + +- [ ] **Step 1: Write the failing tests** + +Find the test that rejects `ts-eids` as a key (`grep -n "identity cookie" crates/trusted-server-core/src/cookies/template_cache_policy.rs`). It shows how the validating function is called. Add two cases next to it with the same call shape: + +```rust + #[test] + fn template_cache_key_rejects_sealed_identity_cookies() { + for name in ["__Host-ts-eids-s", "__Host-ts-eids-r-0a1b2c3d"] { + let error = validate_names_for_test(&[name.to_owned()], &[]) + .expect_err("should reject identity cookie as a cache key"); + assert!( + error.contains("must not key on Trusted Server identity cookie"), + "should explain the rejection for {name}, got {error}" + ); + } + validate_names_for_test(&["session".to_owned()], &[]) + .expect("should still accept an unrelated cookie"); + } +``` + +`validate_names_for_test` stands for the call the existing identity-cookie test makes. Use that exact call, not a new helper. + +- [ ] **Step 2: Run and see it fail** + +Run: `cargo test-fastly -- template_cache_policy::tests::template_cache_key_rejects_sealed` +Expected: FAIL, because both names are accepted today. + +- [ ] **Step 3: Implement** + +Replace the condition at line 35: + +```rust + if field == "template_cache_key_cookies" + && ([COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S, COOKIE_SHAREDID] + .contains(&name.as_str()) + || name.starts_with(COOKIE_TS_EIDS_R_PREFIX)) + { +``` + +and extend the `crate::constants` import with `COOKIE_TS_EIDS_R_PREFIX` and `COOKIE_TS_EIDS_S`. + +- [ ] **Step 4: Run and see it pass** + +Run: `cargo test-fastly -- template_cache_policy` +Expected: PASS. + +- [ ] **Step 5: Docs** + +In `docs/guide/edge-cookies.md`, in the section that lists TS cookies, add a short "Cookies forwarded to the origin" paragraph. Say that `ts-ec` and `ts-ec-pull-complete` still reach the publisher origin, that `ts-eids` and the server-owned `__Host-ts-eids-s` and `__Host-ts-eids-r-*` cookies do not, and that origin code that read `ts-eids` should read the `x-ts-eids` response header instead. Also say that Prebid Server never receives any TS identity cookie. + +In `CHANGELOG.md` under `## [Unreleased]` / `### Changed`, add: + +```markdown +- **Breaking:** The publisher origin no longer receives the `ts-eids` cookie, and Prebid Server no longer receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, `ts-ec-pull-complete`, and the server-owned `__Host-ts-eids-s` and `__Host-ts-eids-r-*` cookies that a later release introduces). The origin still receives `ts-ec`. Origin code that read `ts-eids` should read the `x-ts-eids` response header instead. This release is the rollback floor for the server-side identity provider work. +``` + +Run: `cd docs && npm run format && cd ..` and `docs/node_modules/.bin/prettier --config docs/.prettierrc --write CHANGELOG.md` + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/cookies/template_cache_policy.rs docs/guide/edge-cookies.md CHANGELOG.md +git commit --signoff -S -m "Refuse the sealed identity cookies as template cache keys" +``` + +- [ ] **Step 7: Part 0 gate and PR** + +Run `/check-ci`. Every gate must pass. Open the PR against `spec/1246-server-side-identity-providers` with the `pr-creator` agent. Title: "Strip identity cookies from origin and Prebid Server requests". The description must say this is spec step 0 and the rollback floor for the later parts, and must name the origin `ts-eids` behaviour change. + +--- + +# Part 1: One write per page load, no new vendors (spec 14, step 1) + +Part 1 builds the trait, descriptor, harness and coordinator, moves every browser-driven KV write into the writer request's single flush, adds `__Host-ts-eids-s` and `__Host-ts-eids-r-*`, and replaces pull sync with `GenericPullProvider`. No vendor provider is registered yet, and no provider runs in hold mode or captures. Those come in Part 2. + +Before you start: `git switch feat/1246-identity-cookie-strips && git switch -c feat/1246-identity-single-write`. + +Tasks 5 to 22 build pure, unit-tested modules with no request wiring. Tasks 23 to 29 wire them into requests. Task 30 proves the one-write invariant, and Task 31 ships. + +### Task 5: The `ec::identity` module, its ID newtypes and its error type + +**Files:** + +- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, alphabetical), `crates/trusted-server-core/Cargo.toml` +- Create: `crates/trusted-server-core/src/ec/identity/mod.rs` +- Create: `crates/trusted-server-core/src/ec/identity/ids.rs` +- Create: `crates/trusted-server-core/src/ec/identity/resolution.rs` (error type only in this task, extended in Task 10) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (module list, around line 32) +- Modify: `crates/trusted-server-core/src/migration_guards.rs` (`checked_sources`) + +**Interfaces:** + +- Produces: + - `IdentityProviderId`: `FromStr>`, `as_str(&self) -> &str`, serde as a string. Derives `Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display`. + - `SourceDomain`: `FromStr` through `normalize_partner_source_domain`, `as_str`, `matches(&self, name: &str) -> bool`, serde as a string. Same derives. + - `UnixTime`: `from_secs(u64)`, `from_millis(u64)`, `as_secs(self) -> u64`, `Add`. Derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord`. + - `IdentityProviderError` (all seven variants from spec 4.5) with `config`, `rejected`, `malformed` and `not_implemented` constructors. `Access` and `Input` are referenced by two variants, so this task adds a minimal `descriptor.rs` with just those two enums. Task 8 fills in the rest of the file. + +- [ ] **Step 1: Add the dependency** + +In the workspace `Cargo.toml`, under `[workspace.dependencies]` in alphabetical order: + +```toml +aes-gcm = "0.10" +``` + +In `crates/trusted-server-core/Cargo.toml` `[dependencies]`, alphabetical: + +```toml +aes-gcm = { workspace = true } +``` + +Run: `cargo check-fastly && cargo check-cloudflare` +Expected: both build. `aes-gcm` 0.10 depends on `getrandom` only through its optional `getrandom` feature, which stays off. Nonces come from `rand::thread_rng` (Task 16). If `cargo check-cloudflare` fails on `getrandom`, set `default-features = false, features = ["aes", "alloc"]` in the workspace entry and run it again. + +- [ ] **Step 2: Write the failing tests** + +Create `ec/identity/ids.rs` with only this test module for now: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provider_id_accepts_the_documented_grammar() { + for id in ["id5", "pull-ssp-example-com", "lockr_aim", "a"] { + id.parse::() + .unwrap_or_else(|_| panic!("should accept `{id}`")); + } + let longest = format!("a{}", "b".repeat(62)); + longest.parse::().expect("should accept 63 characters"); + } + + #[test] + fn provider_id_rejects_everything_else() { + let too_long = format!("a{}", "b".repeat(63)); + for id in ["", "1id", "Id5", "id 5", "id.5", too_long.as_str()] { + let error = id.parse::().expect_err("should reject the id"); + assert!( + matches!(error.current_context(), IdentityProviderError::Config { .. }), + "should be a Config error for `{id}`" + ); + } + } + + #[test] + fn source_domain_normalizes_and_matches() { + let source: SourceDomain = "ID5-Sync.com.".parse().expect("should parse"); + assert_eq!(source.as_str(), "id5-sync.com", "should lower-case and drop the trailing dot"); + assert!(source.matches("id5-sync.COM"), "should match another spelling"); + assert!(!source.matches("example.com"), "should not match another domain"); + assert!(!source.matches("https://id5-sync.com"), "should not match a URL"); + } + + #[test] + fn source_domain_round_trips_through_serde() { + let source: SourceDomain = "example.com".parse().expect("should parse"); + let json = serde_json::to_string(&source).expect("should serialize"); + assert_eq!(json, "\"example.com\"", "should serialize as a bare string"); + let back: SourceDomain = serde_json::from_str(&json).expect("should deserialize"); + assert_eq!(back, source, "should round-trip"); + serde_json::from_str::("\"not a domain\"") + .expect_err("should reject an invalid domain on deserialize"); + } + + #[test] + fn unix_time_converts_millis_and_adds_durations() { + assert_eq!(UnixTime::from_millis(1_760_000_000_999).as_secs(), 1_760_000_000, "should floor millis"); + let later = UnixTime::from_secs(100) + Duration::from_secs(50); + assert_eq!(later, UnixTime::from_secs(150), "should add seconds"); + let saturated = UnixTime::from_secs(u64::MAX) + Duration::from_secs(1); + assert_eq!(saturated, UnixTime::from_secs(u64::MAX), "should saturate"); + } +} +``` + +Create `ec/identity/mod.rs`: + +```rust +//! Server-side identity providers. +//! +//! An [`IdentityProvider`](provider::IdentityProvider) turns request signals +//! into OpenRTB EIDs. Providers only build requests and parse buffered +//! responses. The coordinator owns eligibility, consent, transport limits and +//! KV write-back, so every page load makes at most one browser-driven KV +//! write. See `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. + +pub mod descriptor; +pub mod ids; +pub mod resolution; + +pub use descriptor::{Access, Input}; +pub use ids::{IdentityProviderId, SourceDomain, UnixTime}; +pub use resolution::IdentityProviderError; +``` + +Add `pub mod identity;` to the module list in `ec/mod.rs`, in alphabetical order after `pub mod identify;`. + +- [ ] **Step 3: Run and see the tests fail** + +Run: `cargo test-fastly -- ec::identity::ids` +Expected: compile errors, because the types do not exist. + +- [ ] **Step 4: Implement `resolution.rs` (error only) and the minimal `descriptor.rs`** + +`ec/identity/descriptor.rs` for now: + +```rust +//! What a provider is, may write, needs and must respect. + +/// An input a provider can ask for. +/// +/// Consent strings, the provider's stored IDs, the publisher domain, the +/// time and the declared endpoint are always available. Private state is +/// available to providers declared `with_private_state()`. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, +)] +#[non_exhaustive] +pub enum Input { + /// The platform's client address. + #[display("client_ip")] + ClientIp, + /// The request's `User-Agent`. + #[display("user_agent")] + UserAgent, + /// The top-level page URL without query string or fragment. + #[display("page_url")] + PageUrl, + /// Country and subdivision only. + #[display("region")] + Region, + /// A SHA-256 of a normalized email address. Never available in v1. + #[display("hashed_email")] + HashedEmail, +} + +/// Something a provider read without declaring it. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, +)] +#[non_exhaustive] +pub enum Access { + /// An undeclared [`Input`]. + #[display("{_0}")] + Input(Input), + /// The endpoint, when the descriptor declares no `resolves`. + #[display("endpoint")] + Endpoint, + /// Private state, without `with_private_state()`. + #[display("private_state")] + PrivateState, +} +``` + +`ec/identity/resolution.rs` for now: + +```rust +//! Provider results and errors. + +use error_stack::Report; +use http::StatusCode; + +use super::descriptor::{Access, Input}; + +/// Marker message of [`IdentityProviderError::not_implemented`]. The harness +/// matches on it to tell a missing override from a real failure. +pub(crate) const NOT_IMPLEMENTED_MARKER: &str = "capability declared but not implemented: "; + +/// A provider failure. Built through the constructors below. +#[derive(Debug, derive_more::Display)] +#[non_exhaustive] +pub enum IdentityProviderError { + /// Configuration or provider code makes the call impossible. + #[display("identity provider configuration error: {message}")] + Config { + /// What is wrong. Never contains a secret, UID or URL query. + message: String, + }, + /// The provider read something its descriptor did not declare. + #[display("`{access}` was read but not declared")] + UndeclaredInput { + /// What was read. + access: Access, + }, + /// A declared input is not available for this request. + #[display("input `{input}` is unavailable for this request")] + InputUnavailable { + /// The missing input. + input: Input, + }, + /// The provider refused the request. + #[display("provider rejected the request with status {status}")] + Rejected { + /// The response status. + status: StatusCode, + }, + /// The provider's response could not be parsed. + #[display("provider response could not be parsed")] + Malformed, + /// The stored private state does not decode as the provider's type. + #[display("stored provider state could not be decoded")] + InvalidState, + /// Set by the coordinator only. Providers never see transport failures. + #[display("provider request failed in transport")] + Transport, +} + +impl core::error::Error for IdentityProviderError {} + +impl IdentityProviderError { + /// A [`Self::Config`] error. + #[must_use] + pub fn config(message: impl Into) -> Report { + Report::new(Self::Config { message: message.into() }) + } + + /// A [`Self::Rejected`] error. + #[must_use] + pub fn rejected(status: StatusCode) -> Report { + Report::new(Self::Rejected { status }) + } + + /// A [`Self::Malformed`] error. + #[must_use] + pub fn malformed() -> Report { + Report::new(Self::Malformed) + } + + /// The default body of a capability method: a `Config` error with a + /// fixed marker message, so the harness can tell a missing override from + /// a real failure. + #[must_use] + pub fn not_implemented(method: &'static str) -> Report { + Self::config(format!("{NOT_IMPLEMENTED_MARKER}{method}")) + } + + /// Whether this is the [`Self::not_implemented`] marker. + #[must_use] + pub(crate) fn is_not_implemented(&self) -> bool { + matches!(self, Self::Config { message } if message.starts_with(NOT_IMPLEMENTED_MARKER)) + } +} +``` + +- [ ] **Step 5: Implement `ids.rs`** + +Put this above the test module: + +```rust +//! Identifiers shared by the identity provider API. + +use core::ops::Add; +use core::str::FromStr; +use std::time::Duration; + +use error_stack::Report; +use serde::{Deserialize, Serialize}; + +use super::resolution::IdentityProviderError; +use crate::ec::partner::normalize_partner_source_domain; + +/// An identity provider's ID, such as `id5` or `pull-ssp-example-com`. +/// +/// Matches `^[a-z][a-z0-9_-]{0,62}$`, so it can be a bare TOML key and can +/// equal an integration ID that uses `_`. Separate from +/// [`crate::auction::ProviderId`]. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, Serialize, Deserialize, +)] +#[serde(try_from = "String", into = "String")] +pub struct IdentityProviderId(String); + +impl IdentityProviderId { + /// The ID as written in config. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for IdentityProviderId { + type Err = Report; + + fn from_str(value: &str) -> Result { + let mut bytes = value.bytes(); + let valid_first = bytes.next().is_some_and(|byte| byte.is_ascii_lowercase()); + let valid_rest = bytes.all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' || byte == b'-' + }); + if valid_first && valid_rest && value.len() <= 63 { + Ok(Self(value.to_owned())) + } else { + Err(IdentityProviderError::config(format!( + "identity provider id `{value}` must match ^[a-z][a-z0-9_-]{{0,62}}$" + ))) + } + } +} + +impl TryFrom for IdentityProviderId { + type Error = Report; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(id: IdentityProviderId) -> Self { + id.0 + } +} + +/// An EID `source` domain, normalized. Also the key of the KV `ids` map. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, Serialize, Deserialize, +)] +#[serde(try_from = "String", into = "String")] +pub struct SourceDomain(String); + +impl SourceDomain { + /// The normalized domain. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Whether a vendor's spelling of a source names this one, after + /// normalization. + #[must_use] + pub fn matches(&self, name: &str) -> bool { + normalize_partner_source_domain(name).is_ok_and(|normalized| normalized == self.0) + } +} + +impl FromStr for SourceDomain { + type Err = Report; + + fn from_str(value: &str) -> Result { + normalize_partner_source_domain(value) + .map(Self) + .map_err(IdentityProviderError::config) + } +} + +impl TryFrom for SourceDomain { + type Error = Report; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(source: SourceDomain) -> Self { + source.0 + } +} + +/// Unix seconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct UnixTime(u64); + +impl UnixTime { + /// From Unix seconds. + #[must_use] + pub fn from_secs(secs: u64) -> Self { + Self(secs) + } + + /// From Unix milliseconds, for vendors such as Lockr and UID2. Floors to + /// whole seconds. + #[must_use] + pub fn from_millis(millis: u64) -> Self { + Self(millis / 1000) + } + + /// As Unix seconds. + #[must_use] + pub fn as_secs(self) -> u64 { + self.0 + } +} + +impl Add for UnixTime { + type Output = UnixTime; + + fn add(self, rhs: Duration) -> UnixTime { + UnixTime(self.0.saturating_add(rhs.as_secs())) + } +} +``` + +serde's `try_from` needs the error to implement `Display`, and `Report` does. Check that `normalize_partner_source_domain` rejects `"not a domain"` (spaces fail the label check). If it does not, use `"not/a/domain"` in the test. + +Add both new files to `checked_sources()` in `migration_guards.rs`, in alphabetical order, following the `ec/client_eids.rs` entry: + +```rust + ("ec/identity/descriptor.rs", include_str!("ec/identity/descriptor.rs")), + ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), + ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), + ("ec/identity/resolution.rs", include_str!("ec/identity/resolution.rs")), +``` + +Every later task that creates a file under `ec/identity/` adds it to this list the same way. + +- [ ] **Step 6: Run and see the tests pass** + +Run: `cargo test-fastly -- ec::identity` and `cargo test-fastly -- migration_guards` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/trusted-server-core/Cargo.toml crates/trusted-server-core/src/ec/mod.rs crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add the identity module with provider and source ID newtypes" -m "Adds aes-gcm, which the sealed identity cookies use." +``` + +### Task 6: KV schema: record metadata, provider state, `Default` and constructors + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/kv_types.rs` (`KvEntry`, `KvConsent`, `KvGeo`, `KvPartnerId`, `validate`, constructors) +- Modify: every file with a `KvPartnerId {` literal (find them with `grep -rn "KvPartnerId {" crates/`) +- Modify: `crates/trusted-server-core/src/ec/admin.rs` (raw lookup response, around `admin.rs:394`) +- Test: `crates/trusted-server-core/src/ec/kv_types.rs`, `crates/trusted-server-core/src/ec/admin.rs` + +**Interfaces:** + +- Produces: + - `KvPartnerId { uid, w: Option, rv: Option, at: Option, exp: Option, ra: Option, ru: Option, atype: Option, ext: Option }`, deriving `Default`, with `KvPartnerId::new(uid: impl Into) -> Self` and `KvPartnerId::browser(uid: impl Into, acquired_at: Option) -> Self` + - `KvProviderState { n: Option, f: u32, o: Option, oc: Option, iv: Vec, st: Option }`, deriving `Default` + - `KvOptOut { at: u64, fp: String }` + - `KvEntry.ps: BTreeMap`, and `KvEntry` derives `Default` + - Constants `MAX_PARTNER_EXT_BYTES = 256`, `MAX_PROVIDER_STATE_BYTES = 1024`, `MAX_PROVIDER_STATES = 32`, `MAX_INVALIDATED_UIDS = 4`, `RECORD_REVISION_HEX_LEN = 16` + +- [ ] **Step 1: Write the failing tests** + +Add to `kv_types.rs` tests: + +```rust + #[test] + fn legacy_partner_record_reads_and_writes_unchanged() { + let legacy = json!({ "uid": "example-uid" }); + let record: KvPartnerId = serde_json::from_value(legacy.clone()).expect("should read legacy"); + assert_eq!(record, KvPartnerId::new("example-uid"), "should fill every new field with None"); + assert_eq!(serde_json::to_value(&record).expect("should write"), legacy, "should add no fields"); + } + + #[test] + fn provider_record_and_state_round_trip() { + let stored = json!({ + "v": 1, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { + "id5-sync.com": { + "uid": "ID5*example", + "w": "id5", + "rv": "9c1d04b7e6a83f2a", + "at": 1_760_000_000, + "exp": 1_767_776_000, + "ra": 1_760_086_400, + "ru": 1_770_000_000, + "atype": 1, + "ext": { "linkType": 2 } + } + }, + "ps": { + "id5": { + "n": 1_760_003_600, + "f": 1, + "o": { "at": 1_759_000_000, "fp": "0123456789abcdef" }, + "oc": 1_759_500_000, + "iv": ["aaaaaaaaaaaaaaaa"], + "st": { "signature": "example-opaque-state" } + } + } + }); + + let entry: KvEntry = serde_json::from_value(stored.clone()).expect("should read"); + entry.validate().expect("should be valid"); + assert_eq!(serde_json::to_value(&entry).expect("should write"), stored, "should round-trip exactly"); + } + + #[test] + fn validate_rejects_oversized_ext_state_and_too_many_providers() { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ids.get_mut("example.com").expect("should exist").ext = + Some(json!({ "pad": "x".repeat(MAX_PARTNER_EXT_BYTES) })); + assert!(entry.validate().expect_err("should reject ext").contains("ext"), "should name ext"); + + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ps.insert( + "id5".to_owned(), + KvProviderState { st: Some(json!("x".repeat(MAX_PROVIDER_STATE_BYTES))), ..KvProviderState::default() }, + ); + assert!(entry.validate().expect_err("should reject st").contains("st"), "should name st"); + + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + for index in 0..=MAX_PROVIDER_STATES { + entry.ps.insert(format!("p{index}"), KvProviderState::default()); + } + assert!(entry.validate().expect_err("should reject ps size").contains("ps"), "should name ps"); + + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ps.insert( + "id5".to_owned(), + KvProviderState { iv: vec!["a".repeat(16); MAX_INVALIDATED_UIDS + 1], ..KvProviderState::default() }, + ); + assert!(entry.validate().expect_err("should reject iv").contains("iv"), "should name iv"); + + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ids.get_mut("example.com").expect("should exist").rv = Some("not-hex".to_owned()); + assert!(entry.validate().expect_err("should reject rv").contains("rv"), "should name rv"); + + KvEntry::minimal("example.com", "example-uid", 1).validate().expect("control entry should be valid"); + } +``` + +The validation test builds `KvProviderState` with `..KvProviderState::default()` because it tests one field at a time. It is not an exhaustiveness test. + +Add to `admin.rs` tests, next to the existing raw-lookup test (`grep -n "lookup_raw\|fn .*lookup" crates/trusted-server-core/src/ec/admin.rs`). Use the same setup that test uses to put an entry in the in-memory graph and call the handler: + +```rust + #[test] + fn admin_lookup_redacts_provider_state() { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1_760_000_000); + entry.ps.insert( + "id5".to_owned(), + KvProviderState { n: Some(5), st: Some(json!({ "signature": "secret-state" })), ..KvProviderState::default() }, + ); + let body = admin_lookup_body_for_test(&entry); + + assert_eq!(body["ps"]["id5"]["st"], json!("[redacted]"), "should redact the state"); + assert_eq!(body["ps"]["id5"]["n"], json!(5), "should keep the rest of ps"); + assert_eq!(body["ids"]["example.com"]["uid"], json!("example-uid"), "should keep the ids"); + } +``` + +`admin_lookup_body_for_test` stands for whatever the existing raw-lookup test does to get the JSON body: insert the row, call the handler and parse the body. Write it as a small helper in the test module that reuses that test's setup. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::kv_types::tests` and `cargo test-fastly -- ec::admin::tests::admin_lookup_redacts` +Expected: compile errors (`ps`, `KvProviderState` and `KvPartnerId::new` do not exist). + +- [ ] **Step 3: Implement the schema** + +In `kv_types.rs`, add the constants after `MAX_UID_LENGTH`: + +```rust +/// Maximum serialized bytes of a stored UID-level `ext`. +pub const MAX_PARTNER_EXT_BYTES: usize = 256; +/// Maximum serialized bytes of one provider's private state (`ps.st`). +pub const MAX_PROVIDER_STATE_BYTES: usize = 1024; +/// Maximum number of providers with a `ps` entry in one row. +pub const MAX_PROVIDER_STATES: usize = 32; +/// Maximum number of invalidated-UID hashes kept per provider (`ps.iv`). +pub const MAX_INVALIDATED_UIDS: usize = 4; +/// Length of a record revision (`rv`) in lowercase hex digits. +pub const RECORD_REVISION_HEX_LEN: usize = 16; +``` + +Add `Default` to the derives of `KvEntry`, `KvConsent` and `KvGeo`. Add the `ps` field at the end of `KvEntry`: + +```rust + /// Per-provider scheduling, opt-out and private state, keyed by + /// identity provider ID. Never sent to bidders or cookies. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub ps: BTreeMap, +``` + +Replace `KvPartnerId`: + +```rust +/// A partner user ID within a KV entry. +/// +/// Every field but `uid` is optional and omitted when absent, so legacy rows +/// read and write unchanged and `SCHEMA_VERSION` stays 1 (spec 6.1). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvPartnerId { + /// The partner's user identifier. + pub uid: String, + /// Writer: an identity provider ID. Absent for browser ingest and legacy rows. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub w: Option, + /// Record revision: 16 lowercase hex digits, random on every write. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rv: Option, + /// Unix seconds when the ID was acquired or last refreshed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at: Option, + /// Unix seconds after which the ID must not be sent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exp: Option, + /// Unix seconds after which the ID is due for refresh. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ra: Option, + /// Unix seconds after which the ID can no longer be refreshed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ru: Option, + /// Provider-supplied `atype`. Absent: use the partner's `openrtb_atype`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub atype: Option, + /// UID-level `ext`, at most [`MAX_PARTNER_EXT_BYTES`] serialized bytes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +impl KvPartnerId { + /// A bare record holding only a UID. + #[must_use] + pub fn new(uid: impl Into) -> Self { + Self { uid: uid.into(), ..Self::default() } + } + + /// A browser-ingested record: no writer, with its acquisition time. + #[must_use] + pub fn browser(uid: impl Into, acquired_at: Option) -> Self { + Self { uid: uid.into(), at: acquired_at, ..Self::default() } + } +} + +/// A provider's sticky opt-out (spec 8). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct KvOptOut { + /// Unix seconds when the opt-out was reported. + pub at: u64, + /// Consent fingerprint at that time: 16 lowercase hex digits. + pub fp: String, +} + +/// Per-provider state in a KV row (`ps.`). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvProviderState { + /// Unix seconds before which the provider must not be called. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub n: Option, + /// Consecutive failures, for backoff. + #[serde(default, skip_serializing_if = "is_zero")] + pub f: u32, + /// Sticky opt-out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub o: Option, + /// When the provider's opt-out was last cleared. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oc: Option, + /// Keyed hashes of UIDs the owner invalidated, newest last. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub iv: Vec, + /// The provider's private state as embedded JSON. KV only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub st: Option, +} + +fn is_zero(value: &u32) -> bool { + *value == 0 +} +``` + +The listed constructors (`KvEntry::new`, `minimal`, `tombstone`) gain `ps: BTreeMap::new()`. `minimal` uses `KvPartnerId::new(uid)`. + +Extend `validate()`. Inside the existing `for (partner_id, partner_uid) in &self.ids` loop, after the UID length check: + +```rust + if let Some(ext) = &partner_uid.ext + && serde_json::to_vec(ext).map_or(true, |bytes| bytes.len() > MAX_PARTNER_EXT_BYTES) + { + return Err(format!("partner ID '{partner_id}' ext exceeds {MAX_PARTNER_EXT_BYTES} bytes")); + } + if let Some(rv) = &partner_uid.rv + && (rv.len() != RECORD_REVISION_HEX_LEN + || !rv.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))) + { + return Err(format!("partner ID '{partner_id}' rv is not {RECORD_REVISION_HEX_LEN} hex digits")); + } +``` + +After the loop: + +```rust + if self.ps.len() > MAX_PROVIDER_STATES { + return Err(format!("ps exceeds MAX_PROVIDER_STATES ({})", self.ps.len())); + } + for (provider, state) in &self.ps { + if state.iv.len() > MAX_INVALIDATED_UIDS { + return Err(format!("ps '{provider}' iv exceeds {MAX_INVALIDATED_UIDS} hashes")); + } + if let Some(st) = &state.st + && serde_json::to_vec(st).map_or(true, |bytes| bytes.len() > MAX_PROVIDER_STATE_BYTES) + { + return Err(format!("ps '{provider}' st exceeds {MAX_PROVIDER_STATE_BYTES} bytes")); + } + } +``` + +- [ ] **Step 4: Move every literal to the constructor** + +Run `grep -rn "KvPartnerId {" crates/` and change each `KvPartnerId { uid: X }` to `KvPartnerId::new(X)`. That covers about 30 sites in 8 files, including `ec/kv.rs` (`apply_cookie_partner_id_updates`, `apply_partner_id_updates` and others), the batch sync and pull sync writers, the admin tests and `kv_types.rs`. Run `grep -rn "KvEntry {" crates/` too, and add `ps: BTreeMap::new()` to any full literal the compiler flags. + +- [ ] **Step 5: Redact `ps.st` in the admin lookup** + +In `ec/admin.rs`, find where the raw lookup body is returned verbatim (`grep -n "lookup_raw" crates/trusted-server-core/src/ec/admin.rs`). Before it is written to the response, parse it, redact and re-serialize: + +```rust +/// Replaces every `ps..st` with a marker. Provider state is never +/// returned by the admin lookup (spec 6.1, 9.3). +fn redact_provider_state(body: &[u8]) -> Vec { + let Ok(mut value) = serde_json::from_slice::(body) else { + return body.to_vec(); + }; + if let Some(states) = value.get_mut("ps").and_then(serde_json::Value::as_object_mut) { + for state in states.values_mut() { + if let Some(st) = state.get_mut("st") { + *st = serde_json::Value::String("[redacted]".to_owned()); + } + } + } + serde_json::to_vec(&value).unwrap_or_else(|_| body.to_vec()) +} +``` + +Call it on `lookup.body` where the handler builds its response. A body that is not JSON is returned unchanged, as today, because the lookup exists to inspect corrupt rows. + +- [ ] **Step 6: Run and see the tests pass** + +Run: `cargo test-fastly -- ec::` and then `cargo clippy-fastly` +Expected: PASS. Every existing KV, batch sync and pull sync test must still pass unchanged. + +- [ ] **Step 7: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Add provider metadata and provider state to the KV schema" -m "New fields are optional and omitted when absent, so legacy rows read and write unchanged and the schema version stays 1. KvPartnerId and KvEntry derive Default, and literals move to constructors. The admin lookup redacts provider state." +``` + +### Task 7: Browser ingest records when it acquired an ID + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (`PartnerIdUpdate`, `apply_cookie_partner_id_updates`, `apply_partner_id_updates`) +- Modify: `crates/trusted-server-core/src/ec/prebid_eids.rs` (`collect_eid_updates`, `collect_sharedid_update`, `collect_prebid_eid_updates_from_eids`) +- Modify: every `PartnerIdUpdate::new` caller the compiler reports +- Test: `crates/trusted-server-core/src/ec/kv.rs` + +**Interfaces:** + +- Produces: `PartnerIdUpdate { partner_id, uid, acquired_at: Option }`, `PartnerIdUpdate::new(partner_id, uid)` (`acquired_at = None`), `PartnerIdUpdate::with_acquired_at(self, at: u64) -> Self`. A record that browser ingest inserts now carries `at`. + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn browser_ingest_records_acquisition_time() { + let mut entry = KvEntry::minimal("sharedid.org", "existing", 1); + let updates = [PartnerIdUpdate::new("example.com", "new-uid").with_acquired_at(1_760_000_000)]; + + let changed = apply_partner_id_updates(&mut entry, &updates); + + assert!(changed, "should report a change"); + assert_eq!( + entry.ids["example.com"], + KvPartnerId::browser("new-uid", Some(1_760_000_000)), + "should store the UID with at and no writer" + ); + assert_eq!(entry.ids["sharedid.org"], KvPartnerId::new("existing"), "should leave other sources alone"); + } +``` + +- [ ] **Step 2: Run and see it fail** + +Run: `cargo test-fastly -- ec::kv::tests::browser_ingest_records_acquisition_time` +Expected: compile error, because `with_acquired_at` does not exist. + +- [ ] **Step 3: Implement** + +Add the field and builder: + +```rust +pub(crate) struct PartnerIdUpdate { + pub(crate) partner_id: String, + pub(crate) uid: String, + /// Unix seconds this request saw the ID. Stored as the record's `at`. + pub(crate) acquired_at: Option, +} + +impl PartnerIdUpdate { + pub(crate) fn new(partner_id: impl Into, uid: impl Into) -> Self { + Self { partner_id: partner_id.into(), uid: uid.into(), acquired_at: None } + } + + /// Records when this request saw the ID. + #[must_use] + pub(crate) fn with_acquired_at(mut self, at: u64) -> Self { + self.acquired_at = Some(at); + self + } +} +``` + +In both `apply_*` functions, the de-duplicating map now keeps the whole update: `BTreeMap<&str, &PartnerIdUpdate>`. Insert `KvPartnerId::browser(update.uid.clone(), update.acquired_at)`. Equality checks still compare only `uid`, so an unchanged UID is not rewritten just to bump `at`. + +In `prebid_eids.rs`, give `collect_eid_updates` a `now: Option` parameter. Call `.with_acquired_at(now)` on each update when it is `Some`, and pass `checked_current_timestamp()` from the callers in `finalize.rs`. Leave batch sync and pull sync without `acquired_at`. They get it through the mutation in Tasks 15 and 22. + +- [ ] **Step 4: Run and see it pass** + +Run: `cargo test-fastly -- ec::` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/trusted-server-core/src/ec +git commit --signoff -S -m "Record when browser ingest acquired each ID" +``` + +### Task 8: The provider descriptor + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/descriptor.rs` (Task 5 created it with `Input` and `Access`) +- Modify: `crates/trusted-server-core/src/ec/identity/mod.rs` (re-exports) +- Test: `crates/trusted-server-core/src/ec/identity/descriptor.rs` + +**Interfaces:** + +- Consumes: `IdentityProviderId`, `SourceDomain`, `IdentityProviderError` (Task 5) +- Produces, exactly as spec 4.3 lists them: + - `ProviderDescriptor` and its getters: `builder`, `id`, `sources`, `primary_source`, `source_named`, `resolves`, `endpoint`, `captures`, `consent_policy`, `provenance`, `is_expiring` + - `ProviderDescriptorBuilder`: `with_source`, `with_resolves`, `with_captures`, `with_consent_policy`, `with_expiring_ids`, `with_expiring_source`, `with_provenance`, `build` + - `Resolves`: `new`, `with_endpoint(&str)`, `with_host(&str)`, `with_needs`, `with_needs_for`, `with_wants`, `with_kinds`, `with_private_state`, plus `pub(in crate::ec::identity) fn with_ec_id(self)` and crate-visible getters `hosts()`, `needs(kind)`, `wants()`, `kinds()`, `has_private_state()`, `gets_ec_id()` + - `Captures`: `new`, `with_path(Method, &str)`, `with_path_prefix(&str)`, `with_request_consent(pointers)`, plus crate-visible `matches(&Method, &str) -> bool`, `paths()`, `prefixes()`, `consent_pointers()` + - `EidProvenance`: `new`, `with_inserter`, `with_matcher`, `with_mm`, `inserter`, `matcher`, `mm` + - `ConsentPolicy`: `strict`, `with_tcf_purposes`, `with_tcf_vendor`, `allowing_dnt`, `allowing_gpc`, `allowing_us_opt_out`, plus `pub(in crate::ec::identity) fn eid_persistence_only()` for generic pull and crate-visible getters + - `UsOptOut { Sale, Sharing, TargetedAdvertising }` + - `ResolveKind { Initial, Refresh }` lives here rather than in `input.rs` because `Resolves` keys on it. `input.rs` re-exports it. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use http::Method; + + use super::*; + + fn source(name: &str) -> SourceDomain { + name.parse().expect("should parse source") + } + + fn id(name: &str) -> IdentityProviderId { + name.parse().expect("should parse id") + } + + #[test] + fn builds_the_documented_example() { + let descriptor = ProviderDescriptor::builder(id("exampleid")) + .with_source(source("exampleid.example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://api.exampleid.example.com/v1/fetch") + .with_needs([Input::ClientIp, Input::UserAgent]) + .with_wants([Input::PageUrl]), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(9999)) + .build() + .expect("should build"); + + assert_eq!(descriptor.id().as_str(), "exampleid", "should keep the id"); + assert_eq!(descriptor.primary_source(), &source("exampleid.example.com"), "should keep the source"); + assert_eq!( + descriptor.endpoint().map(Url::as_str), + Some("https://api.exampleid.example.com/v1/fetch"), + "should parse the endpoint" + ); + let resolves = descriptor.resolves().expect("should resolve"); + assert_eq!( + resolves.hosts(), + &["api.exampleid.example.com".to_owned()], + "should allow the endpoint host" + ); + assert_eq!( + resolves.kinds(), + &[ResolveKind::Initial, ResolveKind::Refresh], + "should default to Initial and Refresh" + ); + assert!(descriptor.captures().is_none(), "should not capture"); + assert!(!descriptor.is_expiring(&source("exampleid.example.com")), "should not expire by default"); + assert_eq!(descriptor.consent_policy().tcf_vendor(), Some(9999), "should keep the vendor"); + assert_eq!( + descriptor.consent_policy().tcf_purposes().iter().copied().collect::>(), + vec![1, 4], + "strict should require Purposes 1 and 4" + ); + } + + #[test] + fn build_rejects_invalid_descriptors() { + let cases: Vec<(&str, ProviderDescriptorBuilder)> = vec![ + ("no source", ProviderDescriptor::builder(id("p")).with_resolves(Resolves::new().with_endpoint("https://a.example.com"))), + ("no capability", ProviderDescriptor::builder(id("p")).with_source(source("example.com"))), + ("no endpoint or host", ProviderDescriptor::builder(id("p")).with_source(source("example.com")).with_resolves(Resolves::new())), + ("http endpoint", ProviderDescriptor::builder(id("p")).with_source(source("example.com")).with_resolves(Resolves::new().with_endpoint("http://a.example.com"))), + ("bad host", ProviderDescriptor::builder(id("p")).with_source(source("example.com")).with_resolves(Resolves::new().with_host("a.example.com/path"))), + ( + "provenance for undeclared source", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance(&source("other.example.com"), EidProvenance::new().with_mm(2)), + ), + ( + "expiring undeclared source", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_expiring_source(&source("other.example.com")), + ), + ]; + for (name, builder) in cases { + let error = builder.build().expect_err(name); + assert!( + matches!(error.current_context(), IdentityProviderError::Config { .. }), + "should be a Config error: {name}" + ); + } + } + + #[test] + fn expiring_and_provenance_are_per_source() { + let a = source("a.example.com"); + let b = source("b.example.com"); + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(a.clone()) + .with_source(b.clone()) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_expiring_source(&b) + .with_provenance(&a, EidProvenance::new().with_inserter("example-inserter").with_mm(2)) + .build() + .expect("should build"); + + assert!(!descriptor.is_expiring(&a), "a should not expire"); + assert!(descriptor.is_expiring(&b), "b should expire"); + let provenance = descriptor.provenance(&a).expect("a should have provenance"); + assert_eq!(provenance.inserter(), Some("example-inserter"), "should keep the inserter"); + assert_eq!(provenance.matcher(), None, "should leave the matcher unset"); + assert_eq!(provenance.mm(), Some(2), "should keep mm"); + assert!(descriptor.provenance(&b).is_none(), "b should have none"); + assert_eq!(descriptor.source_named("A.EXAMPLE.COM"), Some(&a), "should find a by any spelling"); + } + + #[test] + fn expiring_ids_marks_every_source() { + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(source("a.example.com")) + .with_source(source("b.example.com")) + .with_expiring_ids() + .with_captures(Captures::new().with_path_prefix("/integrations/p/")) + .build() + .expect("should build"); + assert!(descriptor.sources().iter().all(|s| descriptor.is_expiring(s)), "should mark every source"); + } + + #[test] + fn captures_match_exact_paths_and_prefixes() { + let captures = Captures::new() + .with_path(Method::POST, "/integrations/p/api/token") + .with_path_prefix("/integrations/p/stream/"); + assert!(captures.matches(&Method::POST, "/integrations/p/api/token"), "should match the exact path"); + assert!(!captures.matches(&Method::GET, "/integrations/p/api/token"), "should require the method"); + assert!(!captures.matches(&Method::POST, "/integrations/p/api/token/x"), "exact means exact"); + assert!(captures.matches(&Method::GET, "/integrations/p/stream/1"), "prefix matches any method"); + assert!(!captures.matches(&Method::GET, "/integrations/p/settings"), "should not match others"); + } + + #[test] + fn needs_differ_by_kind() { + let resolves = Resolves::new() + .with_endpoint("https://a.example.com") + .with_needs([Input::ClientIp]) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]); + assert_eq!( + resolves.needs(ResolveKind::Initial), + [Input::ClientIp, Input::HashedEmail].into_iter().collect(), + "Initial needs both" + ); + assert_eq!( + resolves.needs(ResolveKind::Refresh), + [Input::ClientIp].into_iter().collect(), + "Refresh needs only the IP" + ); + } +} +``` + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::descriptor` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Add above the existing enums in `descriptor.rs`: + +```rust +use std::collections::{BTreeMap, BTreeSet}; + +use error_stack::Report; +use http::Method; +use url::Url; + +use super::ids::{IdentityProviderId, SourceDomain}; +use super::resolution::IdentityProviderError; +use crate::ec::partner::normalize_partner_source_domain; + +/// What a provider is, may write, needs and must respect. +/// +/// Built from config with no secrets, so `ts config validate` and the running +/// coordinator read the same value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderDescriptor { + id: IdentityProviderId, + sources: Vec, + resolves: Option, + endpoint: Option, + captures: Option, + consent_policy: ConsentPolicy, + provenance: BTreeMap, + expiring: BTreeSet, +} + +impl ProviderDescriptor { + /// Starts a descriptor for `id`. + #[must_use] + pub fn builder(id: IdentityProviderId) -> ProviderDescriptorBuilder { + ProviderDescriptorBuilder { + id, + sources: Vec::new(), + resolves: None, + captures: None, + consent_policy: ConsentPolicy::strict(), + provenance: BTreeMap::new(), + expiring_all: false, + expiring: BTreeSet::new(), + } + } + + /// The provider's ID. + #[must_use] + pub fn id(&self) -> &IdentityProviderId { + &self.id + } + + /// The sources this provider may write, in declaration order. + #[must_use] + pub fn sources(&self) -> &[SourceDomain] { + &self.sources + } + + /// The first source added. Every built descriptor has one. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + &self.sources[0] + } + + /// Finds a declared source by a vendor's spelling of it, normalized. + #[must_use] + pub fn source_named(&self, name: &str) -> Option<&SourceDomain> { + self.sources.iter().find(|source| source.matches(name)) + } + + /// Active resolution, when declared. + #[must_use] + pub fn resolves(&self) -> Option<&Resolves> { + self.resolves.as_ref() + } + + /// The endpoint declared with [`Resolves::with_endpoint`], if any. + #[must_use] + pub fn endpoint(&self) -> Option<&Url> { + self.endpoint.as_ref() + } + + /// Passive capture, when declared. + #[must_use] + pub fn captures(&self) -> Option<&Captures> { + self.captures.as_ref() + } + + /// The consent this provider requires. + #[must_use] + pub fn consent_policy(&self) -> &ConsentPolicy { + &self.consent_policy + } + + /// EID-level fields for `source`, if declared. + #[must_use] + pub fn provenance(&self, source: &SourceDomain) -> Option<&EidProvenance> { + self.provenance.get(source) + } + + /// Whether IDs of `source` are declared as expiring. + #[must_use] + pub fn is_expiring(&self, source: &SourceDomain) -> bool { + self.expiring.contains(source) + } +} + +/// Builds a [`ProviderDescriptor`]. +#[derive(Debug, Clone)] +pub struct ProviderDescriptorBuilder { + id: IdentityProviderId, + sources: Vec, + resolves: Option, + captures: Option, + consent_policy: ConsentPolicy, + provenance: BTreeMap, + expiring_all: bool, + expiring: BTreeSet, +} + +impl ProviderDescriptorBuilder { + /// Adds a source this provider may write. Repeatable. A repeat is ignored. + #[must_use] + pub fn with_source(mut self, source: SourceDomain) -> Self { + if !self.sources.contains(&source) { + self.sources.push(source); + } + self + } + + /// Declares active resolution. + #[must_use] + pub fn with_resolves(mut self, resolves: Resolves) -> Self { + self.resolves = Some(resolves); + self + } + + /// Declares passive capture. + #[must_use] + pub fn with_captures(mut self, captures: Captures) -> Self { + self.captures = Some(captures); + self + } + + /// Replaces the default policy, [`ConsentPolicy::strict`]. + #[must_use] + pub fn with_consent_policy(mut self, policy: ConsentPolicy) -> Self { + self.consent_policy = policy; + self + } + + /// Marks the IDs of every declared source as expiring (spec 6.1). + #[must_use] + pub fn with_expiring_ids(mut self) -> Self { + self.expiring_all = true; + self + } + + /// Marks the IDs of one declared source as expiring. + #[must_use] + pub fn with_expiring_source(mut self, source: &SourceDomain) -> Self { + self.expiring.insert(source.clone()); + self + } + + /// EID-level fields for a declared source, added to every server-side + /// value of that source this provider wrote. No KV field stores them. + #[must_use] + pub fn with_provenance(mut self, source: &SourceDomain, provenance: EidProvenance) -> Self { + self.provenance.insert(source.clone(), provenance); + self + } + + /// Validates and builds the descriptor. + /// + /// # Errors + /// + /// Returns `Config` when the descriptor has no source, no capability, a + /// resolver with no endpoint or host, an endpoint that is not an `https` + /// URL, a host that is not a bare hostname, or provenance or an expiring + /// mark for a source it does not declare. + pub fn build(self) -> Result> { + let id = self.id; + let fail = |what: &str| IdentityProviderError::config(format!("provider `{id}`: {what}")); + + if self.sources.is_empty() { + return Err(fail("declares no source")); + } + if self.resolves.is_none() && self.captures.is_none() { + return Err(fail("declares neither resolves nor captures")); + } + for source in self.provenance.keys().chain(self.expiring.iter()) { + if !self.sources.contains(source) { + return Err(fail(&format!("names undeclared source `{source}`"))); + } + } + + let mut endpoint = None; + let mut resolves = self.resolves; + if let Some(resolves) = resolves.as_mut() { + if let Some(raw) = resolves.raw_endpoint.as_deref() { + let url = Url::parse(raw).map_err(|_| fail("endpoint is not a URL"))?; + if url.scheme() != "https" { + return Err(fail("endpoint must use https")); + } + let host = url.host_str().ok_or_else(|| fail("endpoint has no host"))?.to_owned(); + if !resolves.hosts.contains(&host) { + resolves.hosts.insert(0, host); + } + endpoint = Some(url); + } + if resolves.hosts.is_empty() { + return Err(fail("resolves declares no endpoint or host")); + } + for host in &resolves.hosts { + if normalize_partner_source_domain(host).ok().as_deref() != Some(host.as_str()) { + return Err(fail(&format!("host `{host}` is not a bare lowercase hostname"))); + } + } + } + + let expiring = if self.expiring_all { + self.sources.iter().cloned().collect() + } else { + self.expiring + }; + + Ok(ProviderDescriptor { + id, + sources: self.sources, + resolves, + endpoint, + captures: self.captures, + consent_policy: self.consent_policy, + provenance: self.provenance, + expiring, + }) + } +} + +/// The kind of resolution one source needs. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, +)] +#[non_exhaustive] +pub enum ResolveKind { + /// No stored record for the source. + #[display("initial")] + Initial, + /// A stored record exists, valid or expired, and is due. + #[display("refresh")] + Refresh, +} + +/// Active resolution: the provider calls an endpoint. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Resolves { + raw_endpoint: Option, + hosts: Vec, + needs: BTreeSet, + needs_for: BTreeMap>, + wants: BTreeSet, + kinds: Vec, + private_state: bool, + ec_id: bool, +} + +impl Default for Resolves { + fn default() -> Self { + Self::new() + } +} + +impl Resolves { + /// A resolver with no inputs, handling `Initial` and `Refresh`. + #[must_use] + pub fn new() -> Self { + Self { + raw_endpoint: None, + hosts: Vec::new(), + needs: BTreeSet::new(), + needs_for: BTreeMap::new(), + wants: BTreeSet::new(), + kinds: vec![ResolveKind::Initial, ResolveKind::Refresh], + private_state: false, + ec_id: false, + } + } + + /// The endpoint the provider calls, as config spells it. `build()` parses + /// it and requires `https`. Its host becomes an allowed host. + #[must_use] + pub fn with_endpoint(mut self, endpoint: &str) -> Self { + self.raw_endpoint = Some(endpoint.to_owned()); + self + } + + /// Adds another host the provider may call. Repeatable. + #[must_use] + pub fn with_host(mut self, host: &str) -> Self { + if !self.hosts.iter().any(|known| known == host) { + self.hosts.push(host.to_owned()); + } + self + } + + /// Inputs required for every kind. + #[must_use] + pub fn with_needs(mut self, inputs: impl IntoIterator) -> Self { + self.needs.extend(inputs); + self + } + + /// Inputs required for one kind only. + #[must_use] + pub fn with_needs_for(mut self, kind: ResolveKind, inputs: impl IntoIterator) -> Self { + self.needs_for.entry(kind).or_default().extend(inputs); + self + } + + /// Inputs passed when available, never required. + #[must_use] + pub fn with_wants(mut self, inputs: impl IntoIterator) -> Self { + self.wants.extend(inputs); + self + } + + /// The kinds this provider handles. Default: `Initial` and `Refresh`. + #[must_use] + pub fn with_kinds(mut self, kinds: impl IntoIterator) -> Self { + self.kinds = kinds.into_iter().collect(); + self + } + + /// This provider keeps private state, so it runs only on the writer + /// request, where state can be stored (spec 5.3). + #[must_use] + pub fn with_private_state(mut self) -> Self { + self.private_state = true; + self + } + + /// Gives the provider the full EC ID. Only generic pull may use this. + #[must_use] + pub(in crate::ec::identity) fn with_ec_id(mut self) -> Self { + self.ec_id = true; + self + } + + /// Hosts the provider may call. + #[must_use] + pub(crate) fn hosts(&self) -> &[String] { + &self.hosts + } + + /// Inputs required for `kind`: the common needs plus that kind's. + #[must_use] + pub(crate) fn needs(&self, kind: ResolveKind) -> BTreeSet { + let mut needs = self.needs.clone(); + if let Some(extra) = self.needs_for.get(&kind) { + needs.extend(extra.iter().copied()); + } + needs + } + + /// Every input declared for any kind, needed or wanted. + #[must_use] + pub(crate) fn declared_inputs(&self) -> BTreeSet { + let mut all = self.needs.clone(); + all.extend(self.wants.iter().copied()); + for extra in self.needs_for.values() { + all.extend(extra.iter().copied()); + } + all + } + + /// Inputs passed when available. + #[must_use] + pub(crate) fn wants(&self) -> &BTreeSet { + &self.wants + } + + /// The kinds this provider handles. + #[must_use] + pub(crate) fn kinds(&self) -> &[ResolveKind] { + &self.kinds + } + + /// Whether the provider keeps private state. + #[must_use] + pub(crate) fn has_private_state(&self) -> bool { + self.private_state + } + + /// Whether the provider receives the full EC ID. + #[must_use] + pub(crate) fn gets_ec_id(&self) -> bool { + self.ec_id + } +} + +/// Passive capture from responses TS proxies for this provider. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Captures { + paths: Vec<(Method, String)>, + prefixes: Vec, + consent_pointers: Vec<&'static str>, +} + +impl Captures { + /// No capture rules yet. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Offers proxied responses to exactly this method and path. Repeatable. + #[must_use] + pub fn with_path(mut self, method: Method, path: &str) -> Self { + self.paths.push((method, path.to_owned())); + self + } + + /// Offers proxied responses whose path starts with `prefix`, for any + /// method. Repeatable. + #[must_use] + pub fn with_path_prefix(mut self, prefix: &str) -> Self { + self.prefixes.push(prefix.to_owned()); + self + } + + /// JSON pointers to consent strings in the proxied request's JSON body. + /// The coordinator extracts only these fields and gates capture by the + /// stricter of cookie and body consent. The provider never sees the body. + #[must_use] + pub fn with_request_consent(mut self, pointers: impl IntoIterator) -> Self { + self.consent_pointers.extend(pointers); + self + } + + /// Whether a proxied request matches a rule. + #[must_use] + pub(crate) fn matches(&self, method: &Method, path: &str) -> bool { + self.paths.iter().any(|(rule_method, rule_path)| rule_method == method && rule_path == path) + || self.prefixes.iter().any(|prefix| path.starts_with(prefix.as_str())) + } + + /// Exact path rules. + #[must_use] + pub(crate) fn paths(&self) -> &[(Method, String)] { + &self.paths + } + + /// Prefix rules. + #[must_use] + pub(crate) fn prefixes(&self) -> &[String] { + &self.prefixes + } + + /// Declared body consent pointers. + #[must_use] + pub(crate) fn consent_pointers(&self) -> &[&'static str] { + &self.consent_pointers + } +} + +/// EID-level fields (OpenRTB 2.6 §3.2.27) that bidders read. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct EidProvenance { + inserter: Option, + matcher: Option, + mm: Option, +} + +impl EidProvenance { + /// No fields set. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Sets `inserter`. + #[must_use] + pub fn with_inserter(mut self, inserter: &str) -> Self { + self.inserter = Some(inserter.to_owned()); + self + } + + /// Sets `matcher`. + #[must_use] + pub fn with_matcher(mut self, matcher: &str) -> Self { + self.matcher = Some(matcher.to_owned()); + self + } + + /// Sets `mm`. + #[must_use] + pub fn with_mm(mut self, mm: i32) -> Self { + self.mm = Some(mm); + self + } + + /// `inserter`, if set. + #[must_use] + pub fn inserter(&self) -> Option<&str> { + self.inserter.as_deref() + } + + /// `matcher`, if set. + #[must_use] + pub fn matcher(&self) -> Option<&str> { + self.matcher.as_deref() + } + + /// `mm`, if set. + #[must_use] + pub fn mm(&self) -> Option { + self.mm + } +} + +/// A US opt-out a policy can relax. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, +)] +#[non_exhaustive] +pub enum UsOptOut { + /// Sale of personal information. + #[display("sale")] + Sale, + /// Sharing for cross-context behavioural advertising. + #[display("sharing")] + Sharing, + /// Targeted advertising. + #[display("targeted_advertising")] + TargetedAdvertising, +} + +/// The consent a provider requires. Evaluated by the coordinator (Task 12). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConsentPolicy { + tcf_purposes: BTreeSet, + tcf_vendor: Option, + allow_dnt: bool, + allow_gpc: bool, + allowed_us_opt_outs: BTreeSet, + eid_persistence_only: bool, +} + +impl ConsentPolicy { + /// The default. Blocked when any consent signal present cannot be + /// decoded. Where TCF applies, requires Purposes 1 and 4. Blocked by + /// every US opt-out TS decodes, by `Sec-GPC: 1` and by `DNT: 1`. + #[must_use] + pub fn strict() -> Self { + Self { + tcf_purposes: [1, 4].into_iter().collect(), + tcf_vendor: None, + allow_dnt: false, + allow_gpc: false, + allowed_us_opt_outs: BTreeSet::new(), + eid_persistence_only: false, + } + } + + /// Also requires these TCF purposes, where TCF applies. + #[must_use] + pub fn with_tcf_purposes(mut self, purposes: impl IntoIterator) -> Self { + self.tcf_purposes.extend(purposes); + self + } + + /// Requires vendor consent for this IAB Global Vendor List ID. + #[must_use] + pub fn with_tcf_vendor(mut self, vendor_id: u16) -> Self { + self.tcf_vendor = Some(vendor_id); + self + } + + /// Does not block on `DNT: 1`. Needs a documented legal basis. + #[must_use] + pub fn allowing_dnt(mut self) -> Self { + self.allow_dnt = true; + self + } + + /// Does not block on GPC. Needs a documented legal basis. + #[must_use] + pub fn allowing_gpc(mut self) -> Self { + self.allow_gpc = true; + self + } + + /// Does not block on one US opt-out. Needs a documented legal basis. + #[must_use] + pub fn allowing_us_opt_out(mut self, opt_out: UsOptOut) -> Self { + self.allowed_us_opt_outs.insert(opt_out); + self + } + + /// Blocks exactly what pull sync blocks today: only the global EC and + /// EID-persistence gates. For generic pull's parity (spec 10.4). + #[must_use] + pub(in crate::ec::identity) fn eid_persistence_only() -> Self { + Self { eid_persistence_only: true, ..Self::strict() } + } + + /// Required TCF purposes. + #[must_use] + pub fn tcf_purposes(&self) -> &BTreeSet { + &self.tcf_purposes + } + + /// Required GVL vendor, if any. + #[must_use] + pub fn tcf_vendor(&self) -> Option { + self.tcf_vendor + } + + /// Whether `DNT: 1` is allowed. + #[must_use] + pub fn allows_dnt(&self) -> bool { + self.allow_dnt + } + + /// Whether GPC is allowed. + #[must_use] + pub fn allows_gpc(&self) -> bool { + self.allow_gpc + } + + /// Whether a US opt-out is allowed. + #[must_use] + pub fn allows_us_opt_out(&self, opt_out: UsOptOut) -> bool { + self.allowed_us_opt_outs.contains(&opt_out) + } + + /// Whether this is generic pull's parity policy. + #[must_use] + pub(crate) fn is_eid_persistence_only(&self) -> bool { + self.eid_persistence_only + } +} +``` + +`normalize_partner_source_domain` doubles as a hostname check: it accepts exactly a bare, lowercase-normalized hostname. Comparing its output to the input also rejects uppercase. Update `mod.rs`: + +```rust +pub use descriptor::{ + Access, Captures, ConsentPolicy, EidProvenance, Input, ProviderDescriptor, + ProviderDescriptorBuilder, ResolveKind, Resolves, UsOptOut, +}; +``` + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::descriptor` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity +git commit --signoff -S -m "Add the identity provider descriptor and its builder" +``` + +### Task 9: Provider inputs + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/input.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/mod.rs`, `migration_guards.rs` +- Test: `crates/trusted-server-core/src/ec/identity/input.rs` + +**Interfaces:** + +- Consumes: `ProviderDescriptor`, `Resolves`, `Input`, `Access`, `ResolveKind` (Task 8), and `ConsentContext` (`consent/types.rs`) +- Produces: + - `ProviderConsent` with `gdpr_applies`, `raw_tc_string`, `raw_gpp_string`, `gpp_section_ids`, `raw_us_privacy`, and `pub(crate) fn from_context(&ConsentContext) -> Self` + - `Region` with `country`, `subdivision`, `is`, and `pub(crate) fn from_geo(country: &str, region: Option<&str>) -> Option`, which normalizes to upper case and strips an `XX-` prefix + - `HashedEmail` (32 bytes) with `as_bytes`, `to_hex`, `to_base64`, and a redacting `Debug` + - `StoredId` with `uid`, `atype`, `ext`, `writer`, `acquired_at`, `expires_at`, `refresh_at`, `is_expired`, and `pub(crate) fn from_record(&KvPartnerId) -> Self` + - `ResolveSignals<'a>`: a crate-private struct holding every signal the coordinator has (IP, UA, page URL, region, HEM, consent, publisher domain, now, random seed source, EC ID, stored IDs, raw state JSON, dry-run flag) + - `ResolveInput<'a>` with the spec 4.4 accessors, `pub(crate) fn new(descriptor: &'a ProviderDescriptor, kinds: BTreeMap, signals: &'a ResolveSignals<'a>) -> Self`, `pub(crate) fn recorded_errors(&self) -> Vec`, `pub(crate) fn state_decode_failed(&self) -> bool`, and `pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report>` + - `CaptureInput<'a>` with the spec 4.4 accessors and `pub(crate) fn new(...)` + - `RandomSource`: a trait `fn fill(&self, bytes: &mut [u8])`, with `ThreadRandom` for production and `SeededRandom` (a ChaCha-free xorshift from a `u64` seed, test support only) + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::net::IpAddr; + + use super::*; + use crate::ec::identity::descriptor::{Captures, ProviderDescriptor, Resolves}; + + fn descriptor(resolves: Resolves) -> ProviderDescriptor { + ProviderDescriptor::builder("p".parse().expect("should parse")) + .with_source("example.com".parse().expect("should parse")) + .with_resolves(resolves) + .build() + .expect("should build") + } + + fn signals() -> ResolveSignals<'static> { + ResolveSignals { + client_ip: Some("192.0.2.10".parse().expect("should parse ip")), + user_agent: Some("ExampleBrowser/1.0"), + page_url: None, + region: Region::from_geo("us", Some("us-ca")), + hashed_email: None, + consent: ProviderConsent::default(), + publisher_domain: "www.example.com", + now: UnixTime::from_secs(1_760_000_000), + random: &ThreadRandom, + ec_id: Some("a".repeat(64) + ".ABC123"), + stored: BTreeMap::new(), + state_json: None, + dry_run: false, + } + } + + #[test] + fn declared_inputs_are_available_and_wanted_missing_ones_are_unavailable() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_needs([Input::ClientIp, Input::Region]) + .with_wants([Input::PageUrl]), + ); + let signals = signals(); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Initial), &signals); + + assert_eq!(input.client_ip().expect("should read ip"), "192.0.2.10".parse::().expect("ip"), "ip"); + let region = input.region().expect("should read region"); + assert_eq!((region.country(), region.subdivision()), ("US", Some("CA")), "should normalize the region"); + assert!(region.is("US", "CA"), "should test country and subdivision"); + let error = input.page_url().expect_err("should be unavailable"); + assert!( + matches!(error.current_context(), IdentityProviderError::InputUnavailable { input: Input::PageUrl }), + "a declared but missing input is InputUnavailable" + ); + assert!(input.recorded_errors().is_empty(), "declared reads record nothing"); + } + + #[test] + fn undeclared_reads_are_recorded_even_when_ignored() { + let descriptor = descriptor(Resolves::new().with_endpoint("https://a.example.com")); + let signals = signals(); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Initial), &signals); + + let ignored = input.user_agent().ok(); + let state = input.state::(); + + assert!(ignored.is_none(), "should not return an undeclared input"); + assert!(state.is_err(), "state needs with_private_state"); + assert_eq!( + input.recorded_errors(), + vec![Access::Input(Input::UserAgent), Access::PrivateState], + "should record both undeclared reads" + ); + input.endpoint().expect("the declared endpoint is always available"); + } + + #[test] + fn state_decodes_and_a_bad_state_is_recorded() { + let descriptor = descriptor(Resolves::new().with_endpoint("https://a.example.com").with_private_state()); + let mut good = signals(); + good.state_json = Some(r#"{"signature":"s"}"#.to_owned()); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &good); + let state: serde_json::Value = input.state().expect("should decode").expect("should be present"); + assert_eq!(state["signature"], "s", "should decode the state"); + assert!(!input.state_decode_failed(), "a good state is not a failure"); + + #[derive(serde::Deserialize)] + struct Typed { + #[allow(dead_code)] + signature: String, + } + let mut bad = signals(); + bad.state_json = Some(r#""a bare string""#.to_owned()); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &bad); + let error = input.state::().expect_err("should not decode"); + assert!(matches!(error.current_context(), IdentityProviderError::InvalidState), "InvalidState"); + assert!(input.state_decode_failed(), "should record the failure"); + } + + #[test] + fn dry_run_is_never_true_for_stateful_providers() { + let stateful = descriptor(Resolves::new().with_endpoint("https://a.example.com").with_private_state()); + let stateless = descriptor(Resolves::new().with_endpoint("https://a.example.com")); + let mut dry = signals(); + dry.dry_run = true; + assert!(!ResolveInput::new(&stateful, kinds(&stateful, ResolveKind::Initial), &dry).is_dry_run(), "stateful"); + assert!(ResolveInput::new(&stateless, kinds(&stateless, ResolveKind::Initial), &dry).is_dry_run(), "stateless"); + } + + #[test] + fn ec_id_is_only_for_providers_declared_with_ec_id() { + let generic = descriptor(Resolves::new().with_host("a.example.com").with_ec_id()); + let other = descriptor(Resolves::new().with_host("a.example.com")); + let signals = signals(); + ResolveInput::new(&generic, kinds(&generic, ResolveKind::Initial), &signals) + .ec_id() + .expect("generic pull reads the EC ID"); + let input = ResolveInput::new(&other, kinds(&other, ResolveKind::Initial), &signals); + input.ec_id().expect_err("others may not"); + assert_eq!(input.recorded_errors().len(), 1, "should record the read"); + } + + #[test] + fn stored_ids_and_expiry() { + let record = crate::ec::kv_types::KvPartnerId { + uid: "u".to_owned(), + w: Some("p".to_owned()), + rv: Some("0123456789abcdef".to_owned()), + at: Some(10), + exp: Some(20), + ra: Some(15), + ru: None, + atype: Some(3), + ext: None, + }; + let stored = StoredId::from_record(&record); + assert_eq!(stored.writer().map(IdentityProviderId::as_str), Some("p"), "writer"); + assert!(!stored.is_expired(UnixTime::from_secs(19)), "not expired before exp"); + assert!(stored.is_expired(UnixTime::from_secs(20)), "expired at exp"); + assert_eq!(stored.refresh_at(), Some(UnixTime::from_secs(15)), "ra"); + } + + #[test] + fn hashed_email_debug_is_redacted() { + let email = HashedEmail::from_bytes([0xab; 32]); + assert_eq!(email.to_hex(), "ab".repeat(32), "hex"); + assert_eq!(format!("{email:?}"), "HashedEmail([redacted])", "should redact Debug"); + } + + fn kinds(descriptor: &ProviderDescriptor, kind: ResolveKind) -> BTreeMap { + descriptor.sources().iter().map(|source| (source.clone(), kind)).collect() + } + + #[test] + fn capture_input_exposes_path_and_never_the_request_body() { + let descriptor = ProviderDescriptor::builder("c".parse().expect("id")) + .with_source("example.com".parse().expect("source")) + .with_captures(Captures::new().with_path(http::Method::POST, "/integrations/c/api")) + .build() + .expect("should build"); + let response = crate::ec::identity::exchange::ProviderResponse::for_test(http::StatusCode::OK, b"{}"); + let consent = ProviderConsent::default(); + let input = CaptureInput::new( + &descriptor, + "/integrations/c/api", + &response, + &BTreeMap::new(), + &consent, + "www.example.com", + UnixTime::from_secs(1), + ); + assert_eq!(input.path(), "/integrations/c/api", "path"); + assert_eq!(input.primary_source().as_str(), "example.com", "source"); + } +} +``` + +`ProviderResponse::for_test` arrives in Task 10. Until then, comment out the last test and enable it in Task 10, Step 4. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::input` +Expected: compile errors. + +- [ ] **Step 3: Implement `input.rs`** + +```rust +//! What a provider may read for one call. + +use core::cell::RefCell; +use core::fmt; +use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use error_stack::Report; +use rand::RngCore as _; +use serde::de::DeserializeOwned; +use url::Url; + +use super::descriptor::{Access, Input, ProviderDescriptor, ResolveKind}; +use super::exchange::ProviderResponse; +use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::resolution::IdentityProviderError; +use crate::consent::ConsentContext; +use crate::ec::kv_types::KvPartnerId; + + +/// The raw consent strings a provider forwards, and `gdpr_applies`. +/// +/// Providers never decide consent. The coordinator has already applied the +/// descriptor's policy. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct ProviderConsent { + gdpr_applies: bool, + tc_string: Option, + gpp_string: Option, + gpp_section_ids: Option>, + us_privacy: Option, +} + +impl fmt::Debug for ProviderConsent { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderConsent") + .field("gdpr_applies", &self.gdpr_applies) + .field("tc_string", &self.tc_string.as_ref().map(|_| "[redacted]")) + .field("gpp_string", &self.gpp_string.as_ref().map(|_| "[redacted]")) + .field("gpp_section_ids", &self.gpp_section_ids) + .field("us_privacy", &self.us_privacy.as_ref().map(|_| "[redacted]")) + .finish() + } +} + +impl ProviderConsent { + /// The view of a request's consent context. + #[must_use] + pub(crate) fn from_context(context: &ConsentContext) -> Self { + Self { + gdpr_applies: context.gdpr_applies, + tc_string: context.raw_tc_string.clone(), + gpp_string: context.raw_gpp_string.clone(), + gpp_section_ids: context.gpp_section_ids.clone(), + us_privacy: context.raw_us_privacy.clone(), + } + } + + /// Whether GDPR applies to this request. + #[must_use] + pub fn gdpr_applies(&self) -> bool { + self.gdpr_applies + } + + /// The raw TCF string. + #[must_use] + pub fn raw_tc_string(&self) -> Option<&str> { + self.tc_string.as_deref() + } + + /// The raw GPP string. + #[must_use] + pub fn raw_gpp_string(&self) -> Option<&str> { + self.gpp_string.as_deref() + } + + /// The applicable GPP section IDs. + #[must_use] + pub fn gpp_section_ids(&self) -> Option<&[u16]> { + self.gpp_section_ids.as_deref() + } + + /// The raw US Privacy string. + #[must_use] + pub fn raw_us_privacy(&self) -> Option<&str> { + self.us_privacy.as_deref() + } +} +``` + +Check the real field names on `ConsentContext` (`consent/types.rs:109-148`): `grep -n "pub " crates/trusted-server-core/src/consent/types.rs | sed -n 1,60p`. Use them in `from_context`. If the section IDs are stored as another type, convert here. The rest of the file: + +```rust +/// Country and subdivision, upper case. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Region { + country: String, + subdivision: Option, +} + +impl Region { + /// Normalizes platform geo: upper case, and an ISO 3166-2 `US-CA` style + /// region loses its country prefix. `None` without a two-letter country. + #[must_use] + pub(crate) fn from_geo(country: &str, region: Option<&str>) -> Option { + let country = country.trim().to_ascii_uppercase(); + if country.len() != 2 || !country.bytes().all(|byte| byte.is_ascii_alphabetic()) { + return None; + } + let subdivision = region + .map(|value| value.trim().to_ascii_uppercase()) + .map(|value| match value.split_once('-') { + Some((prefix, rest)) if prefix == country => rest.to_owned(), + _ => value, + }) + .filter(|value| !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric())); + Some(Self { country, subdivision }) + } + + /// ISO 3166-1 alpha-2, upper case, for example `"US"`. + #[must_use] + pub fn country(&self) -> &str { + &self.country + } + + /// The ISO 3166-2 subdivision without the country prefix, for example + /// `"CA"`. `None` when the platform does not know it. + #[must_use] + pub fn subdivision(&self) -> Option<&str> { + self.subdivision.as_deref() + } + + /// Whether this is `country` and `subdivision`. + #[must_use] + pub fn is(&self, country: &str, subdivision: &str) -> bool { + self.country == country && self.subdivision.as_deref() == Some(subdivision) + } +} + +/// The 32 raw bytes of a SHA-256 of a normalized email. Personal data. +#[derive(Clone, PartialEq, Eq)] +pub struct HashedEmail([u8; 32]); + +impl fmt::Debug for HashedEmail { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HashedEmail([redacted])") + } +} + +impl HashedEmail { + /// From digest bytes. No code in v1 calls this outside tests. + #[must_use] + pub(crate) fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// The digest. + #[must_use] + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + /// Lowercase hex, as LiveRamp takes it. + #[must_use] + pub fn to_hex(&self) -> String { + hex::encode(self.0) + } + + /// Standard base64, as UID2 takes it. + #[must_use] + pub fn to_base64(&self) -> String { + STANDARD.encode(self.0) + } +} + +/// A stored record for one of this provider's sources, valid or expired. +#[derive(Clone, PartialEq)] +pub struct StoredId { + uid: String, + atype: Option, + ext: Option, + writer: Option, + acquired_at: Option, + expires_at: Option, + refresh_at: Option, +} + +impl fmt::Debug for StoredId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StoredId") + .field("uid", &"[redacted]") + .field("writer", &self.writer) + .field("acquired_at", &self.acquired_at) + .field("expires_at", &self.expires_at) + .field("refresh_at", &self.refresh_at) + .finish_non_exhaustive() + } +} + +impl StoredId { + /// From a KV record. A writer that does not parse is treated as none. + #[must_use] + pub(crate) fn from_record(record: &KvPartnerId) -> Self { + Self { + uid: record.uid.clone(), + atype: record.atype, + ext: record.ext.clone(), + writer: record.w.as_deref().and_then(|writer| writer.parse().ok()), + acquired_at: record.at.map(UnixTime::from_secs), + expires_at: record.exp.map(UnixTime::from_secs), + refresh_at: record.ra.map(UnixTime::from_secs), + } + } + + /// The UID. + #[must_use] + pub fn uid(&self) -> &str { + &self.uid + } + + /// Provider-supplied `atype`. + #[must_use] + pub fn atype(&self) -> Option { + self.atype + } + + /// UID-level ext, as bidders read it in `uids[].ext`. + #[must_use] + pub fn ext(&self) -> Option<&serde_json::Value> { + self.ext.as_ref() + } + + /// The provider that wrote it. `None` for browser ingest and legacy rows. + #[must_use] + pub fn writer(&self) -> Option<&IdentityProviderId> { + self.writer.as_ref() + } + + /// When it was acquired or last refreshed. + #[must_use] + pub fn acquired_at(&self) -> Option { + self.acquired_at + } + + /// When it must stop being sent. + #[must_use] + pub fn expires_at(&self) -> Option { + self.expires_at + } + + /// When it is due for refresh. + #[must_use] + pub fn refresh_at(&self) -> Option { + self.refresh_at + } + + /// Whether `now` is at or past its expiry. + #[must_use] + pub fn is_expired(&self, now: UnixTime) -> bool { + self.expires_at.is_some_and(|expires| now >= expires) + } +} + +/// A source of random bytes. +pub(crate) trait RandomSource: Send + Sync { + /// Fills `bytes`. + fn fill(&self, bytes: &mut [u8]); +} + +/// The platform generator EC generation uses (`rand::thread_rng`). +pub(crate) struct ThreadRandom; + +impl RandomSource for ThreadRandom { + fn fill(&self, bytes: &mut [u8]) { + rand::thread_rng().fill_bytes(bytes); + } +} + +/// Deterministic bytes for tests: xorshift64* from a seed. +#[cfg(any(test, feature = "test-utils"))] +pub(crate) struct SeededRandom(std::sync::Mutex); + +#[cfg(any(test, feature = "test-utils"))] +impl SeededRandom { + pub(crate) fn new(seed: u64) -> Self { + Self(std::sync::Mutex::new(seed.max(1))) + } +} + +#[cfg(any(test, feature = "test-utils"))] +impl RandomSource for SeededRandom { + fn fill(&self, bytes: &mut [u8]) { + let mut state = self.0.lock().expect("should lock the seed"); + for byte in bytes { + *state ^= *state >> 12; + *state ^= *state << 25; + *state ^= *state >> 27; + *byte = (state.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 56) as u8; + } + } +} + +/// Every signal the coordinator holds for one request. Crate-private: a +/// provider only ever sees a [`ResolveInput`] filtered by its descriptor. +pub(crate) struct ResolveSignals<'a> { + pub(crate) client_ip: Option, + pub(crate) user_agent: Option<&'a str>, + pub(crate) page_url: Option, + pub(crate) region: Option, + pub(crate) hashed_email: Option, + pub(crate) consent: ProviderConsent, + pub(crate) publisher_domain: &'a str, + pub(crate) now: UnixTime, + pub(crate) random: &'a dyn RandomSource, + pub(crate) ec_id: Option, + pub(crate) stored: BTreeMap, + pub(crate) state_json: Option, + pub(crate) dry_run: bool, +} + +/// What one provider may read for one call. +pub struct ResolveInput<'a> { + descriptor: &'a ProviderDescriptor, + kinds: BTreeMap, + signals: &'a ResolveSignals<'a>, + available: BTreeSet, + undeclared: RefCell>, + state_failed: RefCell, +} + +impl fmt::Debug for ResolveInput<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ResolveInput") + .field("provider", self.descriptor.id()) + .field("kinds", &self.kinds) + .finish_non_exhaustive() + } +} + +impl<'a> ResolveInput<'a> { + /// The input for one call. `kinds` holds only the sources to resolve. + #[must_use] + pub(crate) fn new( + descriptor: &'a ProviderDescriptor, + kinds: BTreeMap, + signals: &'a ResolveSignals<'a>, + ) -> Self { + let available = descriptor + .resolves() + .map(|resolves| resolves.declared_inputs()) + .unwrap_or_default(); + Self { + descriptor, + kinds, + signals, + available, + undeclared: RefCell::new(Vec::new()), + state_failed: RefCell::new(false), + } + } + + fn record(&self, access: Access) -> Report { + self.undeclared.borrow_mut().push(access); + Report::new(IdentityProviderError::UndeclaredInput { access }) + } + + fn declared(&self, input: Input, value: Option) -> Result> { + if !self.available.contains(&input) { + return Err(self.record(Access::Input(input))); + } + value.ok_or_else(|| Report::new(IdentityProviderError::InputUnavailable { input })) + } + + /// Undeclared reads recorded so far. The coordinator logs them as + /// `Config`, and the harness fails the test. + #[must_use] + pub(crate) fn recorded_errors(&self) -> Vec { + self.undeclared.borrow().clone() + } + + /// Whether a `state()` call failed to decode. + #[must_use] + pub(crate) fn state_decode_failed(&self) -> bool { + *self.state_failed.borrow() + } + + /// The sources to resolve this time, each with its own kind. + pub fn sources(&self) -> impl Iterator + '_ { + self.kinds.iter().map(|(source, kind)| (source, *kind)) + } + + /// Whether `source` is being resolved for the first time. + #[must_use] + pub fn is_initial(&self, source: &SourceDomain) -> bool { + self.kinds.get(source) == Some(&ResolveKind::Initial) + } + + /// Whether `source` is being refreshed. + #[must_use] + pub fn is_refresh(&self, source: &SourceDomain) -> bool { + self.kinds.get(source) == Some(&ResolveKind::Refresh) + } + + /// The descriptor's primary source. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + self.descriptor.primary_source() + } + + /// The stored record for one of this provider's sources. + #[must_use] + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId> { + self.signals.stored.get(source) + } + + /// This provider's private state, decoded. `Ok(None)` when there is none. + /// + /// # Errors + /// + /// `InvalidState` when the stored state does not decode as `T`. The + /// failure is recorded, and the coordinator clears the state whatever + /// the provider returns. `UndeclaredInput` without `with_private_state()`. + pub fn state(&self) -> Result, Report> { + if !self.descriptor.resolves().is_some_and(|resolves| resolves.has_private_state()) { + return Err(self.record(Access::PrivateState)); + } + let Some(raw) = self.signals.state_json.as_deref() else { + return Ok(None); + }; + serde_json::from_str(raw).map(Some).map_err(|_| { + *self.state_failed.borrow_mut() = true; + Report::new(IdentityProviderError::InvalidState) + }) + } + + /// True during the dry run (spec 5.2). Always false for a provider + /// declared `with_private_state()`. + #[must_use] + pub fn is_dry_run(&self) -> bool { + self.signals.dry_run + && !self.descriptor.resolves().is_some_and(|resolves| resolves.has_private_state()) + } + + /// The endpoint declared with `Resolves::with_endpoint`. + /// + /// # Errors + /// + /// `UndeclaredInput { access: Endpoint }` when none was declared. + pub fn endpoint(&self) -> Result<&Url, Report> { + self.descriptor.endpoint().ok_or_else(|| self.record(Access::Endpoint)) + } + + /// The consent strings to forward. + #[must_use] + pub fn consent(&self) -> &ProviderConsent { + &self.signals.consent + } + + /// The publisher's domain. + #[must_use] + pub fn publisher_domain(&self) -> &str { + self.signals.publisher_domain + } + + /// The request's time. + #[must_use] + pub fn now(&self) -> UnixTime { + self.signals.now + } + + /// Random bytes from the platform generator. Fixed by the harness. + #[must_use] + pub fn random_bytes(&self) -> [u8; N] { + let mut bytes = [0u8; N]; + self.signals.random.fill(&mut bytes); + bytes + } + + /// The platform's client address. + /// + /// # Errors + /// + /// `UndeclaredInput` when not declared, `InputUnavailable` when missing. + pub fn client_ip(&self) -> Result> { + self.declared(Input::ClientIp, self.signals.client_ip) + } + + /// The `User-Agent`. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn user_agent(&self) -> Result<&str, Report> { + self.declared(Input::UserAgent, self.signals.user_agent) + } + + /// The page URL, without query string or fragment. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn page_url(&self) -> Result<&Url, Report> { + self.declared(Input::PageUrl, self.signals.page_url.as_ref()) + } + + /// Country and subdivision. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn region(&self) -> Result<&Region, Report> { + self.declared(Input::Region, self.signals.region.as_ref()) + } + + /// A hashed email. Never available in v1. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn hashed_email(&self) -> Result<&HashedEmail, Report> { + self.declared(Input::HashedEmail, self.signals.hashed_email.as_ref()) + } + + /// The full EC ID. Only for a resolver declared `with_ec_id()`. + /// + /// # Errors + /// + /// `UndeclaredInput` otherwise. `InputUnavailable` (as `ClientIp`, the + /// nearest input) when the request has no EC ID. + pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report> { + if !self.descriptor.resolves().is_some_and(|resolves| resolves.gets_ec_id()) { + return Err(self.record(Access::PrivateState)); + } + self.signals + .ec_id + .as_deref() + .ok_or_else(|| Report::new(IdentityProviderError::InputUnavailable { input: Input::ClientIp })) + } +} + +/// What a capture may read from one proxied response. +pub struct CaptureInput<'a> { + descriptor: &'a ProviderDescriptor, + path: &'a str, + response: &'a ProviderResponse, + stored: &'a BTreeMap, + consent: &'a ProviderConsent, + publisher_domain: &'a str, + now: UnixTime, +} + +impl fmt::Debug for CaptureInput<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CaptureInput") + .field("provider", self.descriptor.id()) + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl<'a> CaptureInput<'a> { + /// The input for one capture. + #[must_use] + pub(crate) fn new( + descriptor: &'a ProviderDescriptor, + path: &'a str, + response: &'a ProviderResponse, + stored: &'a BTreeMap, + consent: &'a ProviderConsent, + publisher_domain: &'a str, + now: UnixTime, + ) -> Self { + Self { descriptor, path, response, stored, consent, publisher_domain, now } + } + + /// The proxied request path, without query string. + #[must_use] + pub fn path(&self) -> &str { + self.path + } + + /// The proxied response, decoded. + #[must_use] + pub fn response(&self) -> &ProviderResponse { + self.response + } + + /// The descriptor's primary source. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + self.descriptor.primary_source() + } + + /// Finds a declared source by a vendor's spelling. + #[must_use] + pub fn source_named(&self, name: &str) -> Option<&SourceDomain> { + self.descriptor.source_named(name) + } + + /// The stored record for a source. + #[must_use] + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId> { + self.stored.get(source) + } + + /// The stricter of the cookie consent and the request-body consent. + #[must_use] + pub fn consent(&self) -> &ProviderConsent { + self.consent + } + + /// The publisher's domain. + #[must_use] + pub fn publisher_domain(&self) -> &str { + self.publisher_domain + } + + /// The request's time. + #[must_use] + pub fn now(&self) -> UnixTime { + self.now + } +} +``` + +`CaptureInput::new` takes 7 arguments, which is the maximum `AGENTS.md` allows. Do not add an eighth. Use a struct if you need one. `hex` is already a dependency of core (check with `grep -n '^hex' crates/trusted-server-core/Cargo.toml`). If it is only a build dependency, add `hex = { workspace = true }` to `[dependencies]`. `ResolveKind` stays in `descriptor.rs` and is re-exported from `mod.rs`. Update `mod.rs`: + +```rust +pub mod input; +pub use input::{CaptureInput, HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; +``` + +Add `ec/identity/input.rs` to `migration_guards.rs`. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::input` +Expected: PASS for every test except the `capture_input_...` test, which waits for Task 10. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add declared-input enforcement for identity providers" +``` + +### Task 10: Requests, responses and resolutions + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/exchange.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/resolution.rs` (add `Resolution`, `ResolvedId`, `When`, `StateChange`) +- Modify: `mod.rs`, `migration_guards.rs` +- Test: both files + +**Interfaces:** + +- Produces: everything listed in spec 4.5, plus: + - `ProviderRequest::call_context_json(&self) -> Option<&str>` (crate-private) + - `ProviderRequest::has_cookie_header(&self) -> bool` (crate-private) + - `ProviderResponse::new(status: StatusCode, headers: HeaderMap, body: Vec, call_context: Option) -> Self` (crate-private) + - `ProviderResponse::for_test(status, body: &[u8]) -> Self` (test support) + - `Resolution::found_ids_by_source(&self)`: the first ID per source, plus the count of duplicates dropped (crate-private) + - `MAX_CALL_CONTEXT_BYTES = 1024`, `MAX_UID_EXT_BYTES = 256` + +- [ ] **Step 1: Write the failing tests** + +In `exchange.rs`: + +```rust +#[cfg(test)] +mod tests { + use http::{HeaderName, HeaderValue, Method, StatusCode}; + use serde_json::json; + + use super::*; + + fn url() -> Url { + Url::parse("https://api.example.com/v1/fetch").expect("should parse") + } + + #[test] + fn post_json_sets_body_and_content_type() { + let request = ProviderRequest::post_json(&url(), &json!({ "a": 1 })).expect("should build"); + assert_eq!(request.method(), Method::POST, "method"); + assert_eq!(request.headers()["content-type"], "application/json", "content type"); + assert_eq!(request.body(), br#"{"a":1}"#, "body"); + } + + #[test] + fn query_bearer_and_headers() { + let request = ProviderRequest::get(&url()) + .with_query("pid", "1 2") + .with_bearer(&Redacted::new("token".to_owned())) + .try_with_header("x-example", "v") + .expect("should accept a valid header"); + assert_eq!(request.url().as_str(), "https://api.example.com/v1/fetch?pid=1+2", "should encode the query"); + assert_eq!(request.headers()["authorization"], "Bearer token", "bearer"); + assert_eq!(request.headers()["x-example"], "v", "header"); + ProviderRequest::get(&url()).try_with_header("bad header", "v").expect_err("should reject a bad name"); + ProviderRequest::get(&url()).try_with_header("x", "bad\nvalue").expect_err("should reject a bad value"); + } + + #[test] + fn cookie_headers_are_flagged_from_either_helper() { + let typed = ProviderRequest::get(&url()) + .with_header(HeaderName::from_static("cookie"), HeaderValue::from_static("a=1")); + let dynamic = ProviderRequest::get(&url()).try_with_header("Cookie", "a=1").expect("should build"); + assert!(typed.has_cookie_header(), "typed helper"); + assert!(dynamic.has_cookie_header(), "dynamic helper"); + assert!(!ProviderRequest::get(&url()).has_cookie_header(), "control"); + } + + #[test] + fn call_context_round_trips_and_is_capped() { + let request = ProviderRequest::get(&url()).try_with_call_context(&json!({ "nonce": [1, 2] })).expect("ok"); + let response = ProviderResponse::new( + StatusCode::OK, + http::HeaderMap::new(), + Vec::new(), + request.call_context_json().map(str::to_owned), + ); + let context: serde_json::Value = response.call_context().expect("decode").expect("present"); + assert_eq!(context, json!({ "nonce": [1, 2] }), "should round-trip"); + + let error = ProviderRequest::get(&url()) + .try_with_call_context(&"x".repeat(MAX_CALL_CONTEXT_BYTES)) + .expect_err("should reject oversize"); + assert!(matches!(error.current_context(), IdentityProviderError::Config { .. }), "Config"); + } + + #[test] + fn success_helpers_check_status_and_encoding() { + let ok = ProviderResponse::for_test(StatusCode::OK, br#"{"uid":"u"}"#); + let value: serde_json::Value = ok.success_json().expect("should parse"); + assert_eq!(value["uid"], "u", "uid"); + + let rejected = ProviderResponse::for_test(StatusCode::FORBIDDEN, br#"{}"#); + let error = rejected.success_json::().expect_err("should reject"); + assert!( + matches!(error.current_context(), IdentityProviderError::Rejected { status } if *status == StatusCode::FORBIDDEN), + "Rejected with the status" + ); + rejected.json::().expect("json ignores the status"); + + let binary = ProviderResponse::for_test(StatusCode::OK, &[0xff, 0xfe]); + let error = binary.success_text().expect_err("should reject non-UTF-8"); + assert!(matches!(error.current_context(), IdentityProviderError::Malformed), "Malformed"); + } +} +``` + +In `resolution.rs`: + +```rust +#[cfg(test)] +mod tests { + use std::time::Duration; + + use serde_json::json; + + use super::*; + + fn source(name: &str) -> SourceDomain { + name.parse().expect("should parse") + } + + #[test] + fn empty_found_is_no_match() { + let resolution = Resolution::found([]); + assert!(resolution.is_no_match(), "should be no_match"); + assert!(resolution.ids().is_empty(), "no ids"); + } + + #[test] + fn found_keeps_the_first_id_per_source() { + let a = source("a.example.com"); + let resolution = Resolution::found([ResolvedId::new(&a, "first"), ResolvedId::new(&a, "second")]); + let (kept, dropped) = resolution.found_ids_by_source(); + assert_eq!(kept.len(), 1, "one per source"); + assert_eq!(kept[0].uid(), "first", "should keep the first"); + assert_eq!(dropped, 1, "should count the duplicate"); + } + + #[test] + fn invalidations_chain_and_state_changes_are_reported() { + let a = source("a.example.com"); + let b = source("b.example.com"); + let resolution = Resolution::found([ResolvedId::new(&a, "u")]) + .also_invalidate(&b) + .with_retry_in(Duration::from_secs(60)) + .replacing_state(&json!({ "signature": "s" })); + assert_eq!(resolution.invalidated(), &[b], "should invalidate b"); + assert_eq!(resolution.retry_in(), Some(Duration::from_secs(60)), "retry"); + assert_eq!(resolution.state_change(), StateChange::Replace(r#"{"signature":"s"}"#), "state"); + assert_eq!(Resolution::no_match().clearing_state().state_change(), StateChange::Clear, "clear"); + assert_eq!(Resolution::no_match().state_change(), StateChange::Keep, "keep"); + } + + #[test] + fn oversized_state_is_marked_invalid_not_dropped_silently() { + let resolution = Resolution::found([ResolvedId::new(&source("a.example.com"), "u")]) + .replacing_state(&"x".repeat(2000)); + assert!(resolution.state_rejected(), "should flag the state as rejected"); + } + + #[test] + fn oversized_ext_is_dropped_and_the_id_kept() { + let id = ResolvedId::new(&source("a.example.com"), "u").with_ext(json!({ "pad": "x".repeat(300) })); + assert_eq!(id.uid(), "u", "should keep the id"); + assert!(id.ext().is_none(), "should drop the ext"); + let small = ResolvedId::new(&source("a.example.com"), "u").with_ext(json!({ "linkType": 2 })); + assert_eq!(small.ext(), Some(&json!({ "linkType": 2 })), "control: a small ext is kept"); + } + + #[test] + fn opted_out_is_reported() { + assert!(Resolution::opted_out().is_opted_out(), "opted out"); + assert!(!Resolution::no_match().is_opted_out(), "control"); + } +} +``` + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::exchange` and `cargo test-fastly -- ec::identity::resolution` +Expected: compile errors. + +- [ ] **Step 3: Implement `exchange.rs`** + +```rust +//! What a provider sends and receives. The coordinator does all I/O. + +use core::fmt; + +use base64::Engine as _; +use error_stack::Report; +use http::header::{AUTHORIZATION, CONTENT_TYPE, COOKIE}; +use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use url::Url; + +use super::resolution::IdentityProviderError; +use crate::redacted::Redacted; + +/// Maximum JSON bytes of a call context. +pub const MAX_CALL_CONTEXT_BYTES: usize = 1024; + +/// One call a provider wants made. +#[derive(Clone)] +pub struct ProviderRequest { + method: Method, + url: Url, + headers: HeaderMap, + body: Vec, + call_context: Option, +} + +impl fmt::Debug for ProviderRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderRequest") + .field("method", &self.method) + .field("host", &self.url.host_str()) + .field("path", &self.url.path()) + .field("headers", &self.headers.keys().collect::>()) + .field("body_len", &self.body.len()) + .finish_non_exhaustive() + } +} + +impl ProviderRequest { + /// Any method, with no body. + #[must_use] + pub fn new(method: Method, url: &Url) -> Self { + Self { method, url: url.clone(), headers: HeaderMap::new(), body: Vec::new(), call_context: None } + } + + /// A `GET`. + #[must_use] + pub fn get(url: &Url) -> Self { + Self::new(Method::GET, url) + } + + /// A `POST` with a JSON body. + /// + /// # Errors + /// + /// `Config` when `body` does not serialize. + pub fn post_json(url: &Url, body: &impl Serialize) -> Result> { + let bytes = serde_json::to_vec(body) + .map_err(|_| IdentityProviderError::config("request body does not serialize"))?; + Self::new(Method::POST, url).with_body("application/json", bytes) + } + + /// A `POST` with a `text/plain` body. + #[must_use] + pub fn post_text(url: &Url, body: &str) -> Self { + let mut request = Self::new(Method::POST, url); + request.headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + request.body = body.as_bytes().to_vec(); + request + } + + /// Sets the body and its `Content-Type`. + /// + /// # Errors + /// + /// `Config` when `content_type` is not a valid header value. + pub fn with_body(mut self, content_type: &str, body: Vec) -> Result> { + let value = HeaderValue::from_str(content_type) + .map_err(|_| IdentityProviderError::config("invalid content type"))?; + self.headers.insert(CONTENT_TYPE, value); + self.body = body; + Ok(self) + } + + /// Appends a query parameter, URL-encoded. + #[must_use] + pub fn with_query(mut self, name: &str, value: &str) -> Self { + self.url.query_pairs_mut().append_pair(name, value); + self + } + + /// Sets a header. A `Cookie` header makes the coordinator refuse the call. + #[must_use] + pub fn with_header(mut self, name: HeaderName, value: HeaderValue) -> Self { + self.headers.insert(name, value); + self + } + + /// Sets a header from strings. + /// + /// # Errors + /// + /// `Config` when the name or value is invalid. + pub fn try_with_header(self, name: &str, value: &str) -> Result> { + let name = HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| IdentityProviderError::config("invalid header name"))?; + let value = HeaderValue::from_str(value) + .map_err(|_| IdentityProviderError::config("invalid header value"))?; + Ok(self.with_header(name, value)) + } + + /// Adds `Authorization: Bearer `. + #[must_use] + pub fn with_bearer(mut self, token: &Redacted) -> Self { + if let Ok(mut value) = HeaderValue::from_str(&format!("Bearer {}", token.expose())) { + value.set_sensitive(true); + self.headers.insert(AUTHORIZATION, value); + } + self + } + + /// Data `response` needs from this call. Kept in memory, never sent. + /// + /// # Errors + /// + /// `Config` when it does not serialize or is over 1,024 bytes of JSON. + pub fn try_with_call_context(mut self, context: &impl Serialize) -> Result> { + let json = serde_json::to_string(context) + .map_err(|_| IdentityProviderError::config("call context does not serialize"))?; + if json.len() > MAX_CALL_CONTEXT_BYTES { + return Err(IdentityProviderError::config("call context exceeds 1024 bytes")); + } + self.call_context = Some(json); + Ok(self) + } + + /// The method. + #[must_use] + pub fn method(&self) -> &Method { + &self.method + } + + /// The URL. + #[must_use] + pub fn url(&self) -> &Url { + &self.url + } + + /// The headers. + #[must_use] + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// The body. + #[must_use] + pub fn body(&self) -> &[u8] { + &self.body + } + + /// The call context JSON. + #[must_use] + pub(crate) fn call_context_json(&self) -> Option<&str> { + self.call_context.as_deref() + } + + /// Whether the provider set a `Cookie` header. + #[must_use] + pub(crate) fn has_cookie_header(&self) -> bool { + self.headers.contains_key(COOKIE) + } +} + +/// A buffered, decoded provider response. +#[derive(Clone)] +pub struct ProviderResponse { + status: StatusCode, + headers: HeaderMap, + body: Vec, + call_context: Option, +} + +impl fmt::Debug for ProviderResponse { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderResponse") + .field("status", &self.status) + .field("body_len", &self.body.len()) + .finish_non_exhaustive() + } +} + +impl ProviderResponse { + /// Built by the coordinator after buffering and decoding. + #[must_use] + pub(crate) fn new(status: StatusCode, headers: HeaderMap, body: Vec, call_context: Option) -> Self { + Self { status, headers, body, call_context } + } + + /// A response for tests. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn for_test(status: StatusCode, body: &[u8]) -> Self { + Self::new(status, HeaderMap::new(), body.to_vec(), None) + } + + /// The status. + #[must_use] + pub fn status(&self) -> StatusCode { + self.status + } + + /// A header as a string, if present and visible ASCII. + #[must_use] + pub fn header(&self, name: &str) -> Option<&str> { + self.headers.get(name).and_then(|value| value.to_str().ok()) + } + + /// The decoded body. + #[must_use] + pub fn body(&self) -> &[u8] { + &self.body + } + + /// Parses the body as JSON, whatever the status. + /// + /// # Errors + /// + /// `Malformed` when it does not parse. + pub fn json(&self) -> Result> { + serde_json::from_slice(&self.body).map_err(|_| IdentityProviderError::malformed()) + } + + /// Requires 2xx, then parses JSON. + /// + /// # Errors + /// + /// `Rejected` for a non-2xx status, `Malformed` for a bad body. + pub fn success_json(&self) -> Result> { + self.require_success()?; + self.json() + } + + /// Requires 2xx and a UTF-8 body. + /// + /// # Errors + /// + /// `Rejected` for a non-2xx status, `Malformed` for non-UTF-8. + pub fn success_text(&self) -> Result<&str, Report> { + self.require_success()?; + core::str::from_utf8(&self.body).map_err(|_| IdentityProviderError::malformed()) + } + + /// The context the request carried, decoded. + /// + /// # Errors + /// + /// `Config` when it does not decode as `T`. + pub fn call_context(&self) -> Result, Report> { + self.call_context + .as_deref() + .map(|json| { + serde_json::from_str(json) + .map_err(|_| IdentityProviderError::config("call context does not decode")) + }) + .transpose() + } + + fn require_success(&self) -> Result<(), Report> { + if self.status.is_success() { + Ok(()) + } else { + Err(IdentityProviderError::rejected(self.status)) + } + } +} +``` + +Check where `Redacted` lives (`grep -rn "pub struct Redacted" crates/trusted-server-core/src`) and fix the import. Drop the unused `base64` import if clippy flags it. + +- [ ] **Step 4: Implement the rest of `resolution.rs`** + +Append to the file from Task 5: + +```rust +use std::time::Duration; + +use serde::Serialize; + +use super::ids::{SourceDomain, UnixTime}; +use crate::ec::kv_types::{MAX_PARTNER_EXT_BYTES, MAX_PROVIDER_STATE_BYTES}; + +/// A time as a provider gave it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum When { + /// Relative to the request's `now`. + In(Duration), + /// Absolute. + At(UnixTime), +} + +impl When { + /// Resolves against `now`. + #[must_use] + pub(crate) fn resolve(self, now: UnixTime) -> UnixTime { + match self { + Self::In(delay) => now + delay, + Self::At(at) => at, + } + } +} + +/// One ID a provider returned. +#[derive(Clone, PartialEq)] +pub struct ResolvedId { + source: SourceDomain, + uid: String, + atype: Option, + ext: Option, + expiry: Option, + refresh: Option, + refresh_until: Option, +} + +impl core::fmt::Debug for ResolvedId { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("ResolvedId") + .field("source", &self.source) + .field("uid", &"[redacted]") + .field("expiry", &self.expiry) + .finish_non_exhaustive() + } +} + +impl ResolvedId { + /// An ID for `source`. + #[must_use] + pub fn new(source: &SourceDomain, uid: impl Into) -> Self { + Self { + source: source.clone(), + uid: uid.into(), + atype: None, + ext: None, + expiry: None, + refresh: None, + refresh_until: None, + } + } + + /// Sets `atype`. + #[must_use] + pub fn with_atype(mut self, atype: i32) -> Self { + self.atype = Some(atype); + self + } + + /// UID-level ext. Over 256 serialized bytes it is dropped and the ID kept. + #[must_use] + pub fn with_ext(mut self, ext: serde_json::Value) -> Self { + let fits = serde_json::to_vec(&ext).is_ok_and(|bytes| bytes.len() <= MAX_PARTNER_EXT_BYTES); + self.ext = fits.then_some(ext); + if !fits { + log::debug!("Dropping oversized ext for source {}", self.source); + } + self + } + + /// Expires after `delay`. + #[must_use] + pub fn with_expires_in(mut self, delay: Duration) -> Self { + self.expiry = Some(When::In(delay)); + self + } + + /// Expires at `at`. + #[must_use] + pub fn with_expires_at(mut self, at: UnixTime) -> Self { + self.expiry = Some(When::At(at)); + self + } + + /// Due for refresh after `delay`. + #[must_use] + pub fn with_refresh_in(mut self, delay: Duration) -> Self { + self.refresh = Some(When::In(delay)); + self + } + + /// Due for refresh at `at`. + #[must_use] + pub fn with_refresh_at(mut self, at: UnixTime) -> Self { + self.refresh = Some(When::At(at)); + self + } + + /// When this ID stops being refreshable. Only for a provider declared + /// `with_private_state()`; the coordinator ignores it otherwise. + #[must_use] + pub fn with_refresh_until(mut self, at: UnixTime) -> Self { + self.refresh_until = Some(at); + self + } + + /// The source. + #[must_use] + pub fn source(&self) -> &SourceDomain { + &self.source + } + + /// The UID. + #[must_use] + pub fn uid(&self) -> &str { + &self.uid + } + + /// `atype`. + #[must_use] + pub fn atype(&self) -> Option { + self.atype + } + + /// UID-level ext. + #[must_use] + pub fn ext(&self) -> Option<&serde_json::Value> { + self.ext.as_ref() + } + + /// The expiry as given. + #[must_use] + pub fn expiry(&self) -> Option { + self.expiry + } + + /// The refresh hint as given. + #[must_use] + pub fn refresh(&self) -> Option { + self.refresh + } + + /// The refresh limit. + #[must_use] + pub fn refresh_until(&self) -> Option { + self.refresh_until + } +} + +/// How a result changes the provider's private state. +#[derive(Debug, PartialEq, Eq)] +pub enum StateChange<'a> { + /// Leave it. + Keep, + /// Replace it with this JSON. + Replace(&'a str), + /// Remove it. + Clear, +} + +#[derive(Debug, Clone, PartialEq)] +enum StateOp { + Keep, + Replace(String), + Clear, + Rejected, +} + +/// What one provider call produced. +#[derive(Debug, Clone, PartialEq)] +pub struct Resolution { + ids: Vec, + invalidated: Vec, + opted_out: bool, + retry_in: Option, + state: StateOp, +} + +impl Resolution { + fn empty() -> Self { + Self { ids: Vec::new(), invalidated: Vec::new(), opted_out: false, retry_in: None, state: StateOp::Keep } + } + + /// One or more IDs, at most one per source. An empty set is `no_match`. + #[must_use] + pub fn found(ids: impl IntoIterator) -> Self { + Self { ids: ids.into_iter().collect(), ..Self::empty() } + } + + /// Nothing recognized, or a capture held nothing. + #[must_use] + pub fn no_match() -> Self { + Self::empty() + } + + /// The stored ID for `source` is no longer valid. + #[must_use] + pub fn invalidate(source: &SourceDomain) -> Self { + Self { invalidated: vec![source.clone()], ..Self::empty() } + } + + /// The provider reports the user opted out. + #[must_use] + pub fn opted_out() -> Self { + Self { opted_out: true, ..Self::empty() } + } + + /// Also invalidates `source`. + #[must_use] + pub fn also_invalidate(mut self, source: &SourceDomain) -> Self { + if !self.invalidated.contains(source) { + self.invalidated.push(source.clone()); + } + self + } + + /// The earliest the coordinator may call again. + #[must_use] + pub fn with_retry_in(mut self, delay: Duration) -> Self { + self.retry_in = Some(delay); + self + } + + /// Replaces the provider's private state. Over 1,024 bytes of JSON, or + /// not serializable, the positive part of the result is rejected whole. + #[must_use] + pub fn replacing_state(mut self, state: &impl Serialize) -> Self { + self.state = match serde_json::to_string(state) { + Ok(json) if json.len() <= MAX_PROVIDER_STATE_BYTES => StateOp::Replace(json), + _ => StateOp::Rejected, + }; + self + } + + /// Clears the provider's private state. + #[must_use] + pub fn clearing_state(mut self) -> Self { + self.state = StateOp::Clear; + self + } + + /// The returned IDs, as given. + #[must_use] + pub fn ids(&self) -> &[ResolvedId] { + &self.ids + } + + /// Invalidated sources. + #[must_use] + pub fn invalidated(&self) -> &[SourceDomain] { + &self.invalidated + } + + /// Whether nothing was found, invalidated or opted out. + #[must_use] + pub fn is_no_match(&self) -> bool { + self.ids.is_empty() && self.invalidated.is_empty() && !self.opted_out + } + + /// Whether the provider reported an opt-out. + #[must_use] + pub fn is_opted_out(&self) -> bool { + self.opted_out + } + + /// The retry delay. + #[must_use] + pub fn retry_in(&self) -> Option { + self.retry_in + } + + /// The state change. + #[must_use] + pub fn state_change(&self) -> StateChange<'_> { + match &self.state { + StateOp::Keep | StateOp::Rejected => StateChange::Keep, + StateOp::Replace(json) => StateChange::Replace(json), + StateOp::Clear => StateChange::Clear, + } + } + + /// Whether `replacing_state` was given an oversized or unserializable + /// value. The coordinator then rejects the positive part (spec 4.5, 8). + #[must_use] + pub(crate) fn state_rejected(&self) -> bool { + self.state == StateOp::Rejected + } + + /// Whether this result changes state at all. + #[must_use] + pub(crate) fn changes_state(&self) -> bool { + self.state != StateOp::Keep + } + + /// The first ID per source, in order, and how many duplicates were dropped. + #[must_use] + pub(crate) fn found_ids_by_source(&self) -> (Vec<&ResolvedId>, usize) { + let mut seen = std::collections::BTreeSet::new(); + let mut kept = Vec::new(); + let mut dropped = 0; + for id in &self.ids { + if seen.insert(id.source()) { + kept.push(id); + } else { + dropped += 1; + } + } + (kept, dropped) + } +} +``` + +Add `exchange.rs` to `mod.rs` (`pub mod exchange; pub use exchange::{ProviderRequest, ProviderResponse};`) and `pub use resolution::{Resolution, ResolvedId, StateChange, When};`. Add `exchange.rs` to `migration_guards.rs`. Uncomment the capture test from Task 9. + +- [ ] **Step 5: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add provider requests, responses and resolutions" +``` + +### Task 11: The `IdentityProvider` trait + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/provider.rs` +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `crates/trusted-server-core/src/ec/identity/provider.rs` + +**Interfaces:** + +- Consumes: Tasks 8 to 10 +- Produces: `pub trait IdentityProvider: Send + Sync` with `descriptor`, `request`, `response` and `capture` exactly as spec 4.2 shows, and `pub trait IdentityProviderConfig: IntegrationConfig { fn build_descriptor(&self) -> Result>; }`. Also the compile-time guard `const _: Option<&dyn IdentityProvider> = None;`. + +- [ ] **Step 1: Write the failing test** + +```rust +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use http::Method; + + use super::*; + use crate::ec::identity::descriptor::Captures; + use crate::ec::identity::exchange::ProviderResponse; + use crate::ec::identity::ids::UnixTime; + use crate::ec::identity::input::{CaptureInput, ProviderConsent}; + + struct DescriptorOnly(ProviderDescriptor); + + impl IdentityProvider for DescriptorOnly { + fn descriptor(&self) -> &ProviderDescriptor { + &self.0 + } + } + + #[test] + fn default_capability_methods_return_the_not_implemented_marker() { + let provider = DescriptorOnly( + ProviderDescriptor::builder("p".parse().expect("id")) + .with_source("example.com".parse().expect("source")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .build() + .expect("should build"), + ); + let boxed: Box = Box::new(provider); + let response = ProviderResponse::for_test(http::StatusCode::OK, b"{}"); + let consent = ProviderConsent::default(); + let stored = BTreeMap::new(); + let input = CaptureInput::new( + boxed.descriptor(), + "/integrations/p/api", + &response, + &stored, + &consent, + "www.example.com", + UnixTime::from_secs(1), + ); + + let error = boxed.capture(&input).expect_err("should be the marker"); + assert!(error.current_context().is_not_implemented(), "should carry the marker"); + assert!( + !IdentityProviderError::config("other").current_context().is_not_implemented(), + "control: an ordinary Config error is not the marker" + ); + } +} +``` + +- [ ] **Step 2: Run and see it fail** + +Run: `cargo test-fastly -- ec::identity::provider` +Expected: compile error. + +- [ ] **Step 3: Implement** + +Copy the trait and config trait verbatim from spec 4.2, with these imports: + +```rust +//! The identity provider trait. + +use error_stack::Report; + +use super::descriptor::ProviderDescriptor; +use super::exchange::{ProviderRequest, ProviderResponse}; +use super::input::{CaptureInput, ResolveInput}; +use super::resolution::{IdentityProviderError, Resolution}; +use crate::settings::IntegrationConfig; + +// ... trait IdentityProvider and trait IdentityProviderConfig exactly as spec 4.2 ... + +// Keeps the trait object-safe. Fails to compile if a method gains generics. +const _: Option<&dyn IdentityProvider> = None; +``` + +Check where `IntegrationConfig` lives (`grep -rn "pub trait IntegrationConfig" crates/trusted-server-core/src`) and fix the path. Add `pub mod provider; pub use provider::{IdentityProvider, IdentityProviderConfig};` to `mod.rs`, and the file to `migration_guards.rs`. + +- [ ] **Step 4: Run and see it pass** + +Run: `cargo test-fastly -- ec::identity` and `cargo doc --no-deps --all-features -p trusted-server-core` +Expected: PASS, with no broken intra-doc links. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add the IdentityProvider trait" +``` + +### Task 12: Consent policy evaluation with the signals TS decodes today + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/policy.rs` +- Modify: `crates/trusted-server-core/src/consent/tcf.rs` (add `has_vendor_consent`) +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `policy.rs`, `consent/tcf.rs` + +**Interfaces:** + +- Consumes: `ConsentPolicy` (Task 8), `ConsentContext`, `allows_eid_persistence`, `ec_consent_granted`, and the TCF and GPP decoded forms in `consent/` +- Produces: + - `pub(crate) struct PolicySignals { dnt: bool }`. DNT is not in `ConsentContext`, so the caller passes it. + - `pub(crate) fn policy_allows(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> bool` + - `pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool`, the same test `consent_allows_server_side_auction` uses (`consent/mod.rs:298-308`) + - `pub(crate) fn has_vendor_consent(tcf: &DecodedTcf, vendor_id: u16) -> bool` in `consent/tcf.rs` + +Part 2 (Task 34) extends `policy_allows` with the sharing and targeted-advertising opt-outs, GPP section 6, and the GPC and known-child fields. Until then, a strict policy blocks on the signals already decoded: TCF purposes and vendor, the US sale opt-out (GPP and US Privacy), `Sec-GPC`, DNT, and any raw string present without a decoded form. + +- [ ] **Step 1: Write the failing tests** + +Build `ConsentContext` values the way existing consent tests do (`grep -n "fn .*consent_context\|ConsentContext {" crates/trusted-server-core/src/consent/mod.rs | head`). Reuse their helper. Then: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::descriptor::{ConsentPolicy, UsOptOut}; + + #[test] + fn strict_blocks_each_signal_it_decodes() { + let strict = ConsentPolicy::strict(); + let allowed = consent_with(Signals::default()); + assert!(policy_allows(&strict, &allowed, PolicySignals { dnt: false }), "control: nothing set"); + + for (name, signals, dnt) in [ + ("gdpr without purpose 4", Signals { gdpr: true, purposes: vec![1], ..Signals::default() }, false), + ("us sale opt-out", Signals { us_sale_opt_out: true, ..Signals::default() }, false), + ("gpc", Signals { gpc: true, ..Signals::default() }, false), + ("dnt", Signals::default(), true), + ("undecodable tc string", Signals { raw_tc_without_decode: true, ..Signals::default() }, false), + ] { + assert!( + !policy_allows(&strict, &consent_with(signals), PolicySignals { dnt }), + "strict should block: {name}" + ); + } + } + + #[test] + fn relaxations_and_vendor_consent() { + let gpc = consent_with(Signals { gpc: true, ..Signals::default() }); + assert!(policy_allows(&ConsentPolicy::strict().allowing_gpc(), &gpc, PolicySignals { dnt: false }), "gpc relaxed"); + let sale = consent_with(Signals { us_sale_opt_out: true, ..Signals::default() }); + assert!( + policy_allows(&ConsentPolicy::strict().allowing_us_opt_out(UsOptOut::Sale), &sale, PolicySignals { dnt: false }), + "sale relaxed" + ); + let gdpr = consent_with(Signals { gdpr: true, purposes: vec![1, 4], vendors: vec![9999], ..Signals::default() }); + assert!(policy_allows(&ConsentPolicy::strict().with_tcf_vendor(9999), &gdpr, PolicySignals { dnt: false }), "vendor granted"); + assert!(!policy_allows(&ConsentPolicy::strict().with_tcf_vendor(1234), &gdpr, PolicySignals { dnt: false }), "vendor missing"); + } + + #[test] + fn eid_persistence_only_ignores_provider_signals() { + let consent = consent_with(Signals { gpc: true, us_sale_opt_out: true, ..Signals::default() }); + assert!( + policy_allows(&ConsentPolicy::eid_persistence_only(), &consent, PolicySignals { dnt: true }), + "generic pull's parity policy checks only the global gates, which the coordinator applies separately" + ); + } +} +``` + +`Signals` and `consent_with` are a small test helper you write in this module. It builds a `ConsentContext` with GDPR applying, the given TCF purposes and vendor consents decoded, the US sale opt-out decoded from a US Privacy string `1YYN`, `gpc` set, and, for `raw_tc_without_decode`, `raw_tc_string = Some(..)` with no decoded TCF. Build it on whatever constructor the consent tests use. Read `consent/types.rs` first. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::policy` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Add to `consent/tcf.rs`, next to the purpose-consent helpers at about line 145: + +```rust +/// Whether the TC string grants consent to an IAB Global Vendor List vendor. +#[must_use] +pub(crate) fn has_vendor_consent(tcf: &TcfConsent, vendor_id: u16) -> bool { + tcf.vendor_consents.contains(&vendor_id) +} +``` + +Use the real decoded type name and its vendor-consent field. Read the struct at the top of `tcf.rs`. If vendor consents are stored as a bitfield, index into it instead. Add a unit test next to the existing TCF tests that decodes a fixture TC string already used in that file and asserts one vendor present and one absent. + +`policy.rs`: + +```rust +//! Evaluates a provider's [`ConsentPolicy`] against a request's consent. + +use super::descriptor::{ConsentPolicy, UsOptOut}; +use crate::consent::ConsentContext; +use crate::consent::jurisdiction::Jurisdiction; + +/// Signals the consent context does not carry. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct PolicySignals { + /// `DNT: 1` was present. + pub(crate) dnt: bool, +} + +/// Where TCF applies: `gdpr_applies`, or a jurisdiction of GDPR or unknown. +#[must_use] +pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool { + consent.gdpr_applies || matches!(consent.jurisdiction, Jurisdiction::Gdpr | Jurisdiction::Unknown) +} + +/// Whether `policy` allows a provider under this request's consent. The +/// global EC and EID-persistence gates are checked separately (eligibility +/// steps 1 and 2) and are not repeated here. +#[must_use] +pub(crate) fn policy_allows(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> bool { + if policy.is_eid_persistence_only() { + return true; + } + // Fail closed: a raw string with no decoded form blocks. + if (consent.raw_tc_string.is_some() && consent.tcf.is_none()) + || (consent.raw_gpp_string.is_some() && consent.gpp.is_none()) + || (consent.raw_us_privacy.is_some() && consent.us_privacy.is_none()) + { + return false; + } + if tcf_applies(consent) { + let Some(tcf) = consent.tcf.as_ref() else { + return false; + }; + if !policy.tcf_purposes().iter().all(|purpose| tcf.has_purpose_consent(*purpose)) { + return false; + } + if let Some(vendor) = policy.tcf_vendor() + && !crate::consent::tcf::has_vendor_consent(tcf, vendor) + { + return false; + } + } + if !policy.allows_us_opt_out(UsOptOut::Sale) && crate::consent::us_sale_opted_out(consent) { + return false; + } + if !policy.allows_gpc() && consent.gpc { + return false; + } + if !policy.allows_dnt() && signals.dnt { + return false; + } + true +} +``` + +Every `consent.*` field name and helper above (`tcf`, `gpp`, `us_privacy`, `gpc`, `jurisdiction`, `has_purpose_consent`, `us_sale_opted_out`) must be matched to the real API in `consent/`. Run `grep -n "pub fn\|pub(crate) fn\|pub [a-z_]*:" crates/trusted-server-core/src/consent/*.rs` and use what exists. If there is no single US-sale helper, combine `decode_us_sale_opt_out` (`gpp.rs:125`) and the US Privacy check (`us_privacy.rs:50-63`) in a private `fn us_sale_opt_out(consent: &ConsentContext) -> bool` in this file. Do not change any existing consent function's behaviour. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::policy` and `cargo test-fastly -- consent::` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/consent/tcf.rs crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Evaluate provider consent policies against decoded consent" +``` + +### Task 13: Revisions and keyed hashes + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/revision.rs` +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `revision.rs` + +**Interfaces:** + +- Produces: + - `pub(crate) struct IdentityKeys`, derived once per settings load, with `pub(crate) fn from_settings(settings: &Settings) -> Self`. It holds an HMAC key for each of: `revision` (legacy `pv` fallback), `invalidated` (`ps.iv`), `visitor` (rate-limit key), `fingerprint` (consent), and the AES keys `cookie_s` and `cookie_r` (Task 16 uses them). Labels: `trusted-server/identity/revision/v1`, `trusted-server/identity/iv/v1`, `trusted-server/identity/visitor/v1`, `trusted-server/identity/fingerprint/v1`, `trusted-server/ts-eids-s/key/v1`, `trusted-server/ts-eids-r/key/v1`. + - `pub(crate) struct Revision(String)`: 16 lowercase hex digits, with `as_str` + - `pub(crate) fn new_revision(random: &dyn RandomSource) -> Revision`: 8 random bytes as hex + - `pub(crate) fn record_revision(keys: &IdentityKeys, record: &KvPartnerId) -> Revision`: `rv` when present, else the first 16 hex digits of HMAC(`revision` key, `uid \0 at`) + - `pub(crate) fn invalidated_uid_hash(keys: &IdentityKeys, provider: &IdentityProviderId, uid: &str) -> String`: 16 hex + - `pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String`: 16 hex + - `pub(crate) fn consent_fingerprint(keys: &IdentityKeys, bits: &str) -> String`: 16 hex. Task 34 builds `bits`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::input::SeededRandom; + use crate::ec::kv_types::KvPartnerId; + use crate::test_support::tests::create_test_settings; + + fn keys() -> IdentityKeys { + IdentityKeys::from_settings(&create_test_settings()) + } + + #[test] + fn new_revisions_are_hex_and_differ() { + let random = SeededRandom::new(7); + let first = new_revision(&random); + let second = new_revision(&random); + assert_eq!(first.as_str().len(), 16, "16 hex digits"); + assert!(first.as_str().bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), "lowercase hex"); + assert_ne!(first, second, "two writes of the same UID get different revisions"); + } + + #[test] + fn record_revision_prefers_rv_and_hashes_legacy_records() { + let keys = keys(); + let with_rv = KvPartnerId { rv: Some("0123456789abcdef".to_owned()), ..KvPartnerId::new("u") }; + assert_eq!(record_revision(&keys, &with_rv).as_str(), "0123456789abcdef", "should use rv"); + + let legacy_a = KvPartnerId::browser("u", Some(10)); + let legacy_b = KvPartnerId::browser("u", Some(11)); + assert_eq!(record_revision(&keys, &legacy_a), record_revision(&keys, &legacy_a.clone()), "stable"); + assert_ne!(record_revision(&keys, &legacy_a), record_revision(&keys, &legacy_b), "at changes it"); + } + + #[test] + fn hashes_are_keyed_and_namespaced() { + let keys = keys(); + let provider: IdentityProviderId = "p".parse().expect("id"); + let iv = invalidated_uid_hash(&keys, &provider, "u"); + let visitor = visitor_key(&keys, "u"); + assert_eq!(iv.len(), 16, "iv length"); + assert_ne!(iv, visitor, "different labels give different hashes for the same input"); + + let mut other_settings = create_test_settings(); + other_settings.ec.passphrase = crate::redacted::Redacted::new("another-example-passphrase-value".to_owned()); + let other = IdentityKeys::from_settings(&other_settings); + assert_ne!(invalidated_uid_hash(&other, &provider, "u"), iv, "keyed by the passphrase"); + } +} +``` + +Find the settings test helper with `grep -rn "fn create_test_settings" crates/trusted-server-core/src`, and fix the import path to match. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::revision` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +```rust +//! Record revisions and keyed hashes of identifiers. + +use hmac::{Hmac, Mac as _}; +use sha2::Sha256; + +use super::ids::IdentityProviderId; +use super::input::RandomSource; +use crate::ec::kv_types::KvPartnerId; +use crate::settings::Settings; + +type HmacSha256 = Hmac; + +/// Keys derived from `ec.passphrase`, once per settings load. +#[derive(Clone)] +pub(crate) struct IdentityKeys { + revision: [u8; 32], + invalidated: [u8; 32], + visitor: [u8; 32], + fingerprint: [u8; 32], + pub(crate) cookie_s: [u8; 32], + pub(crate) cookie_r: [u8; 32], +} + +impl core::fmt::Debug for IdentityKeys { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter.write_str("IdentityKeys([redacted])") + } +} + +fn derive(passphrase: &[u8], label: &[u8]) -> [u8; 32] { + let mut mac = HmacSha256::new_from_slice(passphrase).expect("should accept any HMAC key length"); + mac.update(label); + mac.finalize().into_bytes().into() +} + +fn keyed_hex16(key: &[u8; 32], parts: &[&[u8]]) -> String { + let mut mac = HmacSha256::new_from_slice(key).expect("should accept a 32-byte key"); + for (index, part) in parts.iter().enumerate() { + if index > 0 { + mac.update(b"\0"); + } + mac.update(part); + } + hex::encode(&mac.finalize().into_bytes()[..8]) +} + +impl IdentityKeys { + /// Derives every identity key from `ec.passphrase`. + #[must_use] + pub(crate) fn from_settings(settings: &Settings) -> Self { + let passphrase = settings.ec.passphrase.expose().as_bytes(); + Self { + revision: derive(passphrase, b"trusted-server/identity/revision/v1"), + invalidated: derive(passphrase, b"trusted-server/identity/iv/v1"), + visitor: derive(passphrase, b"trusted-server/identity/visitor/v1"), + fingerprint: derive(passphrase, b"trusted-server/identity/fingerprint/v1"), + cookie_s: derive(passphrase, b"trusted-server/ts-eids-s/key/v1"), + cookie_r: derive(passphrase, b"trusted-server/ts-eids-r/key/v1"), + } + } +} + +/// A record's opaque revision: 16 lowercase hex digits. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct Revision(String); + +impl Revision { + /// Wraps a stored `rv` or `pv`. Callers have validated the format. + #[must_use] + pub(crate) fn from_stored(value: &str) -> Self { + Self(value.to_owned()) + } + + /// The hex digits. + #[must_use] + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +/// A fresh random revision, drawn on every write of a record. +#[must_use] +pub(crate) fn new_revision(random: &dyn RandomSource) -> Revision { + let mut bytes = [0u8; 8]; + random.fill(&mut bytes); + Revision(hex::encode(bytes)) +} + +/// The revision of a stored record: its `rv`, or for a legacy record without +/// one, the keyed hash of its `(uid, at)` until its next write. +#[must_use] +pub(crate) fn record_revision(keys: &IdentityKeys, record: &KvPartnerId) -> Revision { + if let Some(rv) = &record.rv { + return Revision(rv.clone()); + } + let at = record.at.unwrap_or_default().to_string(); + Revision(keyed_hex16(&keys.revision, &[record.uid.as_bytes(), at.as_bytes()])) +} + +/// The keyed hash stored in `ps.iv` for a UID the owner invalidated. +#[must_use] +pub(crate) fn invalidated_uid_hash(keys: &IdentityKeys, provider: &IdentityProviderId, uid: &str) -> String { + keyed_hex16(&keys.invalidated, &[provider.as_str().as_bytes(), uid.as_bytes()]) +} + +/// The per-visitor rate-limit key component, from the full EC ID. +#[must_use] +pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String { + keyed_hex16(&keys.visitor, &[ec_id.as_bytes()]) +} + +/// The consent fingerprint over the signals a provider's policy reads. +#[must_use] +pub(crate) fn consent_fingerprint(keys: &IdentityKeys, bits: &str) -> String { + keyed_hex16(&keys.fingerprint, &[bits.as_bytes()]) +} +``` + +Check that `Settings.ec.passphrase` is `Redacted` with `.expose()`. `pull_sync_marker.rs:293` uses exactly that. Register the module in `mod.rs` (`pub(crate) mod revision;`) and in `migration_guards.rs`. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::revision` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add record revisions and keyed identity hashes" +``` + +--- + +### Task 14: The provider catalog and the shared servability check + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/catalog.rs` +- Create: `crates/trusted-server-core/src/ec/identity/servable.rs` +- Modify: `crates/trusted-server-core/src/ec/eids.rs` (`resolve_partner_ids`, and the stale module comment at lines 1-5) +- Modify: callers of `resolve_partner_ids`: `auction/endpoints.rs:465`, `ec/admin.rs:485` +- Modify: `crates/trusted-server-core/src/ec/identify.rs` (`entry.ids` read at about lines 99-103) +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `servable.rs`, `ec/eids.rs`, `ec/identify.rs` + +**Interfaces:** + +- Produces: + - `pub(crate) struct ProviderCatalog` with `pub(crate) fn new(providers: Vec>, owners: BTreeMap) -> Self`, `get(&IdentityProviderId) -> Option<&Arc>`, `providers() -> impl Iterator>`, `owner_of(&SourceDomain) -> Option<&IdentityProviderId>`, `is_configured(&IdentityProviderId) -> bool`, `configured_sources() -> &[SourceDomain]` (sorted and de-duplicated, the order that `ts-eids-s` indexes use), `fingerprint() -> &str` (`cf`: the first 8 hex digits of SHA-256 over the sorted `provider|source|owner` lines), and `ProviderCatalog::empty()`. Part 1 builds it from generic pull providers only (Task 22), with no owners. + - `pub(crate) struct ServeContext<'a> { now: UnixTime, consent: &'a ConsentContext, signals: PolicySignals, catalog: &'a ProviderCatalog, opted_out: &'a BTreeSet }` + - `pub(crate) fn is_servable(context: &ServeContext<'_>, source: &SourceDomain, record: &KvPartnerId) -> bool` + - `resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry, serve: &ServeContext<'_>) -> Vec`. The new third argument is required. `ResolvedPartnerId` gains `atype` from the record (`record.atype.unwrap_or(partner.openrtb_atype)`), `ext: Option`, and `writer: Option`. + +The rule (spec 6.1 and 7.1), applied in this order: + +1. An empty UID is not servable. +2. `exp` at or before `now` is not servable. +3. A record with a writer `w`: + - The writer must be in the catalog. A disabled or unconfigured writer means not servable. + - The writer must not be in `opted_out`, which is the union of KV `ps.o` and the `ts-eids-r` opt-outs. The caller computes it. + - The writer's consent policy must pass under this request's consent (`policy_allows`). +4. An owned source (the catalog has an owner): + - The owner's policy must pass. + - If the owner's descriptor marks the source as expiring, the record must have `w == owner` and a set, unexpired `exp`. This is the rollback rule. +5. Anything else, meaning no writer and an unowned source, is servable. The caller already applied the generic gates. + +- [ ] **Step 1: Write the failing tests** + +In `servable.rs`: + +```rust +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + use std::sync::Arc; + + use super::*; + use crate::ec::identity::descriptor::{ConsentPolicy, ProviderDescriptor, Resolves}; + use crate::ec::identity::provider::IdentityProvider; + use crate::ec::kv_types::KvPartnerId; + + struct Fixed(ProviderDescriptor); + impl IdentityProvider for Fixed { + fn descriptor(&self) -> &ProviderDescriptor { + &self.0 + } + } + + fn provider(id: &str, source: &str, policy: ConsentPolicy, expiring: bool) -> Arc { + let mut builder = ProviderDescriptor::builder(id.parse().expect("id")) + .with_source(source.parse().expect("source")) + .with_resolves(Resolves::new().with_endpoint("https://api.example.com")) + .with_consent_policy(policy); + if expiring { + builder = builder.with_expiring_ids(); + } + Arc::new(Fixed(builder.build().expect("should build"))) + } + + fn record(writer: Option<&str>, exp: Option) -> KvPartnerId { + KvPartnerId { w: writer.map(str::to_owned), exp, ..KvPartnerId::new("u") } + } + + fn check(catalog: &ProviderCatalog, opted_out: &BTreeSet, source: &str, record: &KvPartnerId) -> bool { + let consent = test_consent_allowing_everything(); + let context = ServeContext { + now: UnixTime::from_secs(100), + consent: &consent, + signals: PolicySignals::default(), + catalog, + opted_out, + }; + is_servable(&context, &source.parse().expect("source"), record) + } + + #[test] + fn expiry_writer_presence_and_opt_out() { + let catalog = ProviderCatalog::new( + vec![provider("p", "a.example.com", ConsentPolicy::eid_persistence_only(), false)], + BTreeMap::new(), + ); + let none = BTreeSet::new(); + assert!(check(&catalog, &none, "a.example.com", &record(Some("p"), Some(101))), "control: valid"); + assert!(!check(&catalog, &none, "a.example.com", &record(Some("p"), Some(100))), "expired at exp"); + assert!(!check(&catalog, &none, "a.example.com", &record(Some("gone"), None)), "unknown writer"); + let opted: BTreeSet<_> = ["p".parse().expect("id")].into_iter().collect(); + assert!(!check(&catalog, &opted, "a.example.com", &record(Some("p"), None)), "opted-out writer"); + assert!(check(&catalog, &none, "b.example.com", &record(None, None)), "browser record, unowned"); + } + + #[test] + fn rollback_rule_for_expiring_owned_sources() { + let owner: IdentityProviderId = "owner".parse().expect("id"); + let catalog = ProviderCatalog::new( + vec![provider("owner", "a.example.com", ConsentPolicy::eid_persistence_only(), true)], + [("a.example.com".parse().expect("source"), owner)].into_iter().collect(), + ); + let none = BTreeSet::new(); + assert!(check(&catalog, &none, "a.example.com", &record(Some("owner"), Some(200))), "control: owner value"); + assert!(!check(&catalog, &none, "a.example.com", &record(None, None)), "browser-seeded, never from server"); + assert!(!check(&catalog, &none, "a.example.com", &record(Some("owner"), None)), "stripped exp"); + } + + #[test] + fn writer_policy_applies_to_unowned_sources() { + let strict = ConsentPolicy::strict().with_tcf_vendor(9999); + let catalog = ProviderCatalog::new(vec![provider("p", "a.example.com", strict, false)], BTreeMap::new()); + let consent = test_consent_gdpr_without_vendor(9999); + let none = BTreeSet::new(); + let context = ServeContext { + now: UnixTime::from_secs(100), + consent: &consent, + signals: PolicySignals::default(), + catalog: &catalog, + opted_out: &none, + }; + let source = "a.example.com".parse().expect("source"); + assert!(!is_servable(&context, &source, &record(Some("p"), None)), "writer's vendor denied"); + assert!(is_servable(&context, &source, &record(None, None)), "control: browser value unaffected"); + } +} +``` + +`test_consent_allowing_everything` and `test_consent_gdpr_without_vendor` reuse the helper you wrote in Task 12. Move it to `ec/identity/test_support.rs` now (create the file with `#![cfg(any(test, feature = "test-utils"))]` items and `pub(crate) mod test_support;` in `mod.rs`) so every identity test can share it. + +In `ec/eids.rs` tests, extend the existing `resolve_partner_ids` test (line 195) to pass a `ServeContext` built from `ProviderCatalog::empty()`, and add: + +```rust + #[test] + fn resolve_partner_ids_skips_expired_records_and_uses_stored_atype() { + let registry = registry_with(&["a.example.com", "b.example.com"]); + let mut entry = KvEntry::minimal("a.example.com", "ua", 1); + entry.ids.insert( + "b.example.com".to_owned(), + KvPartnerId { exp: Some(50), ..KvPartnerId::new("ub") }, + ); + entry.ids.get_mut("a.example.com").expect("a").atype = Some(3); + + let resolved = resolve_partner_ids(®istry, &entry, &serve_context_at(100)); + + assert_eq!(resolved.len(), 1, "should drop the expired b"); + assert_eq!(resolved[0].source_domain, "a.example.com", "should keep a"); + assert_eq!(resolved[0].openrtb_atype, 3, "should prefer the stored atype"); + } +``` + +Write `registry_with` and `serve_context_at` as local helpers. `registry_with` follows the existing test's registry setup, and `serve_context_at` uses `ProviderCatalog::empty()` with permissive consent. + +In `ec/identify.rs` tests, add one that stores a row with an expired record and a valid one and checks that the identify response lists only the valid one. Copy the setup from the nearest existing identify test (`grep -n "fn .*identify" crates/trusted-server-core/src/ec/identify.rs`). + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::servable`, `cargo test-fastly -- ec::eids`, `cargo test-fastly -- ec::identify` +Expected: compile errors, then the identify test fails because it returns the expired ID. + +- [ ] **Step 3: Implement `catalog.rs`** + +```rust +//! The enabled identity providers and source ownership. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use sha2::{Digest as _, Sha256}; + +use super::ids::{IdentityProviderId, SourceDomain}; +use super::provider::IdentityProvider; + +/// Enabled providers, source owners, and the derived source index and `cf`. +/// Built once per settings load. +#[derive(Clone)] +pub(crate) struct ProviderCatalog { + providers: BTreeMap>, + owners: BTreeMap, + sources: Vec, + fingerprint: String, +} + +impl ProviderCatalog { + /// A catalog with no providers. + #[must_use] + pub(crate) fn empty() -> Self { + Self::new(Vec::new(), BTreeMap::new()) + } + + /// Builds the catalog. Validation (ownership, duplicates) happens in + /// config validation before this is called. + #[must_use] + pub(crate) fn new( + providers: Vec>, + owners: BTreeMap, + ) -> Self { + let providers: BTreeMap<_, _> = providers + .into_iter() + .map(|provider| (provider.descriptor().id().clone(), provider)) + .collect(); + let sources: Vec = providers + .values() + .flat_map(|provider| provider.descriptor().sources().iter().cloned()) + .chain(owners.keys().cloned()) + .collect::>() + .into_iter() + .collect(); + let mut hasher = Sha256::new(); + for (id, provider) in &providers { + for source in provider.descriptor().sources() { + let owner = owners.get(source).map(IdentityProviderId::as_str).unwrap_or_default(); + hasher.update(format!("{id}|{source}|{owner}\n").as_bytes()); + } + } + let fingerprint = hex::encode(&hasher.finalize()[..4]); + Self { providers, owners, sources, fingerprint } + } + + /// A provider by ID. + #[must_use] + pub(crate) fn get(&self, id: &IdentityProviderId) -> Option<&Arc> { + self.providers.get(id) + } + + /// Every provider, ordered by ID. + pub(crate) fn providers(&self) -> impl Iterator> { + self.providers.values() + } + + /// Whether `id` is an enabled provider. + #[must_use] + pub(crate) fn is_configured(&self, id: &IdentityProviderId) -> bool { + self.providers.contains_key(id) + } + + /// The owner of `source`, if any. + #[must_use] + pub(crate) fn owner_of(&self, source: &SourceDomain) -> Option<&IdentityProviderId> { + self.owners.get(source) + } + + /// Every configured source, sorted. Indexes into this list are the + /// `s` values of `ts-eids-s`. + #[must_use] + pub(crate) fn configured_sources(&self) -> &[SourceDomain] { + &self.sources + } + + /// `cf`: 8 hex digits over providers, sources and owners. + #[must_use] + pub(crate) fn fingerprint(&self) -> &str { + &self.fingerprint + } +} +``` + +- [ ] **Step 4: Implement `servable.rs`** + +```rust +//! The one servability check shared by the bidstream, the admin view and +//! `/_ts/api/v1/identify` (spec 7.1). + +use std::collections::BTreeSet; + +use super::catalog::ProviderCatalog; +use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::policy::{PolicySignals, policy_allows}; +use crate::consent::ConsentContext; +use crate::ec::kv_types::KvPartnerId; + +/// What a servability decision needs from the request. +pub(crate) struct ServeContext<'a> { + pub(crate) now: UnixTime, + pub(crate) consent: &'a ConsentContext, + pub(crate) signals: PolicySignals, + pub(crate) catalog: &'a ProviderCatalog, + /// KV `ps.o` plus `ts-eids-r` opt-outs. + pub(crate) opted_out: &'a BTreeSet, +} + +/// Whether a stored record may be sent from the server. +#[must_use] +pub(crate) fn is_servable(context: &ServeContext<'_>, source: &SourceDomain, record: &KvPartnerId) -> bool { + if record.uid.is_empty() { + return false; + } + if record.exp.is_some_and(|exp| context.now.as_secs() >= exp) { + return false; + } + let writer = record.w.as_deref().and_then(|writer| writer.parse::().ok()); + if record.w.is_some() { + let Some(writer) = writer.as_ref() else { + return false; + }; + let Some(provider) = context.catalog.get(writer) else { + return false; + }; + if context.opted_out.contains(writer) + || !policy_allows(provider.descriptor().consent_policy(), context.consent, context.signals) + { + return false; + } + } + if let Some(owner) = context.catalog.owner_of(source) { + let Some(owner_provider) = context.catalog.get(owner) else { + return false; + }; + let descriptor = owner_provider.descriptor(); + if !policy_allows(descriptor.consent_policy(), context.consent, context.signals) { + return false; + } + if descriptor.is_expiring(source) && (writer.as_ref() != Some(owner) || record.exp.is_none()) { + return false; + } + } + true +} +``` + +- [ ] **Step 5: Switch the readers** + +In `ec/eids.rs`, replace the module comment at lines 1-5 with one that says KV IDs are filtered by the shared servability check in `ec::identity::servable`. Add `serve: &ServeContext<'_>` to `resolve_partner_ids`, and inside the loop add `if !is_servable(serve, &source, partner_uid) { continue; }` after parsing `source_domain` into a `SourceDomain`. Skip any key that does not parse. Take `atype` from the record and keep `ext` and the writer on `ResolvedPartnerId` (Task 28 adds provenance in `to_eids`). Update `to_eids` to emit the UID-level `ext` when present. + +At each caller, build a `ServeContext`. Until Task 23 puts the catalog and `ts-eids-r` opt-outs on `EcContext`, pass `ProviderCatalog::empty()` and the KV `ps.o` keys as `opted_out`. Task 23 replaces both. `now` comes from `checked_current_timestamp()`. If that fails, resolve no KV IDs, because a zero time would make every expiry check pass (spec 4.4). `consent` is the request's `ConsentContext`. DNT is read from the request header there (`auction/routing.rs:336` shows how). + +In `ec/identify.rs`, replace the direct `entry.ids` read with `resolve_partner_ids`, using the same context built from the request's consent, never the row's stored `consent` (spec 7.1). + +- [ ] **Step 6: Run and see them pass** + +Run: `cargo test-fastly -- ec::` and `cargo test-fastly -- auction::endpoints` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Filter served KV IDs through one servability check" -m "Expired IDs, IDs from disabled or opted-out writers, and IDs whose writer or owner policy fails are no longer sent to bidders, the admin view or /identify. /identify uses the request's consent, never the row's stored snapshot." +``` + +### Task 15: `IdentityMutation` and its merge into a live row + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/mutation.rs` +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `mutation.rs` + +**Interfaces:** + +- Consumes: `KvEntry`, `KvPartnerId`, `KvProviderState`, `KvOptOut` (Task 6), `Revision`, `new_revision`, `record_revision`, `invalidated_uid_hash`, `IdentityKeys` (Task 13), `ProviderCatalog` (Task 14), `RandomSource` (Task 9) +- Produces: + - `pub(crate) enum PartnerIdChange`: + - `FillMissing { source, record: KvPartnerId, writer: Option, staged_at: Option }` + - `OwnerWrite { source, record, writer, expected: Option, staged_at: Option }` + - `OwnerRemove { source, writer, expected: Revision }` + - `WriterRemove { source, writer, staged_at: Option }` + - `pub(crate) enum ExpectedState { NotRead, Absent, Json(String) }` + - `pub(crate) enum StateUpdate { Keep, Replace(serde_json::Value), Clear }` + - `pub(crate) struct ProviderStateChange { n: Option, f: Option, opt_out: Option, opt_out_clear: Option, iv_add: Vec, st: StateUpdate }`, deriving `Default` + - `pub(crate) struct ProviderResult { provider, ids: Vec, state: ProviderStateChange, expected_state: ExpectedState }` + - `pub(crate) struct IdentityMutation`, deriving `Default`, with `add_change(PartnerIdChange)`, `add_result(ProviderResult)`, `merge_state(IdentityProviderId, ProviderStateChange)`, `set_cluster_size(u32)`, `is_empty()`, `has_removals()` + - `pub(crate) struct ApplyContext<'a> { now: u64, keys: &'a IdentityKeys, random: &'a dyn RandomSource, catalog: &'a ProviderCatalog }` + - `pub(crate) struct Applied { entry: KvEntry, changed: bool, removals_only: bool }` + - `pub(crate) fn apply(mutation: &IdentityMutation, live: &KvEntry, context: &ApplyContext<'_>) -> Option`. `None` means drop everything: the live row is a tombstone, or even the removals-only result failed validation. + +Apply order (spec 5.4). Copy it into the doc comment of `apply`: + +1. A tombstone (`consent.ok == false`) gives `None`. +2. Removals. Every `OwnerRemove` whose `expected` equals the live record's revision. Every `WriterRemove` whose live record has `w == writer` and, when `staged_at` is set, `at <= staged_at` (equal seconds go to the removal). Each new opt-out applies when `at > live.o.at` (or there is no live `o`) and `at >= live.oc` (or there is no live `oc`). Applying it removes every live record that provider wrote, at any revision, and clears its `st`. Then each staged clear (`opt_out_clear`) that is strictly later than the provider's latest opt-out `at`, live or staged, is stored as the live `oc`. A clear that is not later is dropped. `iv_add` hashes are appended, de-duplicated, and trimmed to the newest 4. +3. Positive changes. Skip every change by a provider whose (post-step-2) live `o` is set and not cleared (`oc` strictly later than `o.at`), and skip a staged change whose `staged_at <= o.at` even after a clear. A `FillMissing` applies when the source is absent, or its record is expired, or its writer is not configured. It is skipped when its UID's hash is in the source owner's `ps.iv`. An `OwnerWrite` applies when the live revision equals `expected`, or when `expected` is `None` and the source has no valid record with `w == writer`. A written record gets `w = writer`, a fresh `rv`, and `at = record.at.or(now)`. Each `ProviderResult` applies only when every one of its ID changes would apply on its own and the live `ps.st` matches `expected_state`. Its state change then applies with its IDs. Otherwise the whole result is dropped. +4. `ps` changes: `n` takes the later value, `f` is replaced, `st` is replaced or cleared. Then `cluster_size`. Then prune the `ps` entries of providers the catalog does not hold. Then drop expired records, except refresh anchors: the source's owner holds a `st` and the record's `ru` has not passed. +5. `validate()`. On failure, rebuild from the live entry with only step 2 applied (removals, opt-outs, clears, `iv`) and validate again. If that also fails, give `None` and log. +6. `changed` is whether the result differs from `live`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + + use serde_json::json; + + use super::*; + use crate::ec::identity::catalog::ProviderCatalog; + use crate::ec::identity::input::SeededRandom; + use crate::ec::identity::revision::IdentityKeys; + use crate::ec::identity::test_support::{fixed_provider, test_keys}; + use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId, KvProviderState}; + + const NOW: u64 = 1_000; + + fn src(name: &str) -> SourceDomain { + name.parse().expect("source") + } + + fn pid(name: &str) -> IdentityProviderId { + name.parse().expect("id") + } + + fn catalog() -> ProviderCatalog { + ProviderCatalog::new( + vec![fixed_provider("owner", &["a.example.com"]), fixed_provider("other", &["b.example.com"])], + [(src("a.example.com"), pid("owner"))].into_iter().collect(), + ) + } + + fn run(mutation: &IdentityMutation, live: &KvEntry) -> Option { + let keys = test_keys(); + let random = SeededRandom::new(1); + let catalog = catalog(); + apply(mutation, live, &ApplyContext { now: NOW, keys: &keys, random: &random, catalog: &catalog }) + } + + fn owned(uid: &str, rv: &str) -> KvPartnerId { + KvPartnerId { w: Some("owner".to_owned()), rv: Some(rv.to_owned()), at: Some(900), ..KvPartnerId::new(uid) } + } + + #[test] + fn tombstone_drops_everything() { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::FillMissing { + source: src("b.example.com"), + record: KvPartnerId::browser("u", Some(NOW)), + writer: None, + staged_at: None, + }); + assert!(run(&mutation, &KvEntry::tombstone(1)).is_none(), "should drop on a tombstone"); + } + + #[test] + fn fill_missing_fills_absent_and_expired_only() { + let mut live = KvEntry::minimal("b.example.com", "existing", 1); + live.ids.insert("c.example.com".to_owned(), KvPartnerId { exp: Some(NOW), ..KvPartnerId::new("old") }); + let mut mutation = IdentityMutation::default(); + for source in ["b.example.com", "c.example.com", "d.example.com"] { + mutation.add_change(PartnerIdChange::FillMissing { + source: src(source), + record: KvPartnerId::browser("new", Some(NOW)), + writer: None, + staged_at: None, + }); + } + + let applied = run(&mutation, &live).expect("should apply"); + + assert!(applied.changed, "should change"); + assert_eq!(applied.entry.ids["b.example.com"].uid, "existing", "should not replace a live value"); + assert_eq!(applied.entry.ids["c.example.com"].uid, "new", "should replace an expired value"); + assert_eq!(applied.entry.ids["d.example.com"].uid, "new", "should fill an absent source"); + } + + #[test] + fn owner_write_checks_revision_and_gets_a_fresh_rv() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); + + let mut stale = IdentityMutation::default(); + stale.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId::new("v2"), + writer: pid("owner"), + expected: Some(Revision::from_stored("2222222222222222")), + staged_at: None, + }); + let applied = run(&stale, &live).expect("should apply"); + assert!(!applied.changed, "a stale revision loses to the newer write"); + + let mut fresh = IdentityMutation::default(); + fresh.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId::new("v2"), + writer: pid("owner"), + expected: Some(Revision::from_stored("1111111111111111")), + staged_at: None, + }); + let applied = run(&fresh, &live).expect("should apply"); + let record = &applied.entry.ids["a.example.com"]; + assert_eq!(record.uid, "v2", "should write"); + assert_eq!(record.w.as_deref(), Some("owner"), "should set the writer"); + assert_ne!(record.rv.as_deref(), Some("1111111111111111"), "should draw a new rv"); + assert_eq!(record.at, Some(NOW), "should stamp at"); + } + + #[test] + fn browser_seeded_value_never_blocks_the_owner() { + let mut live = KvEntry::minimal("a.example.com", "seeded", 1); + live.ids.get_mut("a.example.com").expect("a").at = Some(5); + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId::new("owned"), + writer: pid("owner"), + expected: None, + staged_at: None, + }); + let applied = run(&mutation, &live).expect("should apply"); + assert_eq!(applied.entry.ids["a.example.com"].uid, "owned", "owner takes over a seeded value"); + } + + #[test] + fn removals_win_ties_and_opt_out_removes_everything_the_writer_wrote() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); + live.ids.insert( + "c.example.com".to_owned(), + KvPartnerId { w: Some("owner".to_owned()), at: Some(950), ..KvPartnerId::new("unowned-by-owner") }, + ); + + let mut mutation = IdentityMutation::default(); + mutation.merge_state( + pid("owner"), + ProviderStateChange { opt_out: Some(KvOptOut { at: NOW, fp: "0".repeat(16) }), ..ProviderStateChange::default() }, + ); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId::new("v2"), + writer: pid("owner"), + expected: None, + staged_at: Some(NOW), + }); + + let applied = run(&mutation, &live).expect("should apply"); + + assert!(!applied.entry.ids.contains_key("a.example.com"), "should remove the owned record"); + assert!(!applied.entry.ids.contains_key("c.example.com"), "should remove the unowned record it wrote"); + assert_eq!(applied.entry.ids["b.example.com"].uid, "x", "control: other writers untouched"); + assert_eq!(applied.entry.ps["owner"].o.as_ref().map(|o| o.at), Some(NOW), "should store the opt-out"); + } + + #[test] + fn writer_remove_without_revision_respects_staged_at() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); + let remove = |staged_at| { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::WriterRemove { + source: src("a.example.com"), + writer: pid("owner"), + staged_at: Some(staged_at), + }); + run(&mutation, &live).expect("should apply") + }; + assert!(remove(900).entry.ids.get("a.example.com").is_none(), "same second: the removal wins"); + assert!(remove(899).entry.ids.get("a.example.com").is_some(), "re-acquired after sa: kept"); + } + + #[test] + fn opt_out_and_clear_in_the_same_second_leave_it_opted_out() { + let live = KvEntry::minimal("b.example.com", "x", 1); + let mut mutation = IdentityMutation::default(); + mutation.merge_state( + pid("owner"), + ProviderStateChange { + opt_out: Some(KvOptOut { at: NOW, fp: "0".repeat(16) }), + opt_out_clear: Some(NOW), + ..ProviderStateChange::default() + }, + ); + let applied = run(&mutation, &live).expect("should apply"); + assert!(applied.entry.ps["owner"].o.is_some(), "should stay opted out"); + assert_eq!(applied.entry.ps["owner"].oc, None, "should drop a clear that is not later"); + } + + #[test] + fn invalidated_uid_is_not_reseeded() { + let keys = test_keys(); + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ps.insert( + "owner".to_owned(), + KvProviderState { + iv: vec![crate::ec::identity::revision::invalidated_uid_hash(&keys, &pid("owner"), "rejected")], + ..KvProviderState::default() + }, + ); + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::FillMissing { + source: src("a.example.com"), + record: KvPartnerId::browser("rejected", Some(NOW)), + writer: None, + staged_at: None, + }); + let applied = run(&mutation, &live).expect("should apply"); + assert!(!applied.entry.ids.contains_key("a.example.com"), "should not re-seed the rejected UID"); + } + + #[test] + fn provider_result_applies_atomically() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert("a.example.com".to_owned(), owned("newer", "3333333333333333")); + live.ps.insert("owner".to_owned(), KvProviderState { st: Some(json!({ "token": "B" })), ..KvProviderState::default() }); + + let mut mutation = IdentityMutation::default(); + mutation.add_result(ProviderResult { + provider: pid("owner"), + ids: vec![PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId::new("mine"), + writer: pid("owner"), + expected: Some(Revision::from_stored("1111111111111111")), + staged_at: None, + }], + state: ProviderStateChange { st: StateUpdate::Replace(json!({ "token": "A" })), f: Some(0), ..ProviderStateChange::default() }, + expected_state: ExpectedState::Json(r#"{"token":"A0"}"#.to_owned()), + }); + + let applied = run(&mutation, &live).expect("should apply"); + + assert_eq!(applied.entry.ids["a.example.com"].uid, "newer", "should keep the other writer's ID"); + assert_eq!(applied.entry.ps["owner"].st, Some(json!({ "token": "B" })), "with that writer's state"); + assert!(!applied.changed, "nothing else changed"); + } + + #[test] + fn refresh_anchor_survives_until_ru_or_state_loss() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { exp: Some(NOW - 1), ru: Some(NOW + 100), ..owned("expired", "1111111111111111") }, + ); + live.ps.insert("owner".to_owned(), KvProviderState { st: Some(json!({ "r": 1 })), ..KvProviderState::default() }); + let mut unrelated = IdentityMutation::default(); + unrelated.set_cluster_size(1); + + let kept = run(&unrelated, &live).expect("should apply"); + assert!(kept.entry.ids.contains_key("a.example.com"), "anchor kept while state lives"); + + live.ps.get_mut("owner").expect("owner").st = None; + let dropped = run(&unrelated, &live).expect("should apply"); + assert!(!dropped.entry.ids.contains_key("a.example.com"), "anchor dropped without state"); + } + + #[test] + fn unconfigured_provider_state_is_pruned() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ps.insert("retired".to_owned(), KvProviderState { n: Some(5), ..KvProviderState::default() }); + live.ps.insert("other".to_owned(), KvProviderState { n: Some(5), ..KvProviderState::default() }); + let mut mutation = IdentityMutation::default(); + mutation.set_cluster_size(1); + let applied = run(&mutation, &live).expect("should apply"); + assert!(!applied.entry.ps.contains_key("retired"), "should prune"); + assert!(applied.entry.ps.contains_key("other"), "control: configured kept"); + } + + #[test] + fn invalid_result_falls_back_to_removals_only() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::FillMissing { + source: src("d.example.com"), + record: KvPartnerId::browser("x".repeat(600), Some(NOW)), + writer: None, + staged_at: None, + }); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + writer: pid("owner"), + expected: Revision::from_stored("1111111111111111"), + }); + let applied = run(&mutation, &live).expect("should apply removals"); + assert!(applied.removals_only, "should fall back"); + assert!(!applied.entry.ids.contains_key("a.example.com"), "removal applied"); + assert!(!applied.entry.ids.contains_key("d.example.com"), "invalid fill dropped"); + } +} +``` + +Add `fixed_provider(id: &str, sources: &[&str]) -> Arc` (a descriptor-only provider with a dummy `https://api.example.com` endpoint and the `eid_persistence_only` policy) and `test_keys() -> IdentityKeys` to `ec/identity/test_support.rs`. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::mutation` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Write `mutation.rs` with the types listed under Interfaces, then implement `apply` as five private steps over a cloned `KvEntry`. Use one function per step so each rule can be read on its own: + +```rust +pub(crate) fn apply(mutation: &IdentityMutation, live: &KvEntry, context: &ApplyContext<'_>) -> Option { + if !live.consent.ok { + return None; + } + + let mut entry = live.clone(); + apply_removals(&mut entry, mutation, context); + let removals_only = entry.clone(); + + apply_positive(&mut entry, mutation, context); + apply_state_and_cleanup(&mut entry, mutation, context); + + let (entry, removals_only_used) = match entry.validate() { + Ok(()) => (entry, false), + Err(message) => { + log::warn!("Identity mutation produced an invalid row, keeping removals only: {message}"); + let mut fallback = removals_only; + prune_and_drop_expired(&mut fallback, context); + if let Err(message) = fallback.validate() { + log::warn!("Identity mutation removals-only row is invalid, dropping mutation: {message}"); + return None; + } + (fallback, true) + } + }; + + let changed = &entry != live; + Some(Applied { entry, changed, removals_only: removals_only_used }) +} +``` + +- `apply_removals`: iterate `OwnerRemove` and `WriterRemove` from `partner_ids` and from every `ProviderResult`'s state-free removals. When building a `ProviderResult`, the coordinator always puts removals in `partner_ids`, never in `ProviderResult.ids` (spec 5.4 comment on `ProviderResult`). Then opt-outs, clears and `iv` from `provider_state` and from each result's `state`. A revision is compared with `record_revision(context.keys, live_record)`. +- `apply_positive`: compute `opted_out(provider)` from the post-removal `ps`. For each plain change, call `fn change_applies(entry, change, context) -> bool` and, when true, `fn write_change(entry, change, context)`. For each `ProviderResult`, first check that every ID change applies and that `expected_state` matches `entry.ps[provider].st` serialized with `serde_json::to_string`. `ExpectedState::NotRead` always matches. Then apply all of them and the result's `state` together. +- `apply_state_and_cleanup`: merge `provider_state` (`n` = max, `f` replaced when `Some`, `st` per `StateUpdate`), set `entry.network`'s `cluster_size` when given (match the existing `KvNetwork` field), then call `prune_and_drop_expired`. +- `prune_and_drop_expired`: `entry.ps.retain(|id, _| parsed id is configured)`. Drop records with `exp <= now` unless they are anchors: the catalog's owner of that source has `entry.ps[owner].st.is_some()` and `record.ru.is_some_and(|ru| now < ru)`. + +`IdentityMutation::merge_state(provider, change)` merges into any existing change for the same provider: later `n`, last `f`, union of `iv_add`, the later `opt_out`, the later `opt_out_clear`, and the last non-`Keep` `st`. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::mutation` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add IdentityMutation and its merge into a live identity row" -m "Removals apply first and win equal-second ties, a provider opt-out removes every record that provider wrote, stateful results apply atomically, refresh anchors survive while their owner holds state, and an invalid row falls back to removals only." +``` + +### Task 16: Sealing and opening the identity cookies + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/cookie.rs` (seal, open, read and format helpers) +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `cookie.rs` + +**Interfaces:** + +- Consumes: `IdentityKeys.cookie_s`, `.cookie_r` (Task 13), `RandomSource` (Task 9), `COOKIE_TS_EIDS_S`, `COOKIE_TS_EIDS_R_PREFIX` (Task 1) +- Produces: + - `pub(crate) fn seal(key: &[u8; 32], ec_id: &str, payload: &[u8], random: &dyn RandomSource) -> String`, which returns `v1..` + - `pub(crate) fn open(key: &[u8; 32], ec_id: &str, value: &str) -> Option>` + - `pub(crate) enum CookieRead { Absent, Duplicate, Value(String) }` and `pub(crate) fn read_single_cookie(headers: &HeaderMap, name: &str) -> CookieRead`, which treats two cookies of the same name as none + - `pub(crate) fn read_prefixed_cookies(headers: &HeaderMap, prefix: &str) -> Vec<(String, String)>` + - `pub(crate) fn set_cookie_header(name: &str, value: &str) -> String` and `pub(crate) fn delete_cookie_header(name: &str) -> String` + - `pub(crate) const COOKIE_MAX_AGE_SECS: u64 = 31_536_000;` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use http::{HeaderMap, HeaderValue, header}; + + use super::*; + use crate::ec::identity::input::SeededRandom; + + const EC: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + const OTHER_EC: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ABC123"; + + #[test] + fn seal_and_open_round_trip_bound_to_the_ec_id() { + let key = [7u8; 32]; + let random = SeededRandom::new(3); + let sealed = seal(&key, EC, br#"{"v":1}"#, &random); + + assert!(sealed.starts_with("v1."), "should carry the version prefix"); + assert_eq!(sealed.split('.').count(), 3, "should have three parts"); + assert!(!sealed.contains('='), "should be unpadded base64url"); + assert_eq!(open(&key, EC, &sealed).as_deref(), Some(&br#"{"v":1}"#[..]), "should open for its EC"); + assert!(open(&key, OTHER_EC, &sealed).is_none(), "should not open for another EC"); + assert!(open(&[8u8; 32], EC, &sealed).is_none(), "should not open with another key"); + let mut tampered = sealed.clone(); + tampered.pop(); + tampered.push(if sealed.ends_with('A') { 'B' } else { 'A' }); + assert!(open(&key, EC, &tampered).is_none(), "should reject tampering"); + assert!(open(&key, EC, "v2.x.y").is_none(), "should reject another version"); + } + + #[test] + fn duplicate_cookie_names_count_as_none() { + let mut headers = HeaderMap::new(); + headers.append(header::COOKIE, HeaderValue::from_static("__Host-ts-eids-s=one")); + headers.append(header::COOKIE, HeaderValue::from_static("other=1; __Host-ts-eids-s=two")); + assert_eq!(read_single_cookie(&headers, "__Host-ts-eids-s"), CookieRead::Duplicate, "duplicate"); + + let mut single = HeaderMap::new(); + single.insert(header::COOKIE, HeaderValue::from_static("__Host-ts-eids-s=one; x=2")); + assert_eq!(read_single_cookie(&single, "__Host-ts-eids-s"), CookieRead::Value("one".to_owned()), "single"); + } + + #[test] + fn prefixed_cookies_and_attributes() { + let mut headers = HeaderMap::new(); + headers.insert( + header::COOKIE, + HeaderValue::from_static("__Host-ts-eids-r-0a1b2c3d=a; __Host-ts-eids-r-11111111=b; x=1"), + ); + let found = read_prefixed_cookies(&headers, "__Host-ts-eids-r-"); + assert_eq!(found.len(), 2, "should find both removal cookies"); + assert_eq!( + set_cookie_header("__Host-ts-eids-s", "v1.a.b"), + "__Host-ts-eids-s=v1.a.b; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=31536000", + "should use the spec attributes and no Domain" + ); + assert_eq!( + delete_cookie_header("__Host-ts-eids-s"), + "__Host-ts-eids-s=; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=0", + "should expire with matching attributes" + ); + } +} +``` + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::cookie` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +```rust +//! The server-owned identity cookies: sealing, reading and formatting. + +use aes_gcm::aead::{Aead as _, KeyInit as _, Payload}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use http::{HeaderMap, header}; + +use super::input::RandomSource; + +/// Lifetime of both identity cookie kinds. +pub(crate) const COOKIE_MAX_AGE_SECS: u64 = 31_536_000; + +const SEAL_VERSION: &str = "v1"; + +/// Seals `payload` with AES-256-GCM, the EC ID as associated data. +#[must_use] +pub(crate) fn seal(key: &[u8; 32], ec_id: &str, payload: &[u8], random: &dyn RandomSource) -> String { + let cipher = Aes256Gcm::new(key.into()); + let mut nonce = [0u8; 12]; + random.fill(&mut nonce); + let ciphertext = cipher + .encrypt(Nonce::from_slice(&nonce), Payload { msg: payload, aad: ec_id.as_bytes() }) + .expect("should encrypt a bounded in-memory payload"); + format!( + "{SEAL_VERSION}.{}.{}", + URL_SAFE_NO_PAD.encode(nonce), + URL_SAFE_NO_PAD.encode(ciphertext) + ) +} + +/// Opens a sealed value. `None` for any failure; never log the value. +#[must_use] +pub(crate) fn open(key: &[u8; 32], ec_id: &str, value: &str) -> Option> { + let mut parts = value.split('.'); + if parts.next()? != SEAL_VERSION { + return None; + } + let nonce = URL_SAFE_NO_PAD.decode(parts.next()?).ok()?; + let ciphertext = URL_SAFE_NO_PAD.decode(parts.next()?).ok()?; + if parts.next().is_some() || nonce.len() != 12 { + return None; + } + Aes256Gcm::new(key.into()) + .decrypt(Nonce::from_slice(&nonce), Payload { msg: &ciphertext, aad: ec_id.as_bytes() }) + .ok() +} + +/// One cookie read by exact name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum CookieRead { + Absent, + /// Two or more cookies share the name; treated as none (spec 6.2). + Duplicate, + Value(String), +} + +fn cookie_pairs(headers: &HeaderMap) -> impl Iterator { + headers + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .filter_map(|pair| pair.trim().split_once('=')) + .map(|(name, value)| (name.trim(), value.trim())) +} + +/// Reads a cookie that must occur at most once. +#[must_use] +pub(crate) fn read_single_cookie(headers: &HeaderMap, name: &str) -> CookieRead { + let mut found = cookie_pairs(headers).filter(|(cookie, _)| *cookie == name).map(|(_, value)| value); + match (found.next(), found.next()) { + (None, _) => CookieRead::Absent, + (Some(_), Some(_)) => CookieRead::Duplicate, + (Some(value), None) => CookieRead::Value(value.to_owned()), + } +} + +/// Every cookie whose name starts with `prefix`, as `(name, value)`. +#[must_use] +pub(crate) fn read_prefixed_cookies(headers: &HeaderMap, prefix: &str) -> Vec<(String, String)> { + cookie_pairs(headers) + .filter(|(name, _)| name.starts_with(prefix)) + .map(|(name, value)| (name.to_owned(), value.to_owned())) + .collect() +} + +/// A `Set-Cookie` value with the identity cookie attributes. +#[must_use] +pub(crate) fn set_cookie_header(name: &str, value: &str) -> String { + format!("{name}={value}; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age={COOKIE_MAX_AGE_SECS}") +} + +/// A `Set-Cookie` value that deletes an identity cookie. +#[must_use] +pub(crate) fn delete_cookie_header(name: &str) -> String { + format!("{name}=; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=0") +} +``` + +If the `aes-gcm` 0.10 key conversion `key.into()` does not infer, use `Aes256Gcm::new_from_slice(key).expect("should accept a 32-byte key")`. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::cookie` and `cargo check-cloudflare` +Expected: PASS and a clean build. The Cloudflare check proves that `aes-gcm` compiles for `wasm32-unknown-unknown`, because core is shared. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Seal and read the server-owned identity cookies" +``` + +### Task 17: The `ts-eids-s` payload, its size cap and its merge with KV + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/cookie.rs` +- Test: `cookie.rs` + +**Interfaces:** + +- Consumes: Task 16, `ProviderCatalog` (Task 14), `KvEntry` (Task 6), `record_revision` (Task 13), `EcKvSnapshot` (`ec/mod.rs:118`) +- Produces: + - Payload types with the spec 6.2 short field names, and serde `skip_serializing_if` on every optional field: + - `SCookie { v: u8, kc: u64, cf: String, m: bool, ids: Vec, ps: BTreeMap }` + - `SEntry { s: usize, u, w, t, e, r, a, x, rv, pv, sa, b, k, d }`. `b`, `k` and `d` serialize as `1`/`0` through small `bool_as_int` helpers. + - `SProviderState { n, f, o: Option, iv: Vec, oc, p, sg, kn }`. `kn` is the KV `n` that a cookie-only `n` was derived from (spec 6.3). + - `pub(crate) const MAX_S_COOKIE_BYTES: usize = 2048;` + - `pub(crate) fn open_s_cookie(keys, ec_id, catalog, value) -> SOpen`, where `SOpen` is `Missing | Failed | ConfigChanged(SCookie) | Valid(SCookie)`. `ConfigChanged` keeps only markers. + - `pub(crate) fn encode_s_cookie(keys, ec_id, cookie: &mut SCookie, random) -> String`, which applies the drop order until the sealed value fits in 2,048 bytes, else falls back to a minimal cookie with `kc = 0` and the markers only (spec 5.6) + - `pub(crate) fn merge_with_kv(cookie: &SCookie, snapshot: &EcKvSnapshot, context: &CookieMergeContext<'_>) -> MergeOutcome`, where `MergeOutcome { cookie: SCookie, changed: bool, expire: bool }` + - `pub(crate) fn cookie_view(cookie: &SCookie, catalog) -> BTreeMap`: the cookie's entries resolved to sources. `Staging` is `Confirmed | Staged { pv, sa, browser }`. + - `pub(crate) fn minimal_missing_cookie(now) -> SCookie` (`m: true`, no entries) + +Merge rules (spec 6.3, step 3). Each is one test: + +- A tombstone or `consent.ok = false` gives `expire: true`. +- `Missing` gives `minimal_missing_cookie`. Markers are dropped too. +- For an owned source, KV wins, except that a staged provider entry `OwnerWrite` would still apply (its `pv` matches the live revision, or it has no `pv` and KV holds only a value the provider did not write) stays `k: 0`. +- A `k: 1` entry whose source is gone from KV, or present at another revision, is replaced by the KV record (when owned) or dropped. +- A staged entry KV already holds (same UID, same writer) becomes `k: 1` with KV's `rv`. +- A `b: 1` entry is dropped once KV holds any value for its source. +- A stub becomes the KV record. +- Markers (`p`) are cleared when answered: KV `n >= p`, KV `o` set, a claimed source with `at >= p` or a source that lost the cookie's value, or the provider ineligible on the row. The last one takes a callback, `answered_by_eligibility`, which Task 20 supplies. +- `m` is cleared when the row is present. +- `kc = now`. `changed` is true when anything but `kc` changed, or the inbound `kc` is older than `kv_revalidate_sec / 2`. + +The cookie caches values only for owned sources once they are in KV (spec 5.3). Unowned confirmed values are not copied into the cookie by the merge. In Part 1 no source is owned, so the cookie holds only staged entries, markers and mirrored `ps`. + +Size-cap drop order (spec 6.2): + +1. `k: 1` entries become stubs `{s, d: 1}`. +2. `ps` entries with no `p` and no `sg` are dropped. +3. Provider `k: 0` entries, lowest `cookie_priority` first. Task 19 adds the priority. Pass it in as `&dyn Fn(&str) -> i32`. +4. `b: 1` entries, last. + +Markers are never dropped. Log each drop with the provider and source only. + +- [ ] **Step 1: Write the failing tests** + +```rust + mod s_cookie { + use super::super::*; + use crate::ec::identity::test_support::{catalog_with_owner, test_keys}; + use crate::ec::identity::input::SeededRandom; + use crate::ec::kv_types::{KvEntry, KvPartnerId, KvProviderState}; + use crate::ec::EcKvSnapshot; + + const EC: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn present(entry: KvEntry) -> EcKvSnapshot { + EcKvSnapshot::Present { ec_id: EC.to_owned(), entry: Box::new(entry), generation: Some(1) } + } + + #[test] + fn round_trips_and_detects_config_change() { + let keys = test_keys(); + let catalog = catalog_with_owner("owner", "a.example.com"); + let random = SeededRandom::new(1); + let mut cookie = SCookie::new(&catalog, 100); + cookie.ps.insert("owner".to_owned(), SProviderState { p: Some(90), ..SProviderState::default() }); + let sealed = encode_s_cookie(&keys, EC, &mut cookie, &random, &|_| 0); + + match open_s_cookie(&keys, EC, &catalog, &sealed) { + SOpen::Valid(opened) => assert_eq!(opened, cookie, "should round-trip"), + other => panic!("should open, got {other:?}"), + } + let other_catalog = catalog_with_owner("owner", "b.example.com"); + match open_s_cookie(&keys, EC, &other_catalog, &sealed) { + SOpen::ConfigChanged(kept) => { + assert!(kept.ids.is_empty(), "should drop positive entries"); + assert_eq!(kept.ps["owner"].p, Some(90), "should keep markers"); + } + other => panic!("should flag the config change, got {other:?}"), + } + } + + #[test] + fn tombstone_expires_and_missing_row_minimizes() { + let catalog = catalog_with_owner("owner", "a.example.com"); + let cookie = SCookie::new(&catalog, 100); + let context = CookieMergeContext::for_test(&catalog, 200); + assert!(merge_with_kv(&cookie, &present(KvEntry::tombstone(1)), &context).expire, "tombstone"); + let missing = merge_with_kv(&cookie, &EcKvSnapshot::Missing { ec_id: EC.to_owned() }, &context); + assert!(missing.cookie.m, "should set m"); + assert!(missing.cookie.ids.is_empty() && missing.cookie.ps.is_empty(), "no entries, no markers"); + } + + #[test] + fn staged_entry_flips_to_confirmed_once_in_kv() { + let catalog = catalog_with_owner("owner", "a.example.com"); + let mut cookie = SCookie::new(&catalog, 100); + cookie.ids.push(SEntry::staged_browser(0, "u", 100)); + let mut entry = KvEntry::minimal("a.example.com", "u", 1); + entry.ids.get_mut("a.example.com").expect("a").rv = Some("1111111111111111".to_owned()); + + let merged = merge_with_kv(&cookie, &present(entry), &CookieMergeContext::for_test(&catalog, 200)); + + assert!(merged.changed, "should change"); + assert!(merged.cookie.ids.is_empty(), "a b:1 entry is dropped once KV holds the source"); + assert_eq!(merged.cookie.kc, 200, "should stamp kc"); + } + + #[test] + fn marker_is_answered_by_a_newer_n() { + let catalog = catalog_with_owner("owner", "a.example.com"); + let mut cookie = SCookie::new(&catalog, 100); + cookie.ps.insert("owner".to_owned(), SProviderState { p: Some(150), ..SProviderState::default() }); + let mut entry = KvEntry::minimal("b.example.com", "x", 1); + entry.ps.insert("owner".to_owned(), KvProviderState { n: Some(160), ..KvProviderState::default() }); + + let merged = merge_with_kv(&cookie, &present(entry), &CookieMergeContext::for_test(&catalog, 200)); + assert_eq!(merged.cookie.ps.get("owner").and_then(|state| state.p), None, "should clear the marker"); + } + + #[test] + fn size_cap_drops_in_order_and_keeps_markers() { + let keys = test_keys(); + let catalog = catalog_with_owner("owner", "a.example.com"); + let random = SeededRandom::new(1); + let mut cookie = SCookie::new(&catalog, 100); + cookie.ps.insert("owner".to_owned(), SProviderState { p: Some(90), ..SProviderState::default() }); + for _ in 0..40 { + cookie.ids.push(SEntry::staged_browser(0, &"u".repeat(60), 100)); + } + let sealed = encode_s_cookie(&keys, EC, &mut cookie, &random, &|_| 0); + assert!(sealed.len() <= MAX_S_COOKIE_BYTES, "should fit the cap"); + assert_eq!(cookie.ps["owner"].p, Some(90), "should keep the marker"); + } + } +``` + +Add `catalog_with_owner(provider, source)` to `test_support.rs`. It builds a `ProviderCatalog` with one `fixed_provider` that owns `source`. Add `CookieMergeContext::for_test(catalog, now)`, which uses `kv_revalidate_sec = 3600`, test keys, and an `answered_by_eligibility` that always returns `false`. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::cookie::tests::s_cookie` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Write the payload types with serde attributes, then `SCookie::new(catalog, now)` (`v: 1`, `kc: now`, `cf: catalog.fingerprint()`), `SEntry::staged_browser(index, uid, sa)`, `open_s_cookie`, `encode_s_cookie`, `merge_with_kv` and `cookie_view`. Give each merge rule above its own private helper, named after the rule (`fn flip_confirmed`, `fn drop_browser_entries_kv_holds`, `fn answer_markers`, and so on). That way a reviewer can match each helper to its spec bullet. A source index that is out of range for `catalog.configured_sources()` is dropped on open. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::cookie` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity +git commit --signoff -S -m "Encode the ts-eids-s cookie and reconcile it with KV" +``` + +### Task 18: The `ts-eids-r` removal cookies + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/cookie.rs` +- Test: `cookie.rs` + +**Interfaces:** + +- Produces: + - `RBatch { rm: Vec, o: BTreeMap, iv: BTreeMap> }` and `RRemoval { s: String /* the source domain itself */, w: String, pv: Option, sa: u64 }` + - `pub(crate) const MAX_R_COOKIE_BYTES: usize = 512;`, `pub(crate) const R_COOKIE_SOFT_LIMIT: usize = 4;` + - `pub(crate) struct OpenedRemovals { cookies: Vec<(String, RBatch)>, unopenable: Vec }` and `pub(crate) fn open_r_cookies(keys, ec_id, headers) -> OpenedRemovals` + - `pub(crate) fn r_batch_to_changes(batch: &RBatch) -> (Vec, BTreeMap)`, which drops entries whose source or provider does not parse + - `pub(crate) fn r_batch_applied(batch: &RBatch, entry: &KvEntry, keys) -> bool`, the "applied" definition of spec 6.2 + - `pub(crate) enum RPlan { Cookie { name, value }, Overflow }` and `pub(crate) fn plan_r_batch(keys, ec_id, batch, inbound_openable: usize, random) -> RPlan`. It returns `Overflow` when the batch does not fit one 512-byte cookie or the request already carries four openable cookies. + - `pub(crate) fn split_r_batch(keys, ec_id, batch, random) -> Vec<(String, String)>`: the overflow fallback, as many 512-byte cookies as it takes, never coarsened (each removal, opt-out and `iv` hash kept exactly) + - `pub(crate) fn new_r_cookie_name(random) -> String`: `__Host-ts-eids-r-` plus 8 random hex digits + +- [ ] **Step 1: Write the failing tests** + +```rust + mod r_cookie { + use super::super::*; + use crate::ec::identity::input::SeededRandom; + use crate::ec::identity::test_support::test_keys; + use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId, KvProviderState}; + + const EC: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn removal(source: &str, writer: &str, sa: u64) -> RRemoval { + RRemoval { s: source.to_owned(), w: writer.to_owned(), pv: None, sa } + } + + #[test] + fn a_typical_batch_fits_one_cookie_and_a_long_domain_fits_alone() { + let keys = test_keys(); + let random = SeededRandom::new(1); + let four = RBatch { rm: (0..4).map(|i| removal(&format!("s{i}.example.com"), "provider", 1)).collect(), ..RBatch::default() }; + assert!(matches!(plan_r_batch(&keys, EC, &four, 0, &random), RPlan::Cookie { .. }), "four typical removals fit"); + + let label = "a".repeat(63); + let long_domain = format!("{label}.{label}.{label}.{}.com", "b".repeat(57)); + assert_eq!(long_domain.len(), 253, "should be a 253-byte domain"); + let one = RBatch { rm: vec![removal(&long_domain, "a-long-provider-id", 1)], ..RBatch::default() }; + assert!(matches!(plan_r_batch(&keys, EC, &one, 0, &random), RPlan::Cookie { .. }), "one long removal fits"); + } + + #[test] + fn overflow_when_too_big_or_four_cookies_present() { + let keys = test_keys(); + let random = SeededRandom::new(1); + let big = RBatch { rm: (0..12).map(|i| removal(&format!("s{i}.example.com"), "provider", 1)).collect(), ..RBatch::default() }; + assert!(matches!(plan_r_batch(&keys, EC, &big, 0, &random), RPlan::Overflow), "too big"); + let small = RBatch { rm: vec![removal("a.example.com", "p", 1)], ..RBatch::default() }; + assert!(matches!(plan_r_batch(&keys, EC, &small, 4, &random), RPlan::Overflow), "four present"); + } + + #[test] + fn split_keeps_every_entry_exactly() { + let keys = test_keys(); + let random = SeededRandom::new(1); + let big = RBatch { + rm: (0..12).map(|i| removal(&format!("s{i}.example.com"), "provider", 1)).collect(), + o: [("provider".to_owned(), 5)].into_iter().collect(), + iv: [("provider".to_owned(), vec!["0123456789abcdef".to_owned()])].into_iter().collect(), + }; + let cookies = split_r_batch(&keys, EC, &big, &random); + assert!(cookies.iter().all(|(_, value)| value.len() <= MAX_R_COOKIE_BYTES), "each fits"); + let mut headers = http::HeaderMap::new(); + let joined = cookies.iter().map(|(name, value)| format!("{name}={value}")).collect::>().join("; "); + headers.insert(http::header::COOKIE, http::HeaderValue::from_str(&joined).expect("header")); + let opened = open_r_cookies(&keys, EC, &headers); + let rebuilt = opened.cookies.iter().fold(RBatch::default(), |mut all, (_, batch)| { + all.rm.extend(batch.rm.clone()); + all.o.extend(batch.o.clone()); + for (provider, hashes) in &batch.iv { + all.iv.entry(provider.clone()).or_default().extend(hashes.clone()); + } + all + }); + assert_eq!(rebuilt.rm.len(), 12, "every removal survives"); + assert_eq!(rebuilt.o, big.o, "the opt-out survives unchanged"); + assert_eq!(rebuilt.iv, big.iv, "the iv hash survives"); + } + + #[test] + fn applied_rules() { + let keys = test_keys(); + let mut entry = KvEntry::minimal("b.example.com", "x", 1); + entry.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { w: Some("p".to_owned()), at: Some(10), ..KvPartnerId::new("u") }, + ); + assert!(!r_batch_applied(&RBatch { rm: vec![removal("a.example.com", "p", 10)], ..RBatch::default() }, &entry, &keys), "pending"); + assert!(r_batch_applied(&RBatch { rm: vec![removal("a.example.com", "p", 9)], ..RBatch::default() }, &entry, &keys), "re-acquired after sa"); + assert!(r_batch_applied(&RBatch { rm: vec![removal("a.example.com", "q", 10)], ..RBatch::default() }, &entry, &keys), "other writer"); + + entry.ps.insert("p".to_owned(), KvProviderState { o: Some(KvOptOut { at: 5, fp: "0".repeat(16) }), ..KvProviderState::default() }); + let opt_out = |at| RBatch { o: [("p".to_owned(), at)].into_iter().collect(), ..RBatch::default() }; + assert!(r_batch_applied(&opt_out(5), &entry, &keys), "KV o.at at or after"); + assert!(!r_batch_applied(&opt_out(6), &entry, &keys), "newer opt-out pending"); + entry.ps.get_mut("p").expect("p").oc = Some(6); + assert!(!r_batch_applied(&opt_out(6), &entry, &keys), "same-second clear does not count"); + entry.ps.get_mut("p").expect("p").oc = Some(7); + assert!(r_batch_applied(&opt_out(6), &entry, &keys), "later clear makes it stale"); + } + + #[test] + fn unopenable_cookies_are_listed_for_deletion() { + let keys = test_keys(); + let mut headers = http::HeaderMap::new(); + headers.insert(http::header::COOKIE, http::HeaderValue::from_static("__Host-ts-eids-r-00000000=v1.bad.value")); + let opened = open_r_cookies(&keys, EC, &headers); + assert!(opened.cookies.is_empty(), "nothing opens"); + assert_eq!(opened.unopenable, vec!["__Host-ts-eids-r-00000000".to_owned()], "listed for deletion"); + } + } +``` + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::cookie::tests::r_cookie` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Serialize `RBatch` as compact JSON and omit empty maps. `plan_r_batch` seals once and measures. `split_r_batch` packs greedily: removals one by one, then each provider's opt-out, then each `iv` hash, sealing after each addition and starting a new cookie when the sealed value would pass 512. A single item that can never fit (impossible within the measured budget) is logged at error and still emitted in its own cookie. Never drop it. `r_batch_applied` implements exactly the three spec 6.2 bullets. `iv` entries never keep a cookie alive. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::cookie` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity +git commit --signoff -S -m "Add per-batch ts-eids-r removal cookies" +``` + +### Task 19: `[ec.identity]` settings + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/config.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` (`Ec`, around line 542) +- Modify: `trusted-server.example.toml` (the `[ec]` section) +- Test: `config.rs`, `settings.rs` + +**Interfaces:** + +- Produces: + - `IdentitySettings { max_concurrency: Option, hold_budget_ms: u64, hold_retry_sec: u64, max_response_bytes: usize, pending_grace_sec: u64, staged_max_age_sec: u64, kv_revalidate_sec: u64, providers: BTreeMap }`, with the defaults from Global Constraints and `Default` + - `ProviderSettings { mode: ProviderMode, timeout_ms: Option, sources: Option>, refresh_after_sec: u64, max_calls_per_minute: u32, breaker_failures_per_minute: u32, cookie_priority: i32 }` + - `enum ProviderMode { Hold, AfterSend }`, serde `snake_case`, default `AfterSend` + - `IdentitySettings::effective_max_concurrency(&self, legacy_pull_concurrency: usize) -> usize` + - `ProviderSettings::effective_timeout_ms(&self, settings: &IdentitySettings) -> u64` + - `IdentitySettings::provider(&self, id: &IdentityProviderId) -> ProviderSettings`, which returns the defaults when there is no table + - `pub(crate) fn validate_identity_settings(settings: &IdentitySettings) -> Result<(), Report>`, covering the numeric rules of spec 10.2: `hold_budget_ms <= 1000`, `max_concurrency != Some(0)`, `refresh_after_sec > 0`, `kv_revalidate_sec < 86_400`, `timeout_ms <= hold_budget_ms` for hold mode. Unknown provider IDs are checked later, in Task 22, when the catalog exists. + - `Ec.identity: IdentitySettings` with `#[serde(default)]` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_without_any_table() { + let settings: IdentitySettings = toml::from_str("").expect("should parse empty"); + assert_eq!(settings.hold_budget_ms, 100, "hold budget"); + assert_eq!(settings.hold_retry_sec, 300, "hold retry"); + assert_eq!(settings.max_response_bytes, 65_536, "max bytes"); + assert_eq!(settings.pending_grace_sec, 300, "grace"); + assert_eq!(settings.staged_max_age_sec, 604_800, "staged age"); + assert_eq!(settings.kv_revalidate_sec, 3600, "revalidate"); + assert_eq!(settings.effective_max_concurrency(4), 4, "falls back to the legacy key"); + let provider = settings.provider(&"anything".parse().expect("id")); + assert_eq!(provider.mode, ProviderMode::AfterSend, "mode"); + assert_eq!(provider.effective_timeout_ms(&settings), 1000, "after-send timeout"); + assert_eq!((provider.refresh_after_sec, provider.max_calls_per_minute, provider.breaker_failures_per_minute, provider.cookie_priority), (86_400, 1, 50, 0), "provider defaults"); + validate_identity_settings(&settings).expect("defaults are valid"); + } + + #[test] + fn hold_mode_timeout_defaults_to_the_budget_and_bounds_are_enforced() { + let settings: IdentitySettings = toml::from_str( + "hold_budget_ms = 120\n[providers.id5]\nmode = \"hold\"\n", + ) + .expect("should parse"); + let provider = settings.provider(&"id5".parse().expect("id")); + assert_eq!(provider.effective_timeout_ms(&settings), 120, "hold timeout = budget"); + + for (name, toml_text) in [ + ("budget over 1000", "hold_budget_ms = 1001"), + ("zero concurrency", "max_concurrency = 0"), + ("revalidate a day", "kv_revalidate_sec = 86400"), + ("zero refresh", "[providers.p]\nrefresh_after_sec = 0"), + ("timeout over budget", "[providers.p]\nmode = \"hold\"\ntimeout_ms = 101"), + ] { + let settings: IdentitySettings = toml::from_str(toml_text).expect(name); + validate_identity_settings(&settings).expect_err(name); + } + toml::from_str::("[providers.p]\nmode = \"sometimes\"").expect_err("bad mode"); + } +} +``` + +Use whichever TOML crate core already uses in tests (`grep -n "toml" crates/trusted-server-core/Cargo.toml`). If only `config` is available, build the value through the same path `settings.rs` tests use. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::config` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Write the structs with `#[serde(default = "...")]` per field and `#[serde(deny_unknown_fields)]`, which matches the rest of `settings.rs`. Check the convention there first. Add to `Ec`: + +```rust + /// Server-side identity provider policy (`[ec.identity]`). + #[serde(default)] + pub identity: IdentitySettings, +``` + +Call `validate_identity_settings(&self.ec.identity)` from both `validate_settings_for_deploy` and `validate_settings_for_runtime` in `config.rs`, next to `PartnerRegistry::...`. Mark `pull_sync_concurrency`'s doc comment as deprecated in favour of `[ec.identity].max_concurrency`, and log a `warn!` once at settings load when `identity.max_concurrency` is unset and `pull_sync_concurrency` is not its default. In `trusted-server.example.toml`, add a commented `[ec.identity]` block that copies spec 10.1's comments and values, with `example.com` partner names. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::config`, `cargo test-fastly -- settings`, `cargo test-fastly -- config::` +Expected: PASS. Existing configs with no `[ec.identity]` still parse. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/trusted-server-core/src trusted-server.example.toml +git commit --signoff -S -m "Add [ec.identity] coordinator settings" +``` + +### Task 20: Coordinator eligibility and outcomes + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (pure logic only in this task) +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `coordinator.rs` + +**Interfaces:** + +- Consumes: catalog, servable, mutation, cookie view, config, policy (Tasks 12 to 19) +- Produces: + - `pub(crate) struct EligibilityView`, the merged view of spec 4.7, built by `EligibilityView::merge(kv: Option<&KvEntry>, cookie: Option<&SCookie>, removals: &[RBatch], catalog, now) -> Self`. It holds per-source records (with KV as the base when read, else the cookie's last confirmed copy, cookie `k: 0` entries on top when their `pv` matches, and `ts-eids-r` removals hiding records), per-provider `n` (the later of cookie and KV), `iv` (union), `o` (the staged opt-out counts unless `ts-eids-s` stages a later `oc`, and the mirrored opt-out yields to KV when KV was read), and markers (cookie only). + - `pub(crate) struct RequestFacts { is_real_browser, ec_persistence_allowed, policy_signals: PolicySignals, writer: bool, new_visitor: bool, runs_auction: bool, now: UnixTime }` + - `pub(crate) enum Phase { Hold, AfterSendDryRun, AfterSend }` + - `pub(crate) enum Eligibility { Eligible { kinds: BTreeMap }, Skip(SkipReason) }`, with a `SkipReason` enum naming each of steps 1 to 8 (step 9, the burst cap, is checked at dispatch) + - `pub(crate) fn eligibility(provider: &dyn IdentityProvider, settings: &ProviderSettings, identity: &IdentitySettings, view: &EligibilityView, facts: &RequestFacts, consent: &ConsentContext, phase: Phase, catalog: &ProviderCatalog) -> Eligibility` + - `pub(crate) enum CallOutcome { Resolved(Resolution), Skipped, InputUnavailable, InvalidState(Option), HoldCutOff, InputFailure, TransportFailure, BreakerOpen }` + - `pub(crate) fn outcome_to_changes(provider: &dyn IdentityProvider, outcome: CallOutcome, view: &EligibilityView, context: &OutcomeContext<'_>) -> OutcomeChanges`, where `OutcomeChanges { removals: Vec, result: Option, state: ProviderStateChange }`. This is the section 8 table, row by row. `OutcomeContext` holds `now`, `keys`, `catalog`, provider settings, `identity` settings, `expected_state` and the consent fingerprint bits. + +Eligibility steps, in order (spec 4.7): + +1. `facts.is_real_browser` +2. `facts.ec_persistence_allowed` +3. `policy_allows(descriptor.consent_policy(), consent, facts.policy_signals)` +4. Per-source kinds: + - **Owned by this provider.** `Initial` when there is no record. `Refresh` when there is one and it is due: expired, written by someone else, `ra` passed, or no `ra` and `at + refresh_after_sec` passed or `at` missing. A browser-seeded UID whose hash is in `iv` is never due. + - **Unowned.** `Initial` only, and only when there is no servable record. An expired record, or one whose writer is unconfigured, counts as absent. + - A source whose kind the provider does not list, or whose needs (`resolves.needs(kind)`) are not available this request, is left out. No sources left means skip. +5. `n` has passed, and no sticky `o` is set (`oc` not strictly later than `o.at`). +6. No unanswered marker for the provider. +7. (Breaker: checked at dispatch in Part 2, Task 35. Part 1 has no breaker.) +8. Mode: + - No provider runs when `facts.new_visitor`. + - `Hold` needs `settings.mode == Hold` and `facts.runs_auction`. A stateful provider also needs `facts.writer`. + - `AfterSendDryRun` and `AfterSend` need `facts.writer`, and either `settings.mode == AfterSend`, or a stateful hold-mode provider on a writer navigation that does not run an auction. + +The needs check in step 4 needs to know which inputs are available. Add `available_inputs: BTreeSet` to `RequestFacts`. + +- [ ] **Step 1: Write the failing tests** + +Write one test per eligibility step and one per section 8 row, using `fixed_provider` and `ProviderSettings::default()`. Name them after the rule, for example `skips_when_not_a_real_browser`, `skips_new_visitor_in_every_mode`, `owned_record_written_by_someone_else_is_due_for_refresh`, `browser_seeded_uid_in_iv_is_never_due`, `unowned_source_with_servable_record_is_not_initial`, `expired_unowned_record_counts_as_absent`, `stateful_hold_provider_runs_after_send_without_auction`, `hold_provider_never_runs_on_stager_when_stateful`, `no_match_waits_retry_in_capped_at_seven_days`, `input_failure_backs_off_exponentially_to_a_day`, `transport_failure_backs_off_to_an_hour`, `hold_cut_off_waits_hold_retry_without_counting_a_failure`, `skip_waits_refresh_after`, `invalid_state_clears_state_and_keeps_f`, `oversized_state_rejects_positive_part_but_keeps_removals`, `opted_out_removes_everything_and_clears_state`, `invalidate_records_iv_hash_and_sets_n_now`. Here are three written out in full. Write the rest in the same shape: + +```rust + #[test] + fn skips_new_visitor_in_every_mode() { + let provider = fixed_provider("p", &["a.example.com"]); + let view = EligibilityView::empty(); + let catalog = ProviderCatalog::new(vec![provider.clone()], BTreeMap::new()); + for phase in [Phase::Hold, Phase::AfterSendDryRun, Phase::AfterSend] { + let facts = RequestFacts { new_visitor: true, ..facts_for_writer() }; + assert!( + matches!( + eligibility(provider.as_ref(), &ProviderSettings::default(), &IdentitySettings::default(), &view, &facts, &allowing_consent(), phase, &catalog), + Eligibility::Skip(SkipReason::NewVisitor) + ), + "no provider runs on a new visitor's first navigation ({phase:?})" + ); + } + let returning = facts_for_writer(); + assert!( + matches!( + eligibility(provider.as_ref(), &ProviderSettings::default(), &IdentitySettings::default(), &view, &returning, &allowing_consent(), Phase::AfterSend, &catalog), + Eligibility::Eligible { .. } + ), + "control: a returning writer is eligible" + ); + } + + #[test] + fn no_match_waits_retry_in_capped_at_seven_days() { + let provider = fixed_provider("p", &["a.example.com"]); + let context = outcome_context_at(1_000); + let default = outcome_to_changes(provider.as_ref(), CallOutcome::Resolved(Resolution::no_match()), &EligibilityView::empty(), &context); + assert_eq!(default.state.n, Some(1_000 + 86_400), "24 h default"); + let long = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::no_match().with_retry_in(Duration::from_secs(30 * 86_400))), + &EligibilityView::empty(), + &context, + ); + assert_eq!(long.state.n, Some(1_000 + 7 * 86_400), "capped at 7 days"); + } + + #[test] + fn transport_failure_backs_off_to_an_hour() { + let provider = fixed_provider("p", &["a.example.com"]); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 5); + let changes = outcome_to_changes(provider.as_ref(), CallOutcome::TransportFailure, &view, &outcome_context_at(1_000)); + assert_eq!(changes.state.f, Some(6), "should count the failure"); + assert_eq!(changes.state.n, Some(1_000 + 3_600), "min(5 min x 2^5, 1 h) = 1 h"); + } +``` + +Add `facts_for_writer()`, `allowing_consent()` and `outcome_context_at(now)` to `test_support.rs`. `facts_for_writer` returns a real browser, persistence allowed, a writer, a returning visitor, no auction, and every input available. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::coordinator` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Implement `EligibilityView::merge`, `eligibility` and `outcome_to_changes` as plain functions. In `outcome_to_changes`, `Resolved(resolution)` does the following: + +- It drops IDs for sources the provider does not claim (after `sources` narrowing from `ProviderSettings`), IDs with an empty UID or one over `MAX_UID_LENGTH`, and duplicates. It logs each drop with the provider ID and source only. +- An ID of an expiring source without an expiry keeps the stored `exp` when its UID is unchanged and is dropped otherwise. +- `with_refresh_until` from a provider without private state is ignored with a warning. +- Each kept ID becomes `OwnerWrite` with `expected = record_revision(stored)` (or `None` when no record) when this provider owns the source, else `FillMissing { writer: Some(provider) }`. +- `invalidated` becomes `OwnerRemove` with the revision read, plus an `iv` hash of the stored UID, plus `n = now`. +- `opted_out` becomes `WriterRemove` for every source the provider claims (the mutation also removes the unowned records it wrote), plus `opt_out = KvOptOut { at: now, fp }` and `st = Clear`. +- When the provider has private state, the positive IDs plus the state change plus `n` and `f` go into one `ProviderResult`. Otherwise they go into `removals`, a plain change list, and `state`. +- `state_rejected()` drops the positive part and applies the `Malformed` backoff. + +Backoff arithmetic uses `saturating_pow` and `min`, so a large `f` cannot overflow. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::coordinator` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add coordinator eligibility and outcome rules" +``` + +### Task 21: Coordinator dispatch + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` (fake HTTP client) +- Test: `coordinator.rs` + +**Interfaces:** + +- Consumes: `RuntimeServices::http_client().send_async`, `services.backend().ensure(&PlatformBackendSpec { .. })` exactly as `dispatch_pull_sync` uses them (`ec/pull_sync.rs`, the body of `dispatch_pull_sync`), the bounded decoders in `streaming_processor.rs`, and the rate limiter's `exceeded_per_minute` +- Produces: + - `pub(crate) struct IdentityDeps<'a> { settings: &'a Settings, catalog: &'a ProviderCatalog, keys: &'a IdentityKeys, services: &'a RuntimeServices, rate_limiter: Option<&'a dyn RateLimiter> }`. This one struct keeps `handle_auction` and `handle_publisher_request` within 7 arguments. + - `pub(crate) struct Call { provider: IdentityProviderId, kinds: BTreeMap }` + - `pub(crate) fn dispatch(deps: &IdentityDeps<'_>, calls: Vec, signals: &ResolveSignals<'_>, budget: Option) -> Vec<(IdentityProviderId, CallOutcome)>` + - `pub(crate) fn check_request(request: &ProviderRequest, descriptor: &ProviderDescriptor) -> Result<(), Report>`: `https` only, the host in `resolves.hosts()`, no `Cookie` header + - `pub(crate) fn decode_body(encoding: Option<&str>, body: Vec, cap: usize) -> Option>`: identity, `gzip` or `br`, the cap applied to the decoded size, `None` when over the cap or for an unknown encoding + +Dispatch per call: + +1. Build the `ResolveInput` and call `request`. + - `Ok(None)` is `Skipped`. + - `Err` with `InputUnavailable` is `InputUnavailable`. + - `Err` with `UndeclaredInput` or any other `Config` is `InputFailure`, logged as `Config` with `current_context()` only. + - Any recorded undeclared read is also logged as `Config` and turns the outcome into `InputFailure`. +2. `check_request`. On failure: log `Config`, `InputFailure`. +3. The burst cap: `rate_limiter.exceeded_per_minute(&format!("identity:{provider}:{}", visitor_key(keys, ec_id)), settings.max_calls_per_minute)`. Exceeded means not dispatched (Task 29 adds the 60-second marker backoff). A limiter error is logged and the call is skipped. +4. `ensure` a backend with first-byte timeout `effective_timeout_ms`, or in hold mode `min(timeout, remaining budget)` rounded up to the orchestrator's buckets (`auction/orchestrator.rs`, search for the bucket rounding). Do not follow redirects. Send with `send_async`, keeping at most `effective_max_concurrency` in flight. +5. Collect responses: check `Content-Length` against `max_response_bytes`, buffer, `decode_body`. Over the cap is `TransportFailure`. Build the `ProviderResponse` with the request's call context. Call `response`. + - `Ok(resolution)` is `Resolved`, or `InvalidState(Some(resolution))` when the input recorded a state decode failure. + - `Rejected` with 429 or 5xx is `TransportFailure`. + - Other `Rejected`, `Malformed` and `Config` are `InputFailure`. + - `InvalidState` is `InvalidState(None)`. +6. A hold-mode call still pending when the budget runs out is dropped and becomes `HoldCutOff`. + +- [ ] **Step 1: Write the failing tests** + +Add a `FakeHttpClient` to `test_support.rs`. It implements the platform HTTP client trait that `RuntimeServices` holds (`grep -n "trait .*HttpClient" crates/trusted-server-core/src/platform/*.rs`) and returns canned responses per host, with an optional delay counter for hold-budget tests. Check first whether `auction/test_support.rs` already has one, and reuse it if so. Then test: + +```rust + #[test] + fn check_request_enforces_https_host_and_no_cookie() { + let descriptor = fixed_provider("p", &["a.example.com"]).descriptor().clone(); + let good = ProviderRequest::get(&"https://api.example.com/x".parse().expect("url")); + check_request(&good, &descriptor).expect("control: allowed"); + for (name, request) in [ + ("http", ProviderRequest::get(&"http://api.example.com/x".parse().expect("url"))), + ("host", ProviderRequest::get(&"https://evil.example.net/x".parse().expect("url"))), + ("cookie", good.clone().try_with_header("cookie", "a=1").expect("header")), + ] { + check_request(&request, &descriptor).expect_err(name); + } + } + + #[test] + fn decode_body_caps_decoded_size() { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + std::io::Write::write_all(&mut encoder, &vec![b'a'; 10_000]).expect("write"); + let gzipped = encoder.finish().expect("finish"); + assert_eq!(decode_body(Some("gzip"), gzipped.clone(), 10_000).map(|b| b.len()), Some(10_000), "fits"); + assert!(decode_body(Some("gzip"), gzipped, 9_999).is_none(), "over the cap after decoding"); + assert!(decode_body(Some("zstd"), vec![1], 10).is_none(), "unknown encoding"); + assert_eq!(decode_body(None, b"ok".to_vec(), 10).as_deref(), Some(&b"ok"[..]), "identity"); + } +``` + +Also write `dispatch_maps_status_codes_to_outcomes` (200 is `Resolved`, 503 is `TransportFailure`, 400 is `InputFailure`, a body over the cap is `TransportFailure`) and `undeclared_read_is_an_input_failure_even_when_ignored`, using a provider that reads `user_agent().ok()` without declaring it, against the fake client. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::coordinator` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Follow the transport code in `dispatch_pull_sync` closely. It already handles backend resolution, `send_async`, draining the in-flight batch (`drain_pull_batch`), the `Content-Length` check (`response_content_length_exceeds_limit`) and buffering (`extract_pull_uid`). Move those three helpers out of `pull_sync.rs` into `coordinator.rs`, generalized to `max_response_bytes`, and make `pull_sync.rs` call the moved versions until Task 29 deletes its dispatcher. For decoding, reuse the bounded decoders in `streaming_processor.rs` if a buffered entry point exists (`grep -n "pub(crate) fn\|pub fn" crates/trusted-server-core/src/streaming_processor.rs`). Otherwise use `flate2::read::MultiGzDecoder` and `brotli::Decompressor` with `Read::take(cap as u64 + 1)` and reject when more than `cap` bytes come out. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity::coordinator` and `cargo test-fastly -- ec::pull_sync` +Expected: PASS. Pull sync behaviour is unchanged so far. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Add coordinator dispatch with request checks and byte caps" +``` + +### Task 22: `GenericPullProvider`, the write-counting store and the basic harness + +**Files:** + +- Create: `crates/trusted-server-core/src/ec/identity/generic_pull.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` (`ProviderHarness` basics, `WriteCountingEcKv`) +- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (`ProviderCatalog::from_settings`) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (`KvIdentityGraph::write_counting` constructor, test support) +- Test: `generic_pull.rs` + +**Interfaces:** + +- Consumes: `PartnerConfig` (`ec/registry.rs:22`), `validated_pull_sync_url` and `build_pull_request_url` (`ec/pull_sync.rs`, made `pub(crate)`) +- Produces: + - `pub(crate) struct GenericPullProvider` with `pub(crate) fn from_partner(partner: &PartnerConfig) -> Result>` + - `pub(crate) fn generic_pull_id(source_domain: &str) -> IdentityProviderId`: `pull-` plus the domain with `.` replaced by `-`. Over 63 characters, it is cut to 54 characters, followed by `-` and the first 8 hex digits of SHA-256 of the full domain. + - `ProviderCatalog::from_settings(settings: &Settings, registry: &PartnerRegistry) -> Result>`. In Part 1 it holds generic pull providers only, with no owners, and rejects a duplicate provider ID and more than 32 providers. + - `pub(crate) fn legacy_rate_limit_per_minute(pull_sync_rate_limit: u32) -> u32` = `max(1, ceil(limit / 60))` + - Test support: + - `WriteCountingEcKv` (an in-memory store that counts `Add`, `IfGenerationMatch` and `Overwrite` writes and lookups) with `pub(crate) fn writes(&self) -> WriteCounts` + - `KvIdentityGraph::write_counting(name) -> (KvIdentityGraph, Arc)` + - `ProviderHarness` with `new`, the `with_*` setters, `request`, `request_json`, `respond`, `respond_raw` and `resolve`. Conformance checks come in Task 39. + +Generic pull's descriptor: + +```rust +let id = generic_pull_id(&partner.source_domain); +let mut resolves = Resolves::new().with_ec_id(); +for host in &partner.pull_sync_allowed_domains { + resolves = resolves.with_host(host); +} +ProviderDescriptor::builder(id) + .with_source(partner.source_domain.parse()?) + .with_resolves(resolves) + .with_consent_policy(ConsentPolicy::eid_persistence_only()) + .build() +``` + +Its `request` is a `GET` of `validated_pull_sync_url(partner)` with `?ec_id=` (via `build_pull_request_url`) and `with_bearer(ts_pull_token)`. Skip (`Ok(None)`) when there is no valid URL or token. Its `response` is `success_json::<{ uid: String }>()`. An empty UID or one over `MAX_UID_LENGTH` (`is_valid_eid_uid`) is `no_match`, and any other UID is `found([ResolvedId::new(source, uid)])`. The provider's settings default to `mode = AfterSend`, `max_calls_per_minute = legacy_rate_limit_per_minute(partner.pull_sync_rate_limit)`, and `refresh_after_sec = partner.pull_sync_ttl_sec` (used only when Part 2's `owner` makes it authoritative). A `[ec.identity.providers.]` table overrides these. Expose `fn default_settings_for(partner) -> ProviderSettings` and make `IdentitySettings::provider` fall back to it for `pull-` IDs. To do that, `provider()` takes an `Option` base. + +- [ ] **Step 1: Port today's pull sync tests** + +List them with `grep -n "fn " crates/trusted-server-core/src/ec/pull_sync.rs | sed -n '/mod tests/,$p'`. For each test of protocol behaviour (URL building, bearer, response parsing, 64 KiB cap, allowlist, empty or oversized UID), write the equivalent against `GenericPullProvider` through `ProviderHarness`. For example: + +```rust + #[test] + fn calls_the_partner_url_with_the_ec_id_and_bearer() { + let partner = partner_config("ssp.example.com", "https://sync.ssp.example.com/pull", "example-token-value-0123456789abcdef"); + let provider = GenericPullProvider::from_partner(&partner).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + + let request = harness.request().expect("should build").expect("should not skip"); + + assert_eq!(request.url().host_str(), Some("sync.ssp.example.com"), "host"); + assert_eq!( + request.url().query_pairs().find(|(name, _)| name == "ec_id").map(|(_, value)| value.into_owned()), + Some(TEST_EC_ID.to_owned()), + "should pass the EC ID" + ); + assert_eq!(request.headers()["authorization"], "Bearer example-token-value-0123456789abcdef", "bearer"); + + let resolution = harness.respond(StatusCode::OK, &json!({ "uid": "partner-uid" })).expect("should parse"); + assert_eq!(resolution.ids()[0].uid(), "partner-uid", "uid"); + } + + #[test] + fn provider_ids_are_derived_and_bounded() { + assert_eq!(generic_pull_id("ssp.example.com").as_str(), "pull-ssp-example-com", "simple"); + let long = format!("{}.example.com", "a".repeat(70)); + let id = generic_pull_id(&long); + assert!(id.as_str().len() <= 63, "bounded"); + assert_ne!(id, generic_pull_id(&format!("{}.example.com", "a".repeat(71))), "unique by hash"); + } + + #[test] + fn legacy_hourly_limit_maps_to_per_minute() { + assert_eq!(legacy_rate_limit_per_minute(0), 1, "at least one"); + assert_eq!(legacy_rate_limit_per_minute(59), 1, "rounds up"); + assert_eq!(legacy_rate_limit_per_minute(65), 2, "65/hr is 2/min, as the limiter does today"); + } +``` + +`ProviderHarness::with_ec_id` is crate-private (only generic pull reads the EC ID). Add it next to the public setters with `pub(crate)`. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::identity::generic_pull` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Write `generic_pull.rs`, `ProviderCatalog::from_settings`, the harness basics and the counting store. The harness builds `ResolveSignals` from its setters (`SeededRandom` from `with_random_seed`, default seed `1`, default `now` 1,760,000,000). It computes kinds as spec 13 says: a source with a `with_stored` record is `Refresh`, any other is `Initial`, and sources whose kind's needs are missing are dropped, with `Ok(None)` and no provider call when none are left. It keeps the built request's call context for `respond`. `respond` builds a `ProviderResponse` from `serde_json::to_vec(body)`. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity` and `cargo test-fastly -- ec::pull_sync` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Model pull sync partners as GenericPullProvider" -m "Adds the provider harness basics and a KV store that counts writes by mode." +``` + +--- + +### Task 23: The writer decision and request-scoped identity state on `EcContext` + +**Files:** + +- Modify: `crates/trusted-server-core/src/http_util.rs` (add `is_document_navigation`, move `is_prefetch_request` here) +- Modify: `crates/trusted-server-core/src/publisher.rs` (`is_prefetch_request` at about line 3200 becomes a re-export or its callers switch) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`EcContext` fields and accessors) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`build_ec_request_state`, the publisher fallback, and the integration-proxy EC generation path) +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (integration-proxy EC generation, about line 1046, gains the real-browser check) +- Test: `http_util.rs`, `ec/mod.rs`, `adapter-fastly/src/app.rs` + +**Interfaces:** + +- Produces: + - `pub fn is_document_navigation(req: &Request) -> bool`: `Sec-Fetch-Dest: document` only, with no `Accept` fallback + - `pub fn is_prefetch_request(req: &Request) -> bool`, moved unchanged and made public + - `pub(crate) struct RequestIdentity` stored on `EcContext`, with accessors: + - `writer: bool` + - `user_agent: Option`, `dnt: bool`, `page_url: Option` (captured in phase 1) + - `s_cookie: SOpen`, `removals: OpenedRemovals` (opened lazily, at most once, Task 16 and 18 types) + - `mutation: IdentityMutation` + - `staged: StagedCookies`: what this response must set or delete. `s_cookie: Option` to set, `expire_s: bool`, `new_r_batch: Option`, `delete_r: Vec`. + - `EcContext::identity(&self) -> &RequestIdentity` and `identity_mut(&mut self) -> &mut RequestIdentity` + - `EcContext::decide_writer(&mut self, req: &Request, organic_route: bool, is_real_browser: bool)`. It sets `writer = organic_route && is_document_navigation(req) && !is_prefetch_request(req) && is_real_browser` and captures the user agent, DNT and the page URL (the request URL without query or fragment). + - `EcContext::open_identity_cookies(&mut self, keys: &IdentityKeys, catalog: &ProviderCatalog, headers: &HeaderMap)`: idempotent, and a no-op without an EC ID + +The writer decision is made in the adapter where routing already decides EC generation (spec 5.1). That is after pre-route filters allow dispatch: + +- In the publisher fallback branch of `app.rs`, next to `let is_navigation = is_navigation_request(&req);`, call `ec.ec_context.decide_writer(&req, true, ec.is_real_browser)`. +- In the integration-proxy branch, call `decide_writer(&req, true, is_real_browser)` before `handle_proxy`, and make proxy EC generation require `is_real_browser` explicitly, not just a present KV graph (`registry.rs:1048`). `ProxyDispatchInput` gains `is_real_browser: bool` for this. +- Every other route leaves `writer = false`. + +- [ ] **Step 1: Write the failing tests** + +In `http_util.rs`: + +```rust + #[test] + fn document_navigation_requires_sec_fetch_dest_document() { + let with_dest = Request::builder() + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("request"); + let accept_only = Request::builder() + .header(header::ACCEPT, "text/html") + .body(EdgeBody::empty()) + .expect("request"); + let iframe = Request::builder() + .header("sec-fetch-dest", "iframe") + .body(EdgeBody::empty()) + .expect("request"); + assert!(is_document_navigation(&with_dest), "document counts"); + assert!(!is_document_navigation(&accept_only), "the Accept fallback does not"); + assert!(is_navigation_request(&accept_only), "control: the broad check still accepts it"); + assert!(!is_document_navigation(&iframe), "iframes do not"); + } +``` + +In `ec/mod.rs` tests: + +```rust + #[test] + fn decide_writer_needs_every_condition() { + let navigation = || { + Request::builder() + .uri("https://www.example.com/a?b=c#d") + .header("sec-fetch-dest", "document") + .header(header::USER_AGENT, "ExampleBrowser/1.0") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("request") + }; + let mut context = EcContext::default(); + context.decide_writer(&navigation(), true, true); + assert!(context.identity().writer(), "organic real-browser document navigation"); + assert_eq!(context.identity().page_url().map(Url::as_str), Some("https://www.example.com/a"), "no query"); + assert_eq!(context.identity().user_agent(), Some("ExampleBrowser/1.0"), "ua captured"); + assert!(context.identity().dnt(), "dnt captured"); + + for (name, organic, browser, request) in [ + ("named route", false, true, navigation()), + ("bot", true, false, navigation()), + ( + "prefetch", + true, + true, + Request::builder() + .header("sec-fetch-dest", "document") + .header("sec-purpose", "prefetch") + .body(EdgeBody::empty()) + .expect("request"), + ), + ] { + let mut context = EcContext::default(); + context.decide_writer(&request, organic, browser); + assert!(!context.identity().writer(), "not a writer: {name}"); + } + } +``` + +Use the prefetch headers `is_prefetch_request` actually checks. Read it before writing the test. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- http_util::tests::document_navigation` and `cargo test-fastly -- ec::tests::decide_writer` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Add `RequestIdentity` (in `ec/identity/mod.rs` as `pub(crate) mod request;`, file `ec/identity/request.rs`, registered in `migration_guards.rs`) with `Default`, the accessors above, and setters used by later tasks. Add the field `identity: RequestIdentity` to `EcContext` (it already derives `Default`, so this needs `RequestIdentity: Default + Clone`). `IdentityMutation`, `SCookie`, `OpenedRemovals` and `RBatch` need `Clone`, so add `#[derive(Clone)]` where it is missing. `EcFinalizeState` must stay `Clone + Send + Sync`, and every type here is plain data, so it does. + +Move `is_prefetch_request` into `http_util.rs` as `pub`. Replace the body in `publisher.rs` with `use crate::http_util::is_prefetch_request;` at the top, and delete the old definition. + +Wire `decide_writer` in `app.rs` as described above. Add `is_real_browser` to `ProxyDispatchInput` and set it from `EcRequestState`. In `IntegrationRegistry::handle_proxy`, require it before proxy EC generation. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly` (full: the adapter and core) +Expected: PASS. Behaviour is unchanged so far, apart from integration-proxy EC generation now needing a real browser. If an existing proxy test minted an EC for a non-browser request, update it and note the change in the commit body. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/ +git commit --signoff -S -m "Decide the writer request once routing allows dispatch" -m "Only an organic real-browser top-level document navigation that is not a prefetch is the writer. Integration-proxy EC generation now requires a real browser explicitly." +``` + +### Task 24: `/identify` stops writing; cluster size rides on the writer + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identify.rs` (the `evaluate_cluster` call at about line 107) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (`evaluate_cluster` at about line 1484 gains a non-writing variant) +- Test: `ec/identify.rs`, `ec/kv.rs` + +**Interfaces:** + +- Produces: + - `KvIdentityGraph::compute_cluster_size(&self, ec_id: &str) -> Result>`: the prefix list and count only, no write + - `/identify` uses a stored `cluster_size` when present and `compute_cluster_size` otherwise, and never writes + - `pub(crate) fn queue_cluster_size(context: &mut EcContext, graph: &KvIdentityGraph, ec_id: &str)`, called by phase 4 (Task 29) when its live row lacks `cluster_size` + +- [ ] **Step 1: Write the failing test** + +```rust + #[test] + fn identify_computes_cluster_size_without_writing() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_row_without_cluster_size(&graph); + let response = identify_for_test(&graph, &ec_id); + + assert_eq!(response.status(), StatusCode::OK, "should answer"); + assert!(response_body_json(response)["cluster_size"].is_number(), "should report a size"); + assert_eq!(store.writes().total(), 0, "should not write"); + } +``` + +`seed_row_without_cluster_size`, `identify_for_test` and `response_body_json` are thin wrappers around what the existing identify tests already do. Write them in the test module from that setup. + +- [ ] **Step 2: Run and see it fail** + +Run: `cargo test-fastly -- ec::identify::tests::identify_computes_cluster_size_without_writing` +Expected: FAIL with one write. + +- [ ] **Step 3: Implement** + +Split `evaluate_cluster` into `compute_cluster_size` (list and count) and the existing write. `/identify` calls only `compute_cluster_size`. Keep `evaluate_cluster` for now. Task 29 calls it through the mutation (`IdentityMutation::set_cluster_size`), and once nothing else calls it, remove it in that task. + +- [ ] **Step 4: Run and see it pass** + +Run: `cargo test-fastly -- ec::identify` and `cargo test-fastly -- ec::kv` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/trusted-server-core/src/ec +git commit --signoff -S -m "Stop writing cluster size from /identify" +``` + +### Task 25: Finalize builds the mutation, and stagers save into cookies + +This is the core behaviour change of Part 1. `ec_finalize_response` stops writing KV for returning visitors. On the writer request it adds changes to the mutation. On stagers it saves changes into `__Host-ts-eids-s` and `__Host-ts-eids-r-*`. + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (`ec_finalize_response`, `finalize_unusable_consent`, `confirm_then_recover_orphaned_ec`, `recover_orphaned_ec`) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`generate_if_needed` takes the browser ingest) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (`create_if_absent` callers pass the seeded entry) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (pass the parsed `ts-eids` and sharedId to `generate_if_needed`) +- Test: `ec/finalize.rs` + +**Interfaces:** + +- Consumes: `RequestIdentity` (Task 23), `IdentityMutation` (Task 15), the cookie functions (Tasks 16 to 18), `ProviderCatalog`, `IdentityKeys` +- Produces: + - `ec_finalize_response` gains one argument, `identity: &IdentityFinalize<'_>`, which bundles `keys`, `catalog`, `identity_settings`, `now` and `random`, so the function stays within 7 arguments + - Writer, returning EC: adds `FillMissing` browser ingest (`acquired_at = now`) for `ts-eids` and sharedId, plus the flush-checked staged entries from `ts-eids-s` and every `ts-eids-r` batch, to `ec_context.identity_mut().mutation`. It writes nothing. + - Stager, returning EC: stages its changes in `staged.s_cookie` (positive) and `staged.new_r_batch` (removals). + - `pub(crate) fn emit_identity_cookies(context: &mut EcContext, identity: &IdentityFinalize<'_>, response: &mut Response)`, called at the end of every finalize path (the unusable-consent path included). It appends the `Set-Cookie` headers for `staged`, and it is the only place `__Host-ts-eids-*` cookies are set. + - `generate_if_needed(settings, kv, seed: &[PartnerIdUpdate])`: the new-visitor create carries the browser ingest, so finalize no longer writes for a new EC (spec 5.2, phase 1) + - Orphan rotation creates the new row carrying this request's browser ingest plus the removals, opt-outs, `oc` and `iv` from `ts-eids-r`, never positive staged IDs. When the create is `Written`, the response deletes the `ts-eids-r` cookies and sets a fresh `ts-eids-s` from the created row. Rotation empties the mutation and the markers. + +What each path does (spec 5.2, 5.3, 5.6): + +| Path | KV | Cookies | +| --------------------------------------------------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| New EC (writer or not) | create before routing, carrying browser ingest (`generate_if_needed`) | `ts-ec` as today. `ts-eids-s` from the created row. | +| Returning EC, writer | none (phase 4 flushes the mutation) | `ts-eids-s` only under phase 3's rules: something staged, a marker, a merge change, or a missing, failed or stale cookie | +| Returning EC, stager | none | `ts-eids-s` when it saved an entry or its KV read changed the cookie. A new `ts-eids-r` when it produced a removal. | +| Unusable consent, not withdrawn | none | a new `ts-eids-r` when it produced a removal. `ts-eids-s` expired. | +| Consent withdrawn | today's tombstone path, unchanged | `ts-eids-s` expired. Every `ts-eids-r` deleted when the tombstone write succeeded, kept when it failed. | +| Any KV read that found a tombstone | none | `ts-eids-s` expired, every `ts-eids-r` deleted | +| Any KV read that found the row missing (no recovery here) | none | `ts-eids-s` replaced by the minimal `m: 1` cookie. `ts-eids-r` kept. | + +Flush checks for positive staged entries (spec 5.3). Write each as a predicate with its own test: + +- The cookie opened with this EC's ID (guaranteed by `open`). +- `now - sa <= staged_max_age_sec`. +- The writer `w` is in the catalog and still claims the source. For an owned source it also owns it. +- `gate_eid_updates_by_consent` and the writer's `policy_allows` pass under this request's consent. +- The writer has no sticky opt-out, unless the cookie stages a clear `oc` later than that opt-out's `at`. +- Conversion: `OwnerWrite { expected: pv }` when the writer owns the source, else `FillMissing`. A `b: 1` entry becomes `FillMissing { writer: None }`. + +Removals and staged `o`, `oc` and `iv` skip every check except the EC binding (spec 5.6). + +- [ ] **Step 1: Write the failing tests** + +Add a `mod identity_flow` test module to `finalize.rs`. Build requests and contexts the way the existing finalize tests do (`grep -n "fn finalize_" crates/trusted-server-core/src/ec/finalize.rs | head -40`). Use `KvIdentityGraph::write_counting`. Write these tests. The first two are shown in full: + +```rust + #[test] + fn returning_writer_navigation_adds_ingest_to_the_mutation_and_writes_nothing() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, KvEntry::minimal("other.example.com", "x", 1)); + let mut context = returning_context(&ec_id, Writer::Yes); + let mut response = html_response(); + + ec_finalize_response( + &settings_with_partner("sharedid.org"), + &mut context, + Some(&graph), + ®istry_with_partner("sharedid.org"), + None, + Some("example-shared-id"), + &mut response, + &identity_finalize_for_test(), + ); + + assert_eq!(store.writes().total(), 0, "finalize must not write for a returning writer"); + assert!(!context.identity().mutation().is_empty(), "should queue the sharedId fill"); + assert!(set_cookie_names(&response).is_empty(), "no Set-Cookie on an unchanged returning HTML response"); + } + + #[test] + fn auction_stager_saves_the_kv_gap_into_ts_eids_s() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, KvEntry::minimal("other.example.com", "x", 1)); + let mut context = returning_context(&ec_id, Writer::No); + context.set_eid_sync_source(EidSyncSource::Auction); + context.set_client_eids(vec![eid("gap.example.com", "body-only-uid")]); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let mut response = json_response(); + + ec_finalize_response( + &settings_with_partner("gap.example.com"), + &mut context, + Some(&graph), + ®istry_with_partner("gap.example.com"), + None, + None, + &mut response, + &identity_finalize_for_test(), + ); + + assert_eq!(store.writes().total(), 0, "a stager never writes KV"); + let cookie = open_set_s_cookie(&response, &ec_id).expect("should set ts-eids-s"); + assert_eq!(cookie.ids.len(), 1, "should save one entry"); + assert!(cookie.ids[0].is_browser_gap(), "as a b:1 KV-gap entry"); + assert_eq!(cookie.ids[0].uid(), "body-only-uid", "with the body's UID"); + } +``` + +The others: + +- `stager_never_saves_positive_entries_when_consent_denies_persistence` (it may still set a `ts-eids-r`) +- `unusable_consent_path_still_sets_removal_cookie_and_expires_ts_eids_s` +- `withdrawal_expires_ts_eids_s_and_deletes_removals_only_after_tombstone_lands` +- `tombstone_read_expires_ts_eids_s_and_deletes_removal_cookies` +- `missing_row_on_stager_sets_minimal_m_cookie_and_keeps_removals` +- `orphan_rotation_carries_removals_but_never_positive_staged_ids` (with a `ts-eids-s` holding a `k: 0` provider entry and a `ts-eids-r` removal: the created row has the removal's opt-out and none of the staged IDs) +- `new_visitor_create_carries_browser_ingest_in_one_write` (one `Add` write, and the row holds the sharedId) +- `flush_checks_drop_stale_unowned_disabled_and_opted_out_entries_but_never_removals` (one assertion per check, each paired with a control entry that survives) +- `config_change_discards_positive_entries_and_keeps_markers_and_removals` + +`identity_finalize_for_test()`, `open_set_s_cookie`, `set_cookie_names`, `seed_live_row`, `returning_context` and the response builders go in the test module. Build them on `ec/identity/test_support.rs`. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::finalize::tests::identity_flow` +Expected: compile errors, then assertion failures (finalize still writes). + +- [ ] **Step 3: Implement** + +Restructure `ec_finalize_response`: + +1. At the top, call `ec_context.open_identity_cookies(...)` once (it is idempotent and needs the EC ID). +2. Unusable consent: keep today's code, then stage any removals the route produced (Task 38 adds captures, so in Part 1 these come only from `ts-eids-r` carry-over), expire `ts-eids-s`, and call `emit_identity_cookies`. On withdrawal, delete the `ts-eids-r` cookies only when every tombstone write returned success. Check what `apply_withdrawal_tombstones` reports. If it returns nothing, make it return whether every write succeeded. +3. Returning EC with a graph: + - Collect browser ingest as today, then consent-gate it. + - If the request read KV (`kv_snapshot` is not `NotRead`), run `merge_with_kv` on the cookie and handle tombstone, missing and orphan recovery as the table says. Today's `confirm_then_recover_orphaned_ec` stays, with its `updates` argument extended to carry the `ts-eids-r` removals into the created row. + - **Writer:** add ingest, then the flush-checked staged entries, then every `ts-eids-r` batch, to the mutation. Set `staged.s_cookie` only under the phase 3 rules. Record which `ts-eids-r` cookies the phase 1 read shows applied (`r_batch_applied`) or unopenable, in `staged.delete_r`. + - **Stager:** run the KV-gap logic (Task 26 adds the `/auction` part) and save. Never touch `staged.delete_r`. + - Remove the call to `sync_eid_cookie_updates` and the `EidCookieSyncOutcome` measurement for these paths. Keep `record_eid_sync_terminal` for the new-EC create only, or delete it if nothing uses it. +4. New EC: `generate_if_needed` already wrote the seeded row. Finalize sets `ts-ec` as today and stages `ts-eids-s` from the created row. +5. Call `reconcile_pull_sync_marker` as today, then `emit_identity_cookies`. + +In `ec/mod.rs`, give `generate_if_needed` a `seed: &[PartnerIdUpdate]` parameter and apply it to the `KvEntry::new(..)` it creates, via `apply_partner_id_updates`, before `create_if_absent`. In `app.rs`, parse `ec.eids_cookie` and `ec.sharedid_cookie` with `collect_eid_updates` and `gate_eid_updates_by_consent` before calling `generate_if_needed`. Both helpers are `pub(crate)` in core, so add a `pub fn browser_seed_updates(settings, registry, consent, eids_cookie, sharedid_cookie, now) -> Vec` wrapper in `ec/finalize.rs` for the adapter. + +`ec_finalize_response` callers: the Fastly adapter's `finalize_with_ec_state` (`main.rs`, about line 597) and `publisher.rs` tests. Build `IdentityFinalize` from `AppState`, which holds the catalog and keys built once per settings load (Task 29 adds them to `AppState`). Until then, construct them in `finalize_with_ec_state` from settings. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::finalize` and then the full `cargo test-fastly` +Expected: PASS. Existing finalize tests that asserted a KV write on a returning navigation, `/auction` or `/_ts/page-bids` must now assert no write plus a queued mutation or a staged cookie. Update each one and list them in the commit body. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/ +git commit --signoff -S -m "Queue browser ingest in the writer's mutation and stage it on stagers" -m "Returning-visitor finalize no longer writes KV. The writer adds ingest, staged entries and removal batches to its single mutation. Stagers save positive entries in __Host-ts-eids-s and removals in a new __Host-ts-eids-r cookie. A new visitor's create now carries browser ingest, so it stays one write." +``` + +### Task 26: `/auction` saves the KV gap; `/_ts/page-bids` stops ingesting + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (after the KV read at about lines 307-311) +- Modify: `crates/trusted-server-core/src/publisher.rs` (the `/_ts/page-bids` ingest at about line 6925) +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (stager branch) +- Test: `auction/endpoints.rs`, `publisher.rs` + +**Interfaces:** + +- Produces: `pub(crate) fn kv_gap_entries(body_eids: &[Eid], snapshot: &EcKvSnapshot, registry: &PartnerRegistry, consent: &ConsentContext, invalidated: &BTreeSet, now: u64) -> Vec`. For each registered source in the consent-gated body set that the KV row lacks or holds only expired, it picks the UID with `collect_eid_updates`'s rules and skips a UID whose hash is in the owner's `iv`. + +Rules (spec 5.3): + +- Save only when the client set came from the body, not from the `ts-eids` cookie. +- Save nothing when the KV read failed, found the row missing, or found a tombstone. +- `/auction` returns early on consent denial before it reads KV. Then nothing is saved. +- `/_ts/page-bids` no longer ingests `ts-eids` into KV. Its `EidSyncSource::PageBids` still lets finalize run cookie upkeep from its KV read. + +- [ ] **Step 1: Write the failing tests** + +In `auction/endpoints.rs`, next to the existing `/auction` + KV tests: + +- `auction_body_only_source_is_staged_not_written`: a registered source in the body and not in KV. Assert zero writes and a `ts-eids-s` `Set-Cookie` holding it. +- `auction_cookie_sourced_client_set_stages_nothing`: no body EIDs, `ts-eids` cookie present. No `ts-eids-s`. +- `auction_with_missing_row_stages_no_gap_and_sets_m`. +- `auction_skips_source_kv_already_holds_and_fills_expired`. + +In `publisher.rs`: + +- `page_bids_never_writes_kv`: with a `ts-eids` cookie holding a registered source missing from KV, zero writes. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- auction::endpoints::tests::auction_` and `cargo test-fastly -- publisher::tests::page_bids_never_writes_kv` +Expected: failures (today `/auction` and page-bids write). + +- [ ] **Step 3: Implement** + +In finalize's stager branch, when `eid_sync_source == Some(EidSyncSource::Auction)` and `client_eids` came from the body (add a `client_eids_from_body: bool` to `EcContext`, set by `handle_auction` where it chooses body or cookie), call `kv_gap_entries` and save each as `SEntry::staged_browser`. In `publisher.rs`, remove the page-bids path that leads to a KV write. With Task 25 in place, finalize no longer writes for a returning EC, so check that page-bids has no other direct write (`grep -n "upsert\|sync_eid\|create" crates/trusted-server-core/src/publisher.rs`). + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- auction::` and `cargo test-fastly -- publisher::` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Stage the /auction KV gap and stop /_ts/page-bids KV ingest" +``` + +### Task 27: The removal overflow write + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (new `apply_removals_now`) +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (`emit_identity_cookies`) +- Test: `ec/kv.rs`, `ec/finalize.rs` + +**Interfaces:** + +- Produces: `KvIdentityGraph::apply_removals_now(&self, ec_id: &str, mutation: &IdentityMutation, context: &ApplyContext<'_>) -> RemovalWriteOutcome` with `RemovalWriteOutcome { Written, AlreadyApplied, Failed }`. It reads the row, applies only the batch's removals, opt-outs, clears and `iv` (through `mutation::apply` on a removals-only mutation), and writes with `IfGenerationMatch`. On a generation mismatch it re-reads and retries, up to 5 attempts in total, as the withdrawal path does. Any other error (a per-key rate limit included) stops at once with `Failed`. + +`emit_identity_cookies` calls `plan_r_batch`. On `RPlan::Overflow` it calls `apply_removals_now` before the response is sent. `Written` and `AlreadyApplied` set no removal cookie. `Failed` sets `split_r_batch`'s cookies. This is the second and last page-load exception to G4 (spec 6.2). + +- [ ] **Step 1: Write the failing tests** + +- `overflow_batch_is_written_before_send_with_no_cookie`: a batch of 12 removals, a live row. Exactly one `IfGenerationMatch` write and no `ts-eids-r` `Set-Cookie`. +- `overflow_write_failure_falls_back_to_split_cookies`: a failing store. Every removal is present across the new cookies, and no other cookie is rewritten. +- `overflow_retries_only_on_generation_mismatch`: a store that fails the CAS twice and then succeeds gives 3 attempts. A store that returns a non-CAS error gives 1 attempt. +- `four_openable_cookies_force_overflow`. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- overflow` +Expected: FAIL, then PASS after implementing. Model the retry loop on `ec/kv.rs`'s withdrawal tombstone loop (`grep -n "fn write_withdrawal_tombstone\|attempt" crates/trusted-server-core/src/ec/kv.rs`). + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Write overflowing removal batches before send" -m "When a removal batch does not fit one cookie, or four removal cookies are already present, it is applied to KV with the withdrawal path's CAS retries. If that write fails it is split across new cookies, never coarsened." +``` + +### Task 28: The server set in the auction merge + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (`merge_auction_eids`, about line 505) +- Modify: `crates/trusted-server-core/src/publisher.rs` (navigation and page-bids merge, about lines 5547-5568) +- Modify: `crates/trusted-server-core/src/ec/eids.rs` (`to_eids` adds provenance) +- Test: `auction/endpoints.rs`, `publisher.rs` + +**Interfaces:** + +- Produces: `merge_auction_eids(client: &[Eid], server: &ServerSet, registry: &PartnerRegistry) -> Vec`. `ServerSet` holds, per source, the servable record from the `ts-eids-s` cookie view (staged entries included) overlaid on KV, minus stubs, removed entries (`ts-eids-r`) and expired entries. Build it with `pub(crate) fn server_set(context: &EcContext, serve: &ServeContext<'_>) -> ServerSet`. + +Precedence (spec 7.1): + +| Source | Browser sent it | Browser did not send it | +| ------- | ----------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| Owned | the owner's valid value (`w == owner`, not expired), else the browser's | the owner's valid value, else a browser-seeded value if the source does not expire | +| Unowned | the browser's (today) | the server value (today) | + +Part 1 has no owners, so only the unowned row applies. Part 2 (Task 36) adds owner denials of browser candidates. A server value uses its stored `atype`, else the partner's, plus its stored UID-level `ext`. A value a provider wrote also gets that provider's EID-level `inserter`, `matcher` and `mm` from `descriptor.provenance(source)`. Sources with `bidstream_enabled = false` are left out. The merged set goes through `gate_eids_by_consent` and into `x-ts-eids` exactly as today. + +- [ ] **Step 1: Write the failing tests** + +- `server_set_includes_staged_ts_eids_s_entry_for_unowned_source_browser_did_not_send` +- `browser_value_wins_for_unowned_source` (control: today's rule) +- `removed_entry_in_ts_eids_r_is_not_sent_before_it_reaches_kv` +- `expired_entry_is_not_sent` +- `stored_atype_and_ext_are_emitted` +- `x_ts_eids_matches_what_was_sent` + +Compare whole `Vec` values with every field written out, with no `..Default::default()`. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- merge` and `cargo test-fastly -- auction::endpoints` +Expected: FAIL, then PASS. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Merge the server identity set into auction EIDs" +``` + +### Task 29: Phase 4: the single flush after send, replacing pull sync + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (`run_edgezero_pull_sync_after_send` and its callers at about lines 482-560 and 618-668) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`EcFinalizeState` gains the writer flag and the response status; `AppState` gains `ProviderCatalog` and `IdentityKeys`) +- Create: `crates/trusted-server-core/src/ec/identity/flush.rs` (`run_after_send`) +- Modify: `crates/trusted-server-core/src/ec/pull_sync.rs` (delete `dispatch_pull_sync`, `build_pull_sync_context` and `PullSyncContext`. Keep `entry_is_pull_complete` and the helpers generic pull uses) +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (writer phase 3: dry runs and pending markers) +- Test: `flush.rs`, `adapter-fastly/src/main.rs` + +**Interfaces:** + +- Produces: `pub fn run_after_send(input: AfterSendInput<'_>) -> FlushOutcome`. `AfterSendInput` holds `settings`, `catalog`, `keys`, `services`, `graph: &KvIdentityGraph`, `context: &EcContext`, `status: StatusCode` and `rate_limiter`. `FlushOutcome` is `Skipped | NoChange | Written | Dropped(&'static str) | WriteFailed`. + +Phase 3 additions in finalize (writer, returning EC, status 2xx or 304; spec 5.2): + +- For each provider that is eligible on the merged view in phase `AfterSendDryRun`: + - **Without private state:** build its `ResolveInput` with `dry_run = true` and call `request`. `Some` sets a pending marker `ps..p = now` in `staged.s_cookie`. `None` sets `ps.n = now + refresh_after_sec` in the mutation. + - **With private state:** no dry run. Set the marker when it is due on the cookie view. +- Unowned dry runs return `Some` only when the phase 1 read showed the source missing and the pull-complete marker is absent (spec 5.6). Enforce this in the coordinator before the dry run. The provider does not decide it. +- Section 6.3's phase-1 read triggers must hold. The adapter reads KV in phase 1 (the existing preload) whenever any trigger is true. Add `pub(crate) fn phase1_read_needed(context: &EcContext, catalog, settings, auction_needs_row: bool) -> bool` in `ec/identity/request.rs`, and make the publisher preload decision (`publisher.rs`, `auction_needs_row` and the preload at about lines 3110-3117 and 4819-4831) consult it in addition to today's rule. + +Phase 4 (`run_after_send`, spec 5.2, steps 1 to 5): + +1. If the request is not a returning-EC writer, or `status` is not 2xx or 304: `Skipped`. Hold-mode answers already live in the cookie. +2. If the mutation is empty and no marker was set this request: `Skipped`. No read. +3. If phase 1 read the row (`Present` with a generation) and no marker is set, use that snapshot. Otherwise call `graph.get(ec_id)`. A tombstone, `consent.ok = false`, a missing row or a failed read means drop and stop. +4. Recompute eligibility steps 4 to 6 from the live row for every marked provider. Rebuild each request from the live row (the live state for stateful providers). Dispatch with `coordinator::dispatch`. Turn the outcomes into changes and add them to a copy of the mutation. A marked provider not dispatched for a reason the live row does not show (the burst cap, or the breaker in Part 2) gets `n = max(n, now + 60)`. +5. If the live row has no `cluster_size`, compute it with `compute_cluster_size` and `set_cluster_size`. +6. `mutation::apply` against the live entry. If `changed`, write once with `IfGenerationMatch(generation)`. Never retry. Log failure at `warn` with `log_id`, and return `WriteFailed`. + +Remove `run_edgezero_pull_sync_after_send` and `prepare_pull_sync_after_send`. Their callers call `run_after_send`. The adapter captures the response status before `send_edgezero_response` consumes the response (`main.rs` around lines 510-517) and passes it in. Build the KV graph after send exactly as today (`require_identity_graph`). + +- [ ] **Step 1: Write the failing tests** + +In `flush.rs`: + +- `writer_flushes_mutation_once_with_generation_match` (1 `IfGenerationMatch` write) +- `writer_with_nothing_to_do_reads_and_writes_nothing` (0 reads, 0 writes) +- `non_2xx_or_304_status_drops_the_mutation` +- `tombstone_on_reread_drops_mutation_and_calls_no_provider` (the fake HTTP client sees zero calls) +- `marked_generic_pull_provider_dispatches_after_reread_and_its_uid_lands_in_the_same_write` +- `cas_conflict_is_not_retried` (1 attempt, `WriteFailed`) +- `burst_capped_marked_provider_gets_sixty_second_backoff` +- `missing_cluster_size_is_added_to_the_one_write` + +In the adapter (`main.rs` tests): replace `pull_sync_noop_states_skip_post_send_graph_factory` with `after_send_noop_states_skip_post_send_graph_factory`, which checks that no graph is built when there is nothing to flush. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- flush` and `cargo test-fastly` +Expected: FAIL, then PASS. + +- [ ] **Step 3: Delete pull sync's dispatcher** + +Delete `dispatch_pull_sync`, `build_pull_sync_context`, `PullSyncContext`, `InFlightPull` and `pull_rate_limit_key` from `ec/pull_sync.rs`. Port every remaining pull sync test whose behaviour is unchanged to `generic_pull.rs` or `flush.rs` before deleting it. For each behaviour change in spec 10.4, write a test named after it: + +- `generic_pull_runs_only_on_returning_writer` +- `new_visitor_reaches_pull_partners_from_second_page_load` +- `generic_pull_results_join_the_single_flush` +- `failing_pull_partner_backs_off_through_ps` +- `rate_limit_key_is_per_visitor_not_per_ip` + +- [ ] **Step 4: Run everything** + +Run: `cargo test-fastly && cargo test-fastly-reuse && cargo clippy-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add -A crates/ +git commit --signoff -S -m "Flush identity changes once after send and run pull partners there" -m "Phase 4 replaces the pull sync after-send hook. It reuses the phase 1 snapshot when no provider is marked, otherwise re-reads the row, dispatches marked providers, and applies the whole mutation in one IfGenerationMatch write that is never retried. Pull partners now run only on a returning visitor's writer navigation, and back off through ps." +``` + +### Task 30: Prove the one-write invariant, reads and G10 + +**Files:** + +- Create: `crates/trusted-server-adapter-fastly/tests/identity_writes.rs` (or a `#[cfg(test)] mod identity_writes` in `app.rs` if the adapter has no integration-test harness; check how existing adapter tests drive a full request: `grep -rn "fn .*end_to_end\|TrustedServerApp" crates/trusted-server-adapter-fastly/src | head`) +- Test only + +**Interfaces:** + +- Consumes: `WriteCountingEcKv`, the fake HTTP client, and the adapter's request-driving test helpers + +- [ ] **Step 1: Write the scenario table as tests** + +One test per row of spec 13's "One-write invariant" table. Each drives a full request through the adapter (routing, finalize, phase 4) and asserts the exact write counts by mode: + +| Test | Writes | +| ----------------------------------------------------------- | ------------------------------------------------- | +| `new_visitor_navigation_writes_once` | 1 `Add` | +| `new_visitor_collision_remints_and_writes_once` | 1 `Add` that lands (a precondition failure first) | +| `returning_visitor_with_changes_writes_once` | 1 `IfGenerationMatch` | +| `returning_visitor_without_changes_writes_nothing` | 0 | +| `revalidation_304_with_changes_writes_once` | 1 | +| `returning_navigation_answered_3xx_4xx_5xx_writes_nothing` | 0 each | +| `new_visitor_prefetch_writes_once` | 1 `Add` | +| `returning_prefetch_writes_nothing` | 0 | +| `new_visitor_without_sec_fetch_dest_writes_once` | 1 `Add` | +| `returning_iframe_or_no_sec_fetch_dest_writes_nothing` | 0 | +| `integration_proxy_navigation_minting_ec_writes_once` | 1 `Add` | +| `auction_page_bids_identify_proxy_and_assets_write_nothing` | 0 each | +| `consent_withdrawal_writes_tombstone_and_marker` | today's count, per ID | +| `overflowing_removal_batch_writes_before_send` | 1 `IfGenerationMatch` | +| `cas_conflict_on_flush_is_one_attempt` | 1 failed, no retry | + +Also write: + +- **Reads:** `returning_visitor_with_recent_cookie_and_nothing_due_reads_zero_times_on_navigation`, and `auction_and_page_bids_keep_one_read`. +- **G10:** `body_only_source_reaches_kv_on_next_navigation` (an `/auction` with a body-only registered source, then a navigation: the source is in KV after the navigation's single write), and `gap_too_large_for_cookie_is_saved_again_by_next_auction`. +- **Failed flush:** `refused_flush_keeps_staged_entries_for_next_navigation`. +- **Orphans:** `deleted_row_is_found_within_kv_revalidate_sec_and_rotated` and `stager_that_sees_missing_row_sets_m_and_next_writer_recovers`. +- **Withdrawal:** `withdrawal_on_auction_after_navigation_send_makes_phase_four_drop_and_call_nothing`. + +- [ ] **Step 2: Run them** + +Run: `cargo test-fastly -- identity_writes` +Expected: PASS. A failure here is a real bug in Tasks 23 to 29. Fix it in the task that owns the behaviour, and do not loosen the test. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-adapter-fastly +git commit --signoff -S -m "Test the one-write-per-page-load invariant end to end" +``` + +### Task 31: Metrics, docs and the Part 1 PR + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` or `ec/identity/request.rs` (metrics) +- Modify: `docs/guide/edge-cookies.md`, `docs/guide/configuration.md`, `docs/guide/ec-setup-guide.md`, `docs/guide/api-reference.md`, `CHANGELOG.md`, `trusted-server.example.toml` + +- [ ] **Step 1: Metrics** + +Find how the repo emits request metrics or counters today (`grep -rn "record_eid_sync_terminal\|metric\|counter" crates/trusted-server-core/src/ec/finalize.rs | head`). Emit the same way, with no IDs and no cookie values: + +- the inbound `Cookie` header size in bytes +- the number and total sealed size of `__Host-ts-eids-r-*` cookies per request +- KV reads and writes per request, labelled by route kind (writer, stager, new EC) +- phase 4 outcome (`FlushOutcome` as a label) + +Add one test that asserts the counts are recorded for a writer navigation. + +- [ ] **Step 2: Docs** + +- `edge-cookies.md`: add sections for `__Host-ts-eids-s` and `__Host-ts-eids-r-*` (purpose, sealed, host-only, `HttpOnly`, one-year lifetime, 2,048 and 512 byte caps, never read by the shim, never sent to the origin or Prebid Server). Also describe the writer-request rule in plain words: only a full page navigation writes KV, once, after the page is sent. +- `configuration.md`: the `[ec.identity]` table, with every key, its default and its meaning, from spec 10.1. Note that `ec.pull_sync_concurrency` is deprecated. +- `ec-setup-guide.md`: pull partners now run from a visitor's second page load and back off on errors. +- `api-reference.md`: `/_ts/api/v1/identify` and `/_ts/page-bids` no longer write KV, and `/identify` omits expired IDs. +- `CHANGELOG.md` `### Changed`: one entry each for fewer KV writes (with the writer-request rule), `/auction`, `/_ts/page-bids` and `/identify` no longer writing, pull partners from the second page load, and the two new cookies. Mark the rollback note: after this release, do not roll back past it once providers write metadata (spec 14). + +Run `cd docs && npm run format` and the root markdown prettier command from `AGENTS.md` gate 8. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/ docs/ CHANGELOG.md trusted-server.example.toml +git commit --signoff -S -m "Document the single identity write and the new identity cookies" +``` + +- [ ] **Step 4: Part 1 gate and PR** + +Run `/check-ci`. Every gate must pass, including `scripts/lint-wasm-clock.sh` (new core code must not call `std::time::{Instant,SystemTime}::now`. Use `checked_current_timestamp()` and `web_time` where you need a clock) and the parity crate. Open the PR against `feat/1246-identity-cookie-strips` with `pr-creator`. Title: "Write identity KV once per page load". The description lists the behaviour changes from spec 14 step 1 and the `test-utils` feature reuse. + +--- + +# Part 2: Providers (spec 14, step 2) + +Part 2 lets integrations register providers, adds ownership, hold mode, passive capture, provider state and sticky opt-outs, the circuit breaker, and the consent decoding that strict policies need. No vendor ships here: ID5, LiveRamp, Lockr and UID2 each get their own spec and plan (spec 2.2). This part proves the framework with the fictional `exampleid` and capture providers from spec 4.6, registered only in tests. + +Before you start: `git switch feat/1246-identity-single-write && git switch -c feat/1246-identity-providers`. + +### Task 32: Registering identity providers + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`IntegrationRegistration`, the builder around line 641, `with_plan` around line 842) +- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (`from_settings` adds registered providers) +- Test: `integrations/registry.rs` + +**Interfaces:** + +- Produces: + - `IntegrationRegistrationBuilder::with_identity_provider(self, provider: Arc) -> Self` + - `IntegrationRegistrationBuilder::try_with_identity_provider(self, provider: Result>) -> Result>`. A failure becomes `TrustedServerError::Integration { integration: , message: }`. + - `IntegrationRegistry::identity_providers(&self) -> Vec>` + - In `with_plan`: every capture path and prefix must lie under one of the registering integration's `IntegrationProxy::routes()`. Otherwise it is a `Configuration` error naming the integration. + - `ProviderCatalog::from_settings(settings, partner_registry, integration_registry)` adds the registered providers after generic pull, and rejects duplicate IDs. + +- [ ] **Step 1: Write the failing tests** + +```rust + #[test] + fn try_with_identity_provider_converts_a_constructor_error_once() { + let failing: Result> = + Err(IdentityProviderError::config("bad partner id")); + let error = IntegrationRegistration::builder("exampleid") + .try_with_identity_provider(failing) + .expect_err("should fail"); + assert!( + matches!(error.current_context(), TrustedServerError::Integration { integration, .. } if integration == "exampleid"), + "should name the integration" + ); + + let built = IntegrationRegistration::builder("exampleid") + .try_with_identity_provider(Ok(ExampleTestProvider::new())) + .expect("should register") + .build(); + assert_eq!(built.identity_providers.len(), 1, "control: one provider registered"); + } + + #[test] + fn capture_rule_outside_the_integrations_routes_is_rejected_at_startup() { + let settings = settings_with_capture_integration("/integrations/other/api"); + let error = IntegrationRegistry::with_plan(&settings, empty_plan()).expect_err("should reject"); + assert!(format!("{error}").contains("examplecapture"), "should name the integration"); + let ok = settings_with_capture_integration("/integrations/examplecapture/api/token"); + IntegrationRegistry::with_plan(&ok, empty_plan()).expect("control: a rule under its own routes"); + } +``` + +`ExampleTestProvider` and the capture integration are test-only types defined in this module's tests. The capture integration registers an `IntegrationProxy` with routes under `/integrations/examplecapture/` and a capture provider whose path comes from the helper's argument. Follow how existing registry tests register a stub integration (`grep -n "from_request_filters\|fn stub_" crates/trusted-server-core/src/integrations/registry.rs | head`). + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- integrations::registry::tests::try_with_identity_provider` and `cargo test-fastly -- integrations::registry::tests::capture_rule` +Expected: FAIL, then PASS. Add `identity_providers: Vec>` to `IntegrationRegistration` and merge it in `with_plan`, as proxies are merged. A route matches a capture path when the path equals the route's path or, for a wildcard route, starts with its prefix. Mirror how `handle_proxy` matches routes today. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Let integrations register identity providers" +``` + +### Task 33: Ownership and deploy-time validation + +**Files:** + +- Modify: `crates/trusted-server-core/src/settings.rs` (`EcPartner` gains `owner`) +- Modify: `crates/trusted-server-core/src/ec/registry.rs` (`PartnerConfig.owner`) +- Modify: `crates/trusted-server-core/src/config.rs` (`validate_identity_integration`, the identity validation pass in deploy and runtime validation) +- Modify: `crates/trusted-server-core/src/ec/identity/config.rs` (`validate_identity_catalog`) +- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (owners from `[[ec.partners]].owner`, plus `sources` narrowing) +- Modify: `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (an owned pull partner uses `pull_sync_ttl_sec` as `refresh_after_sec`) +- Test: `config.rs`, `ec/identity/config.rs` + +**Interfaces:** + +- Produces: + - `EcPartner.owner: Option`, `#[serde(default, skip_serializing_if = "Option::is_none")]` + - `fn validate_identity_integration(settings: &Settings, id: &str) -> Result, Report>`. It parses `[integrations.]` as `T` and, when enabled, calls `build_descriptor`. It never builds a provider and never reads secrets. + - `pub(crate) fn validate_identity_catalog(descriptors: &[ProviderDescriptor], partners: &[EcPartner], identity: &IdentitySettings) -> Result<(), Report>`, with every error of spec 10.2: + - `owner` names a provider that is not enabled + - `owner` names a provider whose (narrowed) descriptor lacks that source + - an enabled provider claims a source with no `[[ec.partners]]` entry + - two enabled providers claim a source one of them owns (`sources` narrowing applied first) + - `sources` names an undeclared source, or narrows to nothing + - two providers have the same ID + - `[ec.identity.providers.]` names an unknown provider + - more than 32 enabled providers, generic pull included + - The validation runs in `validate_settings_for_deploy` (next to `PartnerRegistry::validate_config_for_deploy`) and in `validate_settings_for_runtime`. The deploy path collects descriptors from a new `identity_descriptors_for_deploy(settings)` list, which has one `validate_identity_integration::` line per identity integration. It is empty in this PR except under `#[cfg(test)]`, where the test `ExampleIdConfig` is added, so the list's shape is exercised. + +- [ ] **Step 1: Write the failing tests** + +One test per error above, each paired with a control config that passes. Also: + +- `config_without_ec_identity_is_accepted` +- `build_descriptor_runs_in_deploy_validation_without_secrets`: a config whose secret field holds a key name, not a value, still validates + +For example: + +```rust + #[test] + fn owner_must_name_an_enabled_provider_that_declares_the_source() { + let descriptors = vec![descriptor("exampleid", &["exampleid.example.com"])]; + let ok = vec![partner("exampleid.example.com", Some("exampleid"))]; + validate_identity_catalog(&descriptors, &ok, &IdentitySettings::default()).expect("control"); + + let unknown = vec![partner("exampleid.example.com", Some("missing"))]; + let error = validate_identity_catalog(&descriptors, &unknown, &IdentitySettings::default()).expect_err("unknown owner"); + assert!(format!("{error}").contains("missing"), "should name the owner"); + + let wrong_source = vec![partner("exampleid.example.com", None), partner("other.example.com", Some("exampleid"))]; + validate_identity_catalog(&descriptors, &wrong_source, &IdentitySettings::default()).expect_err("owner lacks source"); + } +``` + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- ec::identity::config` and `cargo test-fastly -- config::` +Expected: FAIL, then PASS. + +- [ ] **Step 3: CLI check** + +Run: `./scripts/test-cli.sh` +Expected: PASS. `ts config validate` and `ts config push` call the deploy validation, so the CLI tests exercise the new pass. If a CLI fixture config now fails, it is a real config error by the new rules. Fix the fixture and say so in the commit. + +- [ ] **Step 4: Commit** + +```bash +git add -A crates/ +git commit --signoff -S -m "Add source ownership and validate identity provider config at deploy" +``` + +### Task 34: Consent decoding for strict policies, and the consent fingerprint + +**Files:** + +- Modify: `crates/trusted-server-core/src/consent/gpp.rs` (around `decode_us_sale_opt_out` at line 125, and the section-6 skip at line 112) +- Modify: `crates/trusted-server-core/src/consent/types.rs` (decoded fields, and a redacting `Debug` for `ConsentContext`) +- Modify: `crates/trusted-server-core/src/consent/mod.rs` (`stricter_of`) +- Modify: `crates/trusted-server-core/src/ec/identity/policy.rs` (new blocks, and the fingerprint bits) +- Test: `consent/gpp.rs`, `consent/mod.rs`, `policy.rs` + +**Interfaces:** + +- Produces: + - Decoded on the GPP form: `us_sharing_opt_out: Option`, `us_targeted_advertising_opt_out: Option`, `us_gpc: Option` and `us_known_child: Option` (US National section), and the section 6 US Privacy string read through the existing US Privacy decoder. Use the `iab_gpp` fields the spec names: `sharing_opt_out` (US National, California) and `targeted_advertising_opt_out` (US National and the state sections that define it). Check the `iab_gpp` version in `Cargo.lock` and read its section structs before writing the extraction. + - `policy_allows` additionally blocks a strict policy on the sharing opt-out, the targeted-advertising opt-out, GPP's GPC field, and a known child without consent (a known child blocks every provider, whatever the relaxations). + - `pub(crate) fn fingerprint_bits(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> String`: a canonical string of the TCF purpose bits and vendor bit the policy reads, the US opt-outs it does not relax, and GPC and DNT unless relaxed. Never the raw consent string. + - `pub(crate) fn stricter_of(cookie: &ConsentContext, body: &ConsentContext) -> ConsentContext`: field by field, any opt-out, missing purpose or missing vendor consent in either wins + - A hand-written `impl fmt::Debug for ConsentContext` that prints `[redacted]` for every raw string + +- [ ] **Step 1: Write the failing tests** + +- In `gpp.rs`: one decode test per new field, per section that defines it, using GPP strings built the way existing tests build them. Each asserts the decoded value, next to a control string where the field is not set. +- In `policy.rs`: `strict_blocks_sharing_targeted_gpc_field_and_known_child`, and `targeted_advertising_opt_out_without_sale_opt_out_blocks_strict`. +- In `consent/mod.rs`: `stricter_of_takes_any_opt_out_or_missing_consent_from_either_side`. +- `consent_context_debug_redacts_raw_strings`: `format!("{context:?}")` does not contain the TC string. +- `fingerprint_is_stable_across_cmp_resaves_and_changes_with_a_purpose`: two TC strings with identical decisions but different `LastUpdated` give the same bits, and dropping purpose 4 changes them. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- consent::` and `cargo test-fastly -- ec::identity::policy` +Expected: FAIL, then PASS. Do not change any existing auction consent decision. Run `cargo test-fastly -- auction::` and `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` to confirm. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src/consent crates/trusted-server-core/src/ec/identity +git commit --signoff -S -m "Decode the GPP opt-outs strict identity policies need" -m "Adds the US sharing and targeted-advertising opt-outs, GPP section 6, and the US National GPC and known-child fields. ConsentContext's Debug output no longer prints raw consent strings." +``` + +### Task 35: Rate-limiter `record` and `count`, and the circuit breaker + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/rate_limiter.rs` (trait) +- Modify: `crates/trusted-server-adapter-fastly/src/rate_limiter.rs` (`FastlyRateLimiter`) +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (breaker in dispatch, step 7) +- Test: all three + +**Interfaces:** + +- Produces: + - `RateLimiter::record(&self, key: &str) -> Result<(), Report>` and `RateLimiter::count(&self, key: &str) -> Result>`. Both are trait methods without defaults, so every implementation must decide. Fastly uses `counter.increment(key, 1)` and `counter.lookup_count(key, CounterDuration::SixtySecs)`. + - Breaker: before dispatch, `count("identity-fail:") >= breaker_failures_per_minute` gives `CallOutcome::BreakerOpen` (not dispatched, `ps` unchanged). After dispatch, only `TransportFailure` outcomes (transport errors, timeouts, 429 and 5xx) call `record("identity-fail:")`. `HoldCutOff` is a budget cut, not a provider failure, and never records (spec 4.7). + - A limiter error is logged, and the breaker is treated as closed. Skip the burst cap check on that request too, failing closed for disclosure: no dispatch. + +- [ ] **Step 1: Write the failing tests** + +In core, with a `FakeRateLimiter` in `test_support.rs` that stores counts in a `Mutex`: + +- `breaker_opens_at_threshold_and_skips_every_visitor` +- `four_xx_and_malformed_never_open_the_breaker`: 1,000 visitors with 400 responses leave `count` at 0 +- `hold_cut_off_does_not_record_a_failure` +- `burst_cap_is_checked_after_every_other_eligibility_step`: an ineligible provider never increments the burst counter + +In the adapter: a unit test for `record` and `count` against Viceroy's rate counter, following the existing `mod tests` in `rate_limiter.rs`. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- rate_limiter` and `cargo test-fastly -- ec::identity::coordinator` +Expected: FAIL, then PASS. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/ +git commit --signoff -S -m "Add a per-provider circuit breaker on transport failures" +``` + +### Task 36: Provider state, sticky opt-outs, re-grants and owner denials + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (opt-out clearing on re-grant) +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` and `publisher.rs` (owner denials filter browser candidates in the merge) +- Modify: `crates/trusted-server-core/src/ec/admin.rs` (an operator clear of `ps.o`) +- Test: `coordinator.rs`, `auction/endpoints.rs`, `ec/admin.rs` + +**Interfaces:** + +- Produces: + - `pub(crate) fn regrant_clears_opt_out(descriptor: &ProviderDescriptor, opt_out: &KvOptOut, consent: &ConsentContext, keys: &IdentityKeys, signals: PolicySignals) -> bool` (spec 8): + - **GVL vendor:** a TC string whose `LastUpdated` is after `o.at` and that grants the vendor and purposes. + - **Any provider, under GPP:** the applicable US section shows none of the opt-outs the policy reads, where the stored `fp` showed one. Compare the current `fingerprint_bits`-derived fingerprint's opt-out part. + - A jurisdiction change, a newly present GPP string, or a relaxed signal never clears. + - When it clears, the writer adds `opt_out_clear = Some(now)` to that provider's `ProviderStateChange`. Clears are writer-only, so they ride the one flush. + - Merge (spec 7.1, "Owner denials apply to browser candidates"): before the precedence table and before `gate_eids_by_consent`, drop the browser's candidate for an owned source when its UID's keyed hash is in the owner's `iv` (KV `ps.iv` ∪ `ts-eids-r` `iv`), or when the owner has a sticky opt-out (KV ∪ `ts-eids-r`). Unowned sources are unaffected. + - The owned rows of the precedence table become active: the owner's valid value wins over the browser's. When the browser did not send the source, a browser-seeded value is used only if the source does not expire. + - Admin: `DELETE /_ts/admin/ec/{id}/providers/{provider}/opt-out`, which clears `ps.o` by writing `oc = now` through the existing admin write path. Add it to the admin route table next to the EC lookup, behind the same Basic-auth coverage, and add the route to the admin coverage test. + +- [ ] **Step 1: Write the failing tests** + +- `tcf_regrant_after_opt_out_clears_it` and `jurisdiction_change_never_clears_it` +- `gpp_section_without_opt_out_clears_when_fingerprint_had_one` +- `invalidated_uid_in_auction_body_is_not_sent_before_or_after_removal_reaches_kv` +- `owner_opt_out_drops_browser_value_for_every_owned_source` +- `owner_valid_value_beats_browser_value` +- `browser_seeded_value_never_beats_browser_current_value` +- `expiring_owned_source_never_serves_browser_seeded_value` +- `unowned_source_unaffected_by_owner_denials` (control) +- `admin_clears_provider_opt_out` + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- ec::identity` and `cargo test-fastly -- auction::endpoints` and `cargo test-fastly -- ec::admin` +Expected: FAIL, then PASS. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Apply owner precedence, owner denials and opt-out re-grants" +``` + +### Task 37: Hold mode + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` (before the navigation auction dispatch, about lines 4886-4889) +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (`handle_auction`, before the merge) +- Modify: `crates/trusted-server-core/src/publisher.rs` (the `/_ts/page-bids` auction, about lines 7077-7090) +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (`run_hold`) +- Test: `coordinator.rs`, `auction/endpoints.rs`, `publisher.rs` + +**Interfaces:** + +- Produces: `pub(crate) fn run_hold(deps: &IdentityDeps<'_>, context: &mut EcContext) -> HoldResults`. It runs eligible `Hold`-phase providers within `hold_budget_ms` from the first dispatch, after recomputing eligibility from the KV row this request read (pre-disclosure revalidation, spec 4.7). It returns the answers as server-set additions for this auction and stages them: + - A stateless provider's IDs go into `staged.s_cookie` as `k: 0` with `pv`, its `ps` changes are marked `sg: 1`, and its removals and opt-outs go into `staged.new_r_batch`. On the writer they also join the mutation. + - A stateful provider (writer only) puts its whole `ProviderResult` in the mutation only. Never in the cookie. + - A cut-off call sets `n = now + hold_retry_sec` and counts no failure. +- `handle_auction` and the navigation take `IdentityDeps` inside the existing deps or context struct so neither passes 7 arguments. +- `/_ts/page-bids` runs hold mode only when its auction runs, because only then does it read KV. + +- [ ] **Step 1: Write the failing tests** + +With the fake HTTP client's latency control: + +- `auction_starts_at_the_hold_budget_when_a_provider_is_slow` (the elapsed fake time equals the budget, and the auction ran) +- `hold_answer_feeds_this_auction_and_is_saved_with_k0` +- `cut_off_call_sets_hold_retry_without_counting_a_failure` +- `stateful_hold_provider_never_runs_on_a_stager` +- `stateful_hold_answer_stays_out_of_the_cookie` +- `writer_navigation_with_non_2xx_status_keeps_hold_answers_in_its_cookie` +- `cookie_only_no_match_stops_a_recall_on_the_next_auction` (the merged view) +- `kv_opt_out_blocks_dispatch_even_when_the_cookie_lacks_it` + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- hold` +Expected: FAIL, then PASS. Use the platform's time source the auction orchestrator uses for its own timeouts. Do not use `std::time::Instant::now` (`scripts/lint-wasm-clock.sh` rejects it in shared crates). + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Run hold-mode identity providers before the auction" +``` + +### Task 38: Passive capture + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`handle_proxy`, about lines 1031-1061) +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (`run_capture`) +- Test: `integrations/registry.rs`, `coordinator.rs` + +**Interfaces:** + +- Produces: `pub(crate) fn run_capture(deps: &IdentityDeps<'_>, context: &mut EcContext, method: &Method, path: &str, request_consent: Option, response_status: StatusCode, response_headers: &HeaderMap, response_body: &[u8])` + +`handle_proxy` per request, only when some provider's `Captures::matches(method, path)`: + +1. **Before `handle`:** if a matching descriptor declares `with_request_consent` and the request has a body, buffer it with `collect_body_bounded` (`integrations/mod.rs:182`, 256 KiB). Extract only the declared JSON pointers, decode them into a `ConsentContext` with the same decoders the cookie path uses, and re-wrap the bytes with `EdgeBody::from(bytes)` before calling `handle`. A body over the cap or not valid JSON gives `request_consent = None` and allows no positive IDs. +2. **After `handle`,** on 2xx only: buffer the response up to `max_response_bytes`. A larger body is passed through uncaptured. Decode a copy with `decode_body`, call `capture` per matching provider, and re-wrap the original bytes so the browser receives the response unchanged. +3. **Apply:** + - Removals and opt-outs always apply, whatever the consent. + - Positive IDs apply only when eligibility steps 1 to 3 pass under `stricter_of(cookie, body)` and the provider has no sticky opt-out, unless this result clears it. A capture-only provider without a GVL ID that returns IDs under consenting cookie and body consent stages `ps.oc = now` beside them. + - Capture never touches `n`, `f` or the breaker. + - A state change in a capture result is ignored with a warning. + - Results are staged on `EcContext` and written by finalize: positive into `ts-eids-s`, removals into `ts-eids-r`. On a writer request they also join the mutation. + +- [ ] **Step 1: Write the failing tests** + +Using a test capture integration (from Task 32) and its provider: + +- `capture_runs_only_for_matching_rules` +- `gzip_and_brotli_responses_are_decoded_for_capture_and_forwarded_unchanged` (byte-equal body to the browser) +- `only_declared_consent_pointers_are_read_from_the_request_body` (a body with an email field. The provider sees no body, and a log capture shows no email) +- `stricter_body_consent_blocks_positive_ids_but_not_removals` +- `revoke_style_opt_out_removes_every_record_the_provider_wrote_including_stubbed_sources` +- `capture_removal_survives_a_concurrent_auction_rewriting_ts_eids_s` (two responses built from the same inbound cookies: the `ts-eids-r` cookie from the capture is still present after the `/auction` response's `ts-eids-s` is applied) +- `revoke_then_reconsent_capture_ends_with_new_ids_and_oc_in_kv` (across a navigation, and again after a failed flush) +- `capture_never_touches_n_f_or_the_breaker` +- `capture_state_change_is_ignored` +- `oversized_proxied_body_passes_through_uncaptured` + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- capture` +Expected: FAIL, then PASS. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Capture identity IDs from proxied provider responses" +``` + +### Task 39: The full provider harness and conformance checks + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` +- Create: `crates/trusted-server-core/src/ec/identity/examples.rs` (`#[cfg(test)]` only): the spec 4.6 `ExampleIdProvider` and the capture example, as reference providers +- Test: `examples.rs` + +**Interfaces:** + +- Produces everything in spec 13's harness listing: `ProviderHarness::{with_setup, with_client_ip, with_user_agent, with_page_url, with_region, with_hashed_email, with_consent, with_stored, with_state, with_now, with_random_seed, as_dry_run, request, request_json, respond, respond_raw, capture, resolve, state_of, check_conformance, check_conformance_with_state}`, `HarnessSetup`, `TestConsent`, `CaptureFixture`, `ConformanceCases`, `StoredId::{for_test, written_by, expiring_at}`, `HashedEmail::for_test`, and `check_descriptor_ignores_secrets`. All of them sit behind `#[cfg(any(test, feature = "test-utils"))]` and are reachable from outside the crate as `trusted_server_core::ec::identity::test_support::*`, so the module is `pub` under the gate. + +`check_conformance` asserts every bullet of spec 13's conformance list: + +- inputs hold only what the descriptor declares +- a resolver returns `Some` for at least one listed setup, and its `response` never returns the `not_implemented` marker +- a capturer returns a non-`no_match` result for at least one fixture, and never the marker +- no undeclared input or state read is recorded +- no state is returned without `with_private_state()` +- requests go only to declared hosts, over HTTPS, with no `Cookie` header and with valid headers +- IDs are only for declared sources, at most one per source +- `expiry()` and `refresh()` report what was set, and a fixture ID of an expiring source carries an expiry +- `capture` never returns a state change +- a bare-JSON-string stored state is never treated as valid +- the dry run is discarded unsent, and `as_dry_run()` fails for a stateful provider + +`check_conformance` fails with a pointer to `check_conformance_with_state` for a stateful provider. `check_conformance_with_state::` also round-trips `S` through `replacing_state` and `input.state()`. + +- [ ] **Step 1: Write the tests (the spec's examples are the first harness cases)** + +In `examples.rs`, copy `ExampleIdConfig`, `ExampleIdProvider` and the capture-only example verbatim from spec 4.6, then: + +```rust + #[test] + fn example_provider_round_trip() { + let provider = ExampleIdProvider::new(&example_config()).expect("should build the provider"); + let mut harness = ProviderHarness::new(&provider) + .with_client_ip("192.0.2.10") + .with_user_agent("ExampleBrowser/1.0"); + + let request = harness.request().expect("should build a request").expect("should not skip the request"); + assert_eq!(request.url().host_str(), Some("api.exampleid.example.com"), "should call the declared host"); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "uid": "example-uid", "signature": "example-signature" })) + .expect("should parse the response"); + assert_eq!(resolution.ids()[0].uid(), "example-uid", "should return the uid"); + let state: ExampleIdState = harness.state_of(&resolution).expect("should replace state"); + assert_eq!(state.signature, "example-signature", "should keep the signature"); + } + + #[test] + fn example_provider_conforms() { + let provider = ExampleIdProvider::new(&example_config()).expect("should build"); + ProviderHarness::new(&provider).check_conformance_with_state::( + &ConformanceCases::default().with_resolve_case( + HarnessSetup::default().with_client_ip("192.0.2.10").with_user_agent("ExampleBrowser/1.0"), + StatusCode::OK, + &json!({ "uid": "example-uid", "signature": "s" }), + ), + ); + } + + #[test] + fn conformance_catches_an_undeclared_read() { + let provider = UndeclaredUserAgentReader::new(); + let result = std::panic::catch_unwind(|| { + ProviderHarness::new(&provider).check_conformance( + &ConformanceCases::default().with_resolve_case(HarnessSetup::default(), StatusCode::OK, &json!({})), + ); + }); + assert!(result.is_err(), "should fail a provider that reads an undeclared input through .ok()"); + } + + #[test] + fn descriptor_ignores_secrets() { + check_descriptor_ignores_secrets(&example_config(), &example_config()); + } +``` + +Add negative conformance tests the same way for: a `Cookie` header, a request to an undeclared host, two IDs for one source, a capability declared but not overridden (the `not_implemented` marker), a stateless provider returning state, a capture returning a state change, and an expiring source without an expiry. Each uses a tiny test provider written in the test module, and each asserts the harness panics. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- ec::identity::examples` +Expected: FAIL, then PASS. + +- [ ] **Step 3: Check the feature gate from outside the crate** + +Run: `cargo check -p trusted-server-adapter-fastly --target wasm32-wasip1 --tests --features trusted-server-core/test-utils` (or whichever adapter already enables `test-utils` in its dev-dependencies) +Expected: builds. This proves a provider crate can use the harness. + +- [ ] **Step 4: Commit** + +```bash +git add -A crates/trusted-server-core/src +git commit --signoff -S -m "Add the provider harness and conformance checks" +``` + +### Task 40: Batch sync never replaces an owned value + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/batch_sync.rs` (around line 247) +- Test: `ec/batch_sync.rs` + +- [ ] **Step 1: Write the failing test** + +`batch_sync_update_to_owned_source_applies_only_when_absent`: with the source owned by a provider, a batch update applies when the source is absent, and is `Unchanged` (no write) when a value exists. A control update to an unowned source keeps today's behaviour. + +- [ ] **Step 2: Run, see it fail, implement, see it pass** + +Run: `cargo test-fastly -- ec::batch_sync` +Expected: FAIL, then PASS. The batch sync handler needs to know the owners. Pass `&ProviderCatalog` (or just the owned source set) through its existing deps. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/trusted-server-core/src/ec/batch_sync.rs +git commit --signoff -S -m "Keep batch sync from replacing an owned source's value" +``` + +### Task 41: Docs, the provider author guide, and final verification + +**Files:** + +- Create: `docs/guide/identity-providers.md` (provider author guide) +- Modify: `docs/guide/configuration.md` (`[[ec.partners]].owner`, per-provider tables), `docs/guide/edge-cookies.md`, `docs/.vitepress` sidebar config (find it with `grep -rn "edge-cookies" docs/.vitepress`), `CHANGELOG.md` + +- [ ] **Step 1: Write the author guide** + +Base it on spec 4.2 to 4.6 and 13, using only the fictional `exampleid` provider: + +- what a provider is +- the descriptor +- declaring inputs and reading them with `?` or `.ok()` +- building requests +- parsing responses +- returning a `Resolution` +- private state +- capture +- registration with `try_with_identity_provider` +- testing with `ProviderHarness` and `check_conformance` +- what the coordinator does for you (consent, transport, backoff, KV) + +Link to the spec for the protocol. + +- [ ] **Step 2: Config and changelog** + +Document `owner`, `mode`, `timeout_ms`, `sources` narrowing, `refresh_after_sec`, `max_calls_per_minute`, `breaker_failures_per_minute` and `cookie_priority`, each with its default and a validation error example. In `CHANGELOG.md` under `### Added`, add the provider framework, ownership, hold mode, passive capture and the breaker. Under `### Changed`, add stricter consent decoding for identity providers. + +Run the docs and markdown format commands. + +- [ ] **Step 3: Commit** + +```bash +git add docs/ CHANGELOG.md +git commit --signoff -S -m "Document the identity provider framework for operators and authors" +``` + +- [ ] **Step 4: Full verification** + +Run `/check-ci`, then `cargo test-fastly-reuse`, `./scripts/test-cli.sh`, the parity crate, `scripts/lint-wasm-clock.sh`, and `cargo doc --no-deps --all-features -p trusted-server-core`. Every one must pass. Then use the `verify-app` agent for an independent pass. + +- [ ] **Step 5: PR** + +Open the PR against `feat/1246-identity-single-write` with `pr-creator`. Title: "Add server-side identity providers". The description lists spec 14 step 2's contents, says no vendor ships in it, and names the follow-up vendor specs (ID5, LiveRamp, Lockr, UID2). + +--- + +## Spec coverage + +| Spec section | Tasks | +| --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| 2.1 G1, G2, G11 (trait, coordinator, ergonomics) | 8 to 11, 20, 21, 39 | +| 2.1 G3 (hold or after-send) | 19, 20, 29, 37 | +| 2.1 G4 (one write) | 25, 26, 27, 29, 30 | +| 2.1 G5 (cookie first, then KV) | 16, 17, 25, 28 | +| 2.1 G6 (ownership) | 33, 36 | +| 2.1 G7 (expiry, refresh) | 14, 15, 20, 36 | +| 2.1 G8 (pull sync on the trait) | 22, 29 | +| 2.1 G9 (HEM slot) | 9 (always empty in v1) | +| 2.1 G10 (no lost EIDs) | 25, 26, 30 | +| 2.1 G12 (removals win) | 15, 18, 25, 27, 38 | +| 3.4 platform facts (status capture, real-browser gate, clock failure) | 14, 23, 29 | +| 4.2 trait and additivity rules | 11, 39 | +| 4.3 descriptor | 8, 32 (capture-path check) | +| 4.4 inputs | 9, 23 (phase-1 captures) | +| 4.5 requests, responses, results | 10, 20 | +| 4.6 example provider and registration | 32, 39 | +| 4.7 eligibility, dispatch, hold timing, breaker, capture | 20, 21, 35, 37, 38 | +| 5.1 writer and stagers; withdrawal exception | 23, 25 | +| 5.2 phases 1 to 4 | 23, 25, 29 | +| 5.3 stagers, KV gap, flush checks | 25, 26 | +| 5.4 `IdentityMutation` | 15 | +| 5.5 orphans, collisions, deletion | 25, 30 (the deletion framework itself is deferred; its tombstone rule is a note for that work) | +| 5.6 removal, marker and cookie rules | 15, 17, 18, 25, 29 | +| 6.1 KV record, rollback rule, admin redaction | 6, 14 | +| 6.2 cookies, sealing, size caps, overflow, outbound strips | 1 to 4, 16, 17, 18, 27 | +| 6.3 read order | 17, 29 | +| 7.1 merge, servability, `/identify` consent | 14, 28, 36 | +| 7.2 ownership semantics, batch sync | 33, 36, 40 | +| 8 errors and outcomes, sticky opt-outs, fingerprint | 13, 20, 34, 36 | +| 9.1 consent | 12, 34 | +| 9.2 outbound requests | 1 to 3, 21 | +| 9.3 logging and redaction | 9, 10, 34, every task's logging | +| 9.4 caching | 25 (cookies only when needed; the privacy rule already applies) | +| 10.1 to 10.3 configuration, validation, registration | 19, 32, 33 | +| 10.4 pull sync on the trait | 22, 29 | +| 11 adapters | 3 (strips on every adapter), 31 | +| 12 vendor examples | Out of scope (spec 2.2). The fictional examples in 39 prove the API | +| 13 testing | each task, 30, 39 | +| 14 rollout | Parts 0, 1, 2 | +| 15 open questions | Not implemented. Track as issues after Part 1 ships | From b170c44138e36fec2fc25944ede916046c148777 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 23 Oct 2026 19:18:20 +0530 Subject: [PATCH 05/89] Revise the identity providers spec to revision 12 Fold in findings from #1246 and the #1260 review: the ORTB2 EID dependency, the first-auction race metric, the empty GPP string, capture off by default, a capture rollout checklist and the cookie lifetime decision. Propose, pending review, a real-browser gate on capture effects, an operator-gated global revoke and dedicated rotating cookie keys. Signed-off-by: dhruv8sh --- ...0-server-side-identity-providers-design.md | 490 ++++++++++++++---- 1 file changed, 385 insertions(+), 105 deletions(-) diff --git a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md index e6fa017e64..e4df65fdac 100644 --- a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md +++ b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md @@ -1,6 +1,6 @@ # Server-side identity providers -Status: Draft, revision 11. Revised after review rounds 1 to 10. Section 17 +Status: Draft, revision 12. Revised after review rounds 1 to 11. Section 17 maps each finding to its fix. Issue: none filed yet. Related: [#1245](https://github.com/IABTechLab/trusted-server/issues/1245) @@ -19,7 +19,11 @@ with `adapter-fastly/src/`. Prebid.js references are to version 10.26.0, read from `crates/trusted-server-js/lib/node_modules/prebid.js/modules/`. This spec depends on the EID fidelity branch for `merge_auction_eids`, the shared client EID parser and the `ts-eids` cookie carrying the same set the shim -sends. +sends. It assumes that work forwards configured ORTB2 EIDs together with +User ID module EIDs, deduplicated, with EID-level provenance preserved +(EID fidelity spec D2, D3, §4.2, §4.4). Lockr writes its ID5 EID only into +`pbjs.getConfig('ortb2').user.ext.eids`, which a collector that reads only +`getUserIdsAsEids()` misses (#1246). This spec does not depend on PR #1260. Its `IdentityCapture` trait, `identity_owner` field and `ts-eids-s` cookie are not on `main`. Where this @@ -155,8 +159,9 @@ can be written, and each extra write is another CAS race between tabs. mode. - **G4.** At most one browser-driven KV write per page load, and a write that fails is not retried. This is strict. The only page-load exceptions - are a consent withdrawal's tombstone (section 5.1) and a removal batch that - the removal cookies cannot hold (section 6.2). Writers outside page loads + are a consent withdrawal's tombstone (section 5.1), which also covers a + capture provider's global revoke (section 8), and a removal batch from a + real browser that the removal cookies cannot hold (section 6.2). Writers outside page loads keep their own writes: batch sync, the admin endpoints and the deferred deletion framework (sections 2.2, 5.5). - **G5.** Provider-collected IDs are stored in both a first-party cookie @@ -183,9 +188,10 @@ can be written, and each extra write is another CAS race between tabs. methods (section 4.2). - **G12.** Removals win. An ID removed from KV (opt-out, invalidation, withdrawal, deletion) stops reaching bidders within a bounded time, even - from the cookie. A removal staged by a stager stops that browser's - bidstream at once and reaches KV at the next writer navigation; a client - that never makes one keeps it in `ts-eids-r` until it does. + from the cookie. A removal staged by a stager stops that client's + bidstream at once and reaches KV at the client's next writer navigation; a + client that never makes one, such as a script that fails the real-browser + check (section 4.7), keeps it in `ts-eids-r` until it does. ### 2.2 Non-goals @@ -1407,8 +1413,19 @@ counters are kept per POP, so the breaker is too. **Passive capture.** `IntegrationRegistry::handle_proxy` (`integrations/registry.rs:1031-1060`) holds the `EcContext`. `IntegrationProxy::handle` itself has no EC context -(`integrations/registry.rs:319-324`). For each provider with a capture -rule (`with_path` or `with_path_prefix`) matching the request: +(`integrations/registry.rs:319-324`). + +Capture is off by default. `capture_enabled` (section 10.1) defaults to +`false` for every provider that declares `captures`. While it is false, the +coordinator still runs `capture`, discards the result's positive IDs and +logs at debug what it discarded, by provider ID and source only. The +result's removals and opt-outs still apply as below, because removals win +(G12): an ID captured while capture was on must stay removable after it is +turned off. An operator turns capture on per provider after the rollout +checklist (section 14). + +For each provider with a capture rule (`with_path` or `with_path_prefix`) +matching the request: - **Before `handle`.** When the descriptor's `Captures` declares `with_request_consent` and the request has a body, `handle_proxy` buffers it with the existing @@ -1426,13 +1443,22 @@ rule (`with_path` or `with_path_prefix`) matching the request: - **After `handle`,** on 2xx responses only, the coordinator decodes `Content-Encoding` under the byte caps, builds a `CaptureInput` and calls `capture`. -- **It applies the result.** Removals and opt-outs always apply, whatever the +- **It applies the result.** Removals and opt-outs apply whatever the consent, because a revocation arrives after consent is denied. Positive - IDs apply only when eligibility steps 1 to 3 pass under the stricter - consent, and the provider has no sticky `ps.o`, unless this result clears - it: a capture-only provider without a GVL ID that returns IDs under - consenting cookie and body consent stages a clear, `ps.oc = now`, beside - its IDs in `ts-eids-s` (section 8). + IDs apply only when `capture_enabled` is true, eligibility steps 1 to 3 + pass under the stricter consent, and the provider has no sticky `ps.o`, + unless this result clears it: a capture-only provider without a GVL ID + that returns IDs under consenting cookie and body consent stages a clear, + `ps.oc = now`, beside its IDs in `ts-eids-s` (section 8). +- **The real-browser gate.** Eligibility step 1 already keeps positive IDs + from a client that fails `looks_like_browser()`. Such a client's removals + and opt-outs go only into its own response's `ts-eids-r` cookie. They never + trigger the removal overflow write (section 6.2), and a global revoke + (section 8) is staged as a provider opt-out instead. The batch reaches KV + only if that client later makes a writer navigation, which needs a real + browser (section 5.1). So a script replaying a copied `ts-ec` cookie + through the proxy can neither enrich the row nor strip or tombstone it. + Consent withdrawal keeps its own exception (section 5.1). The result is staged on `EcContext`. `ec_finalize_response`, which already runs for proxy responses (`adapter-fastly/src/app.rs:909`, @@ -1516,9 +1542,12 @@ browsers (`adapter-fastly/src/app.rs:446-452`). Revocation cannot wait for a navigation that may never come. The exception covers the whole of today's withdrawal path: a CAS tombstone with up to five attempts, then its completion-marker write, for each withdrawn EC ID (one or two; -`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). It and the -removal overflow write (section 6.2) are the only page-load exceptions to -G4. The tombstone is not part of any `IdentityMutation`. A +`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). The same +exception covers a captured opt-out from a provider with +`revoke_scope = "global"` (section 8), from a real browser only: it runs this +path on the request that carries it. It and the removal overflow write +(section 6.2) are the only page-load exceptions to G4. The tombstone is not +part of any `IdentityMutation`. A writer whose phase 4 re-read finds the tombstone drops its mutation (section 5.4, step 1). @@ -1597,8 +1626,8 @@ writer whose phase 4 re-read finds the tombstone drops its mutation (section returned `Some`, only when the response status is 2xx or 304 (finalize already holds the response) - when the phase 1 read changed the cookie under the section 6.3 merge - - when the cookie was missing, failed to open or was past its - revalidation age + - when the cookie was missing, failed to open, opened under a previous + key (section 6.2) or was past its revalidation age **Phase 4, after send. Returning ECs only. The only write.** @@ -1901,7 +1930,11 @@ that seems to allow otherwise: re-gate, the writer-still-owns check and `staged_max_age_sec` in the flush checks (section 5.3), and keep only the EC binding and, when they carry one, the revision check. Withdrawing consent, disabling a provider or a - week in an SPA can delay a positive ID, never a removal. + week in an SPA can delay a positive ID, never a removal. The one gate a + removal does face is the real-browser gate on capture (section 4.7): a + capture removal from a client that fails `looks_like_browser()` stays in + that client's own `ts-eids-r` and reaches KV only through that client's + own writer navigation, never through the removal overflow write. - **A config change keeps removals.** When the `cf` of `ts-eids-s` does not match, its positive entries are discarded and it is rebuilt from KV. Pending markers are kept. `ts-eids-r` carries no `cf`, so its removals @@ -1937,8 +1970,10 @@ that seems to allow otherwise: cannot bring IDs back. It may still set removals in a new `ts-eids-r` cookie. - **The cookie is rewritten only when it changes.** A request that read KV - re-emits `ts-eids-s` only when its entries, markers or `ps` changed, or - when the inbound `kc` is older than half of `kv_revalidate_sec`. Phase 3 + re-emits `ts-eids-s` only when its entries, markers or `ps` changed, when + the inbound `kc` is older than half of `kv_revalidate_sec`, or when it + opened under a previous key (section 6.2), so it moves to the current key. + Phase 3 stamps `kc` only when this request read KV. This keeps `Set-Cookie` off most responses and narrows the window for concurrent overwrites. - **Cluster size rides on a write.** `/_ts/api/v1/identify` is a @@ -2058,7 +2093,9 @@ Two server-owned cookies: Each response that produces a removal or opt-out sets a new one, with a fresh random 8-hex-digit ``, holding only its own batch. No response rewrites another's cookie, so neither a concurrent `/auction` rewriting - `ts-eids-s` nor another removal in flight can lose a batch. Stagers never + `ts-eids-s` nor another removal in flight can lose a batch. The one + rewrite is the writer's re-seal after a key rotation (below), which keeps + the same name and the same batch. Stagers never delete one. They are deleted, by name, in three cases only: by the writer navigation, after its phase 1 read shows a cookie's entries all applied or the cookie fails to open; by any request whose KV read confirms a @@ -2083,12 +2120,16 @@ every request from the browser, through any intermediary that logs headers. A signed cookie would show server-only IDs, such as UID2 tokens and LiveRamp envelopes, in plain text there, and to the origin if the strip ever regressed. So each value is sealed with AES-256-GCM: -`v1..`, base64url without padding. The key is -derived from `ec.passphrase` with a namespaced label per cookie -(`trusted-server/ts-eids-s/key/v1`, `trusted-server/ts-eids-r/key/v1`), -following the pull-complete marker's pattern. The nonce is 12 random bytes -from the platform generator. The full EC ID is the associated data, so it is -authenticated but not stored: +`v1...`, with the nonce and ciphertext in +base64url without padding. `` names the sealing key: 1 to 8 +characters of `[a-z0-9]`, in plain text, because it is not secret. The keys +are dedicated secrets from the secret store, listed in +`[ec.identity.cookie_keys]` (section 10.1) and in `secret_fields()` (section +10.2). They are not derived from `ec.passphrase`. Each cookie's AES key is +derived from a keyring secret with a namespaced label +(`trusted-server/ts-eids-s/key/v1`, `trusted-server/ts-eids-r/key/v1`). The +nonce is 12 random bytes from the platform generator. The full EC ID is the +associated data, so it is authenticated but not stored: - A cookie copied from another visitor, or left over after an EC rotation, fails to open and is ignored. @@ -2106,6 +2147,47 @@ request that reads KV. The failure is logged without the value. Sealing needs an AES-GCM implementation, a new dependency (section 14), which the UID2 provider needs anyway (section 12.3). +**Key rotation.** TS seals every new cookie with the `current` key and +opens a cookie with any key in the keyring: the current key and the +previous keys still configured. A cookie whose `` is not in the +keyring fails to open, as any other cookie that fails to open. Rotation must +never drop an unapplied removal batch, so: + +- A writer whose phase 1 read finds a `ts-eids-r` cookie opened under a + previous key and not yet all applied re-seals it under the current key, in + place: same name, same batch, a fresh nonce and a new `Max-Age`. A partly + applied cookie is re-sealed whole. Every writer navigation reads KV when a + `ts-eids-r` cookie is present (section 6.3), so it re-seals or deletes + every one it carries. Stagers never re-seal. +- A `ts-eids-s` opened under a previous key is re-emitted under the current + key by the next request that reads KV and sets it (section 5.6). +- A rotation is two deploys: add the new key to the keyring, wait until + every POP serves that config, then make it `current`. A POP that does not + yet know a key ID would otherwise fail to open the new cookies, and its + writer would delete their batches. + +How long a previous key must stay configured: + +- **`ts-eids-s`:** at least `staged_max_age_sec` (default 7 days). By then + every staged entry sealed under it is too old to flush anyway, and a + cookie that fails to open is rebuilt from KV. A client that keeps + navigating moves to the current key within `kv_revalidate_sec`. +- **`ts-eids-r`:** one year, the cookies' `Max-Age`, after the key stops + being current. A client's batch moves to the current key at its first + writer navigation after the rotation, so a shorter retention covers every + client that navigates within it. Only the full year guarantees that no + browser can still hold a batch sealed under the removed key. + +Residual risk: a client that holds an unapplied batch and makes no writer +navigation before its key is removed loses that batch when it returns. Its +writer deletes the cookie as failing to open, and the removed ID can reach +that browser's bidstream again until the provider removes it again. A +client that never returns holds a batch nobody applies, rotation or not. An +emergency rotation that removes a compromised key at once accepts this loss +for every batch sealed under it. + +The hashes of section 9.3 keep their `ec.passphrase`-derived keys. + **Why it is sealed.** Staged entries become KV writes, and sealing proves TS wrote them. Without it, a visitor could plant an arbitrary ID in KV, where it would also be sent from the server on routes without the cookie. Encryption @@ -2189,14 +2271,16 @@ when the live row shows: `iv` entries never keep a cookie alive: they are applied with the cookie's other entries and then dropped with it. The writer's phase 1 read deletes each cookie whose entries are all applied, and any `__Host-ts-eids-r-` -cookie that fails to open, for example one sealed under an old key. A -cookie that is only partly applied is kept whole, and its applied entries -are skipped when re-applied. +cookie that fails to open, for example one sealed under a key no longer in +the keyring. A cookie that is only partly applied is kept whole, and its +applied entries are skipped when re-applied. One opened under a previous +key is re-sealed (above). **Size.** Each cookie's sealed value is at most 512 bytes. The `v1.` -prefix, the base64url nonce and its separator take 20 bytes, base64url -turns every 3 bytes into 4, and the tag takes 16, so a cookie holds at most -353 bytes of payload JSON. A removal costs 56 bytes plus its source domain +prefix, a key ID of at most 8 characters, the base64url nonce and their +separators take at most 29 bytes, base64url turns every 3 bytes into 4, and +the tag takes 16, so a cookie holds at most 346 bytes of payload JSON. A +removal costs 56 bytes plus its source domain and provider ID: about four fit with typical names (a 15-byte source and an 8-byte provider), and one with a 253-byte source domain. An `iv` hash costs 19 bytes, and an opt-out about 15 bytes plus the provider ID. @@ -2205,9 +2289,12 @@ A batch is never coarsened to fit. It is not merged into another response's cookie, its removals are not widened, its `iv` hashes are not dropped, and an invalidation never becomes an opt-out. -**Overflow.** A response writes its batch to KV before send, instead of -setting a cookie, when the batch does not fit one cookie or the request -already carries four openable `ts-eids-r` cookies. This removal overflow +**Overflow.** A response to a real browser writes its batch to KV before +send, instead of setting a cookie, when the batch does not fit one cookie +or the request already carries four openable `ts-eids-r` cookies. A +response to a client that fails `looks_like_browser()` never makes this +write (section 4.7): it sets the batch in new cookies of its own, split as +needed, as when the write fails. This removal overflow write is the second page-load exception to G4 (section 5.1). It reads the row and applies only the batch's removals, opt-outs and `iv` hashes under the rules of section 5.4, with a CAS and the withdrawal path's five @@ -2271,9 +2358,11 @@ source. responses are already private. - **`ts-eids-r`:** any response that produces a removal or opt-out, on any route, including the unusable-consent path (section 5.3), adds a new - cookie for its batch, unless its overflow write landed (above); and the - writer navigation's HTML response deletes - the cookies its phase 1 read showed applied. + cookie for its batch, unless its overflow write landed (above; a client + that fails the real-browser check never makes one); and the writer + navigation's HTML response deletes the cookies its phase 1 read showed + applied and re-seals, in place, those opened under a previous key that + are not yet all applied. - **A consent withdrawal, or any KV read that finds a tombstone,** sets `Max-Age=0` on `ts-eids-s`. Once the tombstone is confirmed (written by this request, or read), the `ts-eids-r` cookies are pointless and are @@ -2385,7 +2474,9 @@ hold-mode provider): caches them with their `ra` and `exp`, so the cookie view already knows when they are due. - On the writer only, each `ts-eids-r` cookie whose entries the row shows - all applied is deleted. Stagers leave the `ts-eids-r` cookies alone. + all applied is deleted, and each one opened under a previous key that + is not yet all applied is re-sealed under the current key (section + 6.2). Stagers leave the `ts-eids-r` cookies alone. The writer's HTML response carries the result only under phase 3's rules, which include "the phase 1 read changed the cookie". @@ -2531,6 +2622,8 @@ cookies: positive changes into `ts-eids-s`, removals and opt-outs into | `Rejected` with 429 or 5xx, `Transport`, timeout | unchanged | `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`; breaker `record` | info | | Breaker open | not dispatched | unchanged | debug | | Capture: any result or error | IDs and removals as above | only `o`; never `n`, `f` or the breaker | debug | +| Capture with `capture_enabled = false` | positive IDs discarded; removals as above | only `o` | debug | +| Captured `opted_out`, `revoke_scope = "global"`, real browser | the withdrawal tombstone removes the row (section 5.1) | removed with the row | info | **Sticky opt-outs.** `ps.o = {at, fp}` records a provider-reported opt-out and the consent fingerprint at that time. Server-side calls do not carry the @@ -2560,6 +2653,18 @@ browser's mirrored copy yields to it (section 4.7). A consent withdrawal's tombstone removes everything, provider opt-outs included: after it, the visitor starts again with a new EC. +**Revoke scope.** A provider that declares `captures` has a `revoke_scope` +(section 10.1), default `"provider"`. With `"provider"`, a captured +`opted_out` is the provider opt-out above. With `"global"`, a captured +`opted_out` from a real browser runs today's consent-withdrawal tombstone +path on that request instead (section 5.1): the row is tombstoned, the EC +cookie is cleared, and everything is removed, provider opt-outs included. +It uses the withdrawal's existing exception, so G4 keeps exactly two +page-load exceptions. From a client that fails `looks_like_browser()`, a +global revoke is staged as a provider opt-out in that client's own +`ts-eids-r` (section 4.7), so a replayed `ts-ec` cannot tombstone another +browser's row. An opt-out from `resolves` is always provider-scoped. + The **consent fingerprint** is the first 16 hex digits of an HMAC, keyed from `ec.passphrase`, over the consent signals the provider's policy reads: @@ -2608,7 +2713,11 @@ Rules: auction (`auction/routing.rs:336`). - **Fail closed.** `strict()` blocks when a raw TC, GPP or US Privacy string is present but has no decoded form, for example in proxy mode or after a - decode failure (`consent/mod.rs:104-127`, `:197-210`). + decode failure (`consent/mod.rs:104-127`, `:197-210`). A GPP string that + decodes but has no applicable section, such as the header-only `DBAA` a + CMP sends when no section applies, has a decoded form and carries no + opt-out, so it does not block `strict()` by itself. Where TCF applies, the + policy still needs TCF consent from a TC string or a GPP TCF section. - **Before disclosure:** the live-row read and eligibility recomputation described in section 4.7. - **At write:** the CAS on that read's generation. @@ -2621,7 +2730,8 @@ Rules: request's visitor consent (section 7.1). Owner denials also filter the browser's candidates. - **On withdrawal:** the tombstone is written synchronously on whichever - request carries it, and the response expires `ts-eids-s` (section 5.1). + request carries it, and the response expires `ts-eids-s` (section 5.1). A + global revoke from a real browser takes the same path (section 8). ### 9.2 Outbound requests @@ -2658,7 +2768,16 @@ Rules: (`integrations/lockr.rs:202-204`). - Hashes of identifiers (the rate-limit visitor key, `pv`, `ps.iv`, the consent fingerprint) are HMACs with keys derived from `ec.passphrase`, so - they cannot be linked to an EC ID without the key. + they cannot be linked to an EC ID without the key. They do not move to the + rotating cookie keys (section 6.2). `ps.iv`, the consent fingerprint and + the legacy revision hash a `pv` can hold are stored and compared on later + requests, so each rotation would break them: a rejected UID could be + re-seeded, an opt-out's fingerprint would never match again, and a staged + removal with a legacy `pv` would look applied and be skipped. The visitor + rate-limit key would survive a rotation, because its windows last 60 + seconds, but it stays with the others. +- Cookie sealing keys come from the secret store and are never logged. A + key ID may be logged. - `CaptureInput` never exposes request bodies or request headers. Proxied requests can contain an email or HEM. TS forwards them, and reads from a body only the consent pointers a descriptor declares (section 4.7). It @@ -2723,6 +2842,28 @@ breaker_failures_per_minute = 50 # Lower is dropped first when ts-eids-s exceeds its cap. Default 0. cookie_priority = 10 +[ec.identity.providers.lockr] +# Only for providers that declare `captures`. Default false: captured +# positive IDs are discarded and logged at debug; removals still apply. +capture_enabled = true +# Only for providers that declare `captures`. "provider" or "global". +# Default "provider": a captured opt-out removes this provider's records. +# "global" runs the consent-withdrawal tombstone instead (section 8). +revoke_scope = "provider" + +# Keys that seal ts-eids-s and ts-eids-r (section 6.2). No default: +# required when `ec.ec_store` is set. +[ec.identity.cookie_keys] +# The key ID that seals new cookies. It must be one of `keys`. +current = "k2" +# The current key and every previous key still accepted for opening. Each +# `secret` is a secret-store key name at rest, resolved at startup to 32 +# bytes in base64. +keys = [ + { id = "k2", secret = "ts-identity-cookie-key-k2" }, + { id = "k1", secret = "ts-identity-cookie-key-k1" }, +] + # Source policy: the existing table, with one new field. [[ec.partners]] name = "ID5" @@ -2738,8 +2879,10 @@ enabled = true partner_id = 1234 ``` -`[ec.identity]` and each `[ec.identity.providers.]` table are optional. -A provider with no table uses the defaults. +`[ec.identity]` and each `[ec.identity.providers.]` table are optional, +except `[ec.identity.cookie_keys]`, which is required when `ec.ec_store` is +set, because the cookies cannot be sealed without it. A provider with no +table uses the defaults. ### 10.2 Validation @@ -2779,6 +2922,15 @@ Each of these is an error: - `timeout_ms` is greater than `hold_budget_ms` for a hold-mode provider. - `hold_budget_ms` is over 1,000, `max_concurrency` is 0, `refresh_after_sec` is 0, or `kv_revalidate_sec` is not below 86,400. +- `capture_enabled` or `revoke_scope` is set for a provider whose + descriptor declares no `captures`, or `revoke_scope` is not `provider` or + `global`. +- `ec.ec_store` is set and `[ec.identity.cookie_keys]` is missing. +- `current` is not one of `keys`, two keys share an `id`, or an `id` is not + 1 to 8 characters of `[a-z0-9]`. +- At startup, after secret resolution, a key's secret does not decode from + base64 to exactly 32 bytes. Deploy validation sees only key names, so this + check runs in `validate_settings_for_runtime`. Validation never calls a provider. Live approvals, such as a LiveRamp placement or an approved origin, stay on the rollout checklist of each vendor @@ -2789,7 +2941,9 @@ secret, are `Redacted` config fields. At rest the config holds their secret-store key names, and they are resolved in memory before runtime settings are built (`secret_resolution.rs`). Each identity integration adds its secret paths to `TrustedServerAppConfig::secret_fields()` -(`config.rs:136-175`). `build_descriptor` must never read a secret field, +(`config.rs:136-175`). The cookie keys' secrets are listed there too, at +`ec.identity.cookie_keys.keys[].secret`, as an optional path like the +partner tokens. `build_descriptor` must never read a secret field, because deploy validation runs before resolution and would see key names. The harness's `check_descriptor_ignores_secrets` builds the descriptor from a config whose secrets are placeholder key names and from one with resolved @@ -3194,8 +3348,17 @@ What this shows: - **Revocation outlives consent.** `revoke-consent` arrives after the visitor has denied consent, so the request's consent gate would block it. A capture's removals must apply whatever the consent (section 12.5). It is - provider-scoped here: it removes Lockr-written records only, never the EC. - Lockr's plan settles the real scope. + provider-scoped by default (`revoke_scope = "provider"`): it removes + Lockr-written records only, never the EC. An operator who sets + `revoke_scope = "global"` makes it a full withdrawal that tombstones the + row, from real browsers only (section 8). Lockr's plan settles which + scope Lockr's revocation means. +- **Capture sees a token once.** Lockr returns a token only when it first + issues one. On a reload its SDK sends the saved ID with `noGenerate`, and + the response carries no new token (#1246). So capture sees an ID only at + its first issue, and the browser path carries it on later page views: + Lockr writes the ID into Prebid's `ortb2` config, which the EID fidelity + work forwards in the `/auction` body and `ts-eids`. - **Consent in the request body.** Lockr's SDK sends its own consent strings in the body. The descriptor names them, the coordinator extracts and decodes only those fields, and capture is gated by the stricter of the @@ -3703,6 +3866,18 @@ refresh and opt-out. rule matching the request, decodes gzip and brotli bodies under the caps, reads only the declared consent pointers from a request body, and never touches `n`, `f` or the breaker. +- With `capture_enabled = false`, a capture's positive IDs reach neither + cookie nor KV and are logged at debug without values, while its removals + and opt-outs still apply. +- A capture from a client that fails `looks_like_browser()` stores no + positive ID, puts its removals and opt-outs only in its own `ts-eids-r`, + never makes the removal overflow write, and turns a global revoke into a + provider opt-out. A later writer navigation from that client applies the + batch. A replayed `ts-ec` on the proxy changes no KV row. +- With `revoke_scope = "global"`, a captured opt-out from a real browser + tombstones the row and clears the EC cookie on that request, as a consent + withdrawal does. With `"provider"`, it removes only that provider's + records. - A Lockr `revoke-consent` captured on the proxy removes every Lockr-written record, owned or unowned, including sources the cookie holds only as stubs, and survives a concurrent `/auction` rewriting `ts-eids-s`. @@ -3729,6 +3904,8 @@ refresh and opt-out. US National GPC and known-child fields decode from each section that defines them. Each one blocks a strict policy. - A raw consent string that does not decode blocks a strict policy. +- A decodable GPP string with no applicable section, the header-only + `DBAA` shape, does not block a strict policy where TCF does not apply. - A targeted-advertising opt-out without a sale opt-out produces no server-side IDs in `user.eids`, no flush of that provider's entries, and no IDs written into the cookie. @@ -3738,23 +3915,25 @@ refresh and opt-out. so adapter tests can use it. `KvIdentityGraph::counting` counts reads only (`ec/kv.rs:1596-1605`, `ec/kv_backend.rs:205-221`). -| Scenario | KV writes | -| ------------------------------------------------------------------ | ---------------------------------------------------- | -| New visitor, navigation | 1 (create, before send) | -| New visitor whose first ID collides | 1 (create with a re-minted ID) | -| Returning visitor, changes pending | 1 | -| Returning visitor, nothing changed | 0 | -| 304 revalidation with changes pending | 1 | -| Returning EC, navigation answered with 3xx, 4xx or 5xx | 0 | -| New visitor, prefetch or prerender navigation | 1 (create, before send) | -| Returning visitor, prefetch or prerender navigation | 0 | -| New visitor, client without `Sec-Fetch-Dest` | 1 (create, before send) | -| Returning visitor, iframe HTML or no `Sec-Fetch-Dest` | 0 | -| Integration-proxy navigation that mints an EC | 1 (create) | -| `/auction`, `/_ts/page-bids`, `/identify`, Lockr API proxy, assets | 0 | -| Consent withdrawal, on any request | CAS tombstone (up to 5 attempts) plus marker, per ID | -| Removal batch that overflows the removal cookies, on any request | CAS removal write before send (up to 5 attempts) | -| CAS conflict or per-key limit on the flush | 1 attempt, not retried | +| Scenario | KV writes | +| -------------------------------------------------------------------- | ---------------------------------------------------- | +| New visitor, navigation | 1 (create, before send) | +| New visitor whose first ID collides | 1 (create with a re-minted ID) | +| Returning visitor, changes pending | 1 | +| Returning visitor, nothing changed | 0 | +| 304 revalidation with changes pending | 1 | +| Returning EC, navigation answered with 3xx, 4xx or 5xx | 0 | +| New visitor, prefetch or prerender navigation | 1 (create, before send) | +| Returning visitor, prefetch or prerender navigation | 0 | +| New visitor, client without `Sec-Fetch-Dest` | 1 (create, before send) | +| Returning visitor, iframe HTML or no `Sec-Fetch-Dest` | 0 | +| Integration-proxy navigation that mints an EC | 1 (create) | +| `/auction`, `/_ts/page-bids`, `/identify`, Lockr API proxy, assets | 0 | +| Consent withdrawal, on any request | CAS tombstone (up to 5 attempts) plus marker, per ID | +| Global revoke captured from a real browser | as a consent withdrawal | +| Removal batch from a real browser that overflows the removal cookies | CAS removal write before send (up to 5 attempts) | +| Capture removal, opt-out or global revoke from a non-browser client | 0 | +| CAS conflict or per-key limit on the flush | 1 attempt, not retried | **Withdrawal and removal.** @@ -3782,6 +3961,8 @@ existing one read and no more. - A gap too large for the cookie is saved again by the next `/auction` and eventually written. - Every source today's `/auction` ingest would write reaches KV. +- An EID present only in Prebid's `ortb2` config, as Lockr writes its ID5, + reaches `ts-eids` and KV. **Orphans and collisions (section 5.5).** @@ -3840,8 +4021,20 @@ existing one read and no more. and removes one acquired in the same second as `sa` - it is honoured on the unusable-consent path, where `ts-eids-s` is expired, until the next writer applies it - - the 512-byte sealed cap per cookie, the one-year `Max-Age`, and that - stagers never delete one + - the 512-byte sealed cap per cookie with an 8-character key ID, the + one-year `Max-Age`, and that stagers never delete one +- Cookie keys: + - a cookie sealed under a previous key opens; one whose key ID is not in + the keyring fails to open + - a writer re-seals an unapplied `ts-eids-r` cookie opened under a + previous key, in place, with the same batch, and deletes an applied one; + a stager never re-seals + - a `ts-eids-s` opened under a previous key is re-emitted under the + current key by the next request that reads KV + - across a rotation, with the new key added before it becomes current, + no unapplied removal batch is lost + - `ps.iv`, the consent fingerprint and a legacy `pv` hash still match + after a cookie key rotation - The admin lookup redacts `ps.st`. **Outbound requests.** Requests to Prebid Server, in every @@ -3878,8 +4071,15 @@ nothing is left. test. **Config.** Each validation error in section 10.2, `build_descriptor` -running in `ts config validate` without secrets, and acceptance of a config -with no `[ec.identity]` at all. +running in `ts config validate` without secrets, acceptance of a config +with no `[ec.identity]` at all when `ec.ec_store` is unset, and acceptance +of one with only `[ec.identity.cookie_keys]` when it is set. The +strips-only release accepts `[ec.identity.cookie_keys]` (section 14). + +**First-auction metric.** The shim reports the race flag of section 14 on +the first `/auction` of a page load only, and only where the Lockr +integration is active. The log line carries the flag and no identifier +value. **Adapters.** Phase 4 runs after send on Fastly. Axum, Cloudflare and Spin behave as before apart from the Prebid Server and origin cookie strips. The parity test crate @@ -3896,18 +4096,32 @@ Three steps, each shippable on its own: list. Stripping `ts-eids` from origin requests is a behaviour change for any origin that read it. A rollback after step 1 then lands on code that still keeps the new one-year cookies away from the origin and its header - limit. This release is the rollback floor. + limit. This release is the rollback floor. It also parses and validates + `[ec.identity.cookie_keys]` and lists its secrets in `secret_fields()`, + without using them. `Ec` denies unknown fields (`settings.rs:541-542`), + so without this a config holding the keys would fail to load after a + rollback to it. Operators provision the keys in the secret store and add + the table during this release, so step 1 finds them. 1. **One write, no new vendors.** This step includes: - the writer-request rule and the `IdentityMutation` single flush, with no retry; the new visitor's create stays before routing, as today - `/identify` and `/_ts/page-bids` stopping their writes - `ts-eids-s`: the host-only cookie sealed with AES-256-GCM (a new - `aes-gcm` dependency, which the UID2 provider later reuses), `kc` - revalidation, KV-gap staging on `/auction`, pending and `m` markers, - the size cap - - the `ts-eids-r` removal cookies + `aes-gcm` dependency, which the UID2 provider later reuses) under the + rotating cookie keys, `kc` revalidation, KV-gap staging on `/auction`, + pending and `m` markers, the size cap + - the `ts-eids-r` removal cookies, and their re-seal after a key + rotation (section 6.2) - a metric for the inbound `Cookie` header size, and for the number and total size of `ts-eids-r` cookies (section 6.2) + - a first-auction race metric for the open question in section 15. The + shim adds one boolean to the first `/auction` body of each page load, + only where the Lockr integration is active: whether the browser's + resource timing already shows a completed response from the Lockr + proxy's `page-view` path. TS counts first auctions with and without it. + It is counted once per page load and logs no identifier value. Capture + comes in step 2, but the race does not depend on it, so step 1 measures + it before step 2 relies on the cookie - metadata-preserving `KvPartnerId` and `ps`, with `Default` and constructors - `GenericPullProvider` replacing pull sync, which brings the trait, the @@ -3921,10 +4135,31 @@ Three steps, each shippable on its own: 2. **Providers.** `.with_identity_provider` and `.try_with_identity_provider` for vendors, ownership and its - validation, hold mode, passive capture, provider state, the rate-limiter - `record` and `count` methods and the circuit breaker, and the consent - decoder extensions of section 9.1. Vendor modules follow in their own - specs. + validation, hold mode, passive capture with `capture_enabled` off by + default and `revoke_scope` defaulting to `"provider"`, the real-browser + gate on capture effects, provider state, the rate-limiter `record` and + `count` methods and the circuit breaker, and the consent decoder + extensions of section 9.1. Vendor modules follow in their own specs. + Capture stays off for a provider until the checklist below has passed + once on a live service. + +**Capture provider checklist.** Before setting `capture_enabled = true`: + +- Each captured source has an `[[ec.partners]]` entry with + `bidstream_enabled = true` and the right `openrtb_atype`. Lockr's ID5 uses + `1` (#1246). +- A source moving to an owner is moved off pull sync first. A + `pull_sync_enabled` entry registers a generic pull provider that claims + the source, and two providers cannot claim an owned source (section 10.2). +- Live acceptance: after an eligible request, `/_ts/admin/eids` lists the + source as matched (`ingest.matched`, `ec/admin.rs:532-536`), and then + `/_ts/admin/ec` shows the stored UID (`ec/admin.rs:276`). + +**Rotating the cookie keys.** Add the new key to `keys` and deploy. Once +every POP serves that config, make it `current` and deploy again. Keep the +previous key for the retention section 6.2 gives: `staged_max_age_sec` for +`ts-eids-s`, one year for `ts-eids-r`. Removing it sooner can drop an +unapplied removal batch of a client that has not navigated since. After providers write metadata, do not roll back past step 1. Step 1's reader preserves the new fields. A rollback past it strips them: @@ -3955,35 +4190,47 @@ sending what it staged. on Fastly's per-key limit when another write to the same key landed within about a second, on a read failure, or on a KV error. Should a failed tombstone be retried after send, with a delay? +3. **How often the first auction misses Lockr's ID.** On the affected route, + Lockr's `page-view` call and the first `/auction` start close together. + How often does the first `/auction` leave before the proxied Lockr + response sets `ts-eids-s`? Then that auction lacks the captured ID, and a + first visit's first auction gets it only from the browser path. Step 1's + race metric (section 14) measures it before capture is turned on (#1260 + review). +4. **Global revoke and a granting CMP.** A global revoke clears the EC + cookie, as a withdrawal does, but the CMP's consent may still grant. The + next navigation then mints a new EC. Should a global revoke also stop + minting until consent changes? ## 16. Decisions log Decisions made while writing this spec, with the user: -| Decision | Choice | -| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Scope | Framework and trait. Vendors get their own specs. | -| HEM input | Typed slot, always empty in v1. Intake is a later spec. | -| Browser vs server precedence | Configured per source through `owner`. The owner's valid value wins. | -| Cookie for server IDs | Separate `ts-eids-s`, server-owned and HttpOnly, sealed with AES-256-GCM so the origin and its logs never see server-only IDs. | -| Structure | Trait plus coordinator, with an optional passive-capture capability. | -| Trait shape | One required method, `descriptor`. Capabilities are default methods. Synchronous. Additive by construction. | -| Inputs | Declared in the descriptor and enforced: a provider can read only what it declared. | -| Page load for the one-write rule | A navigation plus every request it causes. Only the navigation writes. Strict. | -| New-visitor write | One create before routing, as today, carrying browser EIDs. No provider runs on the first navigation. Changed in rounds 3 and 4. | -| New-visitor recovery | Today's rotation. The cookie-proof recreation of revision 3 is removed with the deferred create. | -| Failed write | Not retried. Changed from one merge-retry once Fastly's per-key limit was confirmed. | -| Browser EIDs beyond the cookie | `/auction` saves the KV gap in `ts-eids-s`. The next navigation writes it. | -| Delay on body-less routes | Accepted: a browser ID first seen in an `/auction` body reaches KV one page view later. | -| SPA views (`/_ts/page-bids`) | Stagers. They never write KV. | -| Hold budget | `hold_budget_ms` defaults to 100 ms, reduced from 300 ms (section 4.7 gives the reasons). | -| Outbound cookies | TS's identity cookies are stripped from Prebid Server requests. The origin keeps `ts-ec` but loses `ts-eids`, `ts-eids-s` and `ts-eids-r`, to stay under header limits. | -| Deletion | A tombstone with the 24-hour `TOMBSTONE_TTL` replaces the hard delete. No extra writes. | -| GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | -| `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | -| Consent withdrawal | Synchronous tombstone on any request, one of two G4 exceptions with the removal overflow write. Proposed in review; not yet confirmed. | -| Removal overflow | A synchronous, removal-only KV write before send, the second G4 exception. It retries only a CAS conflict and falls back to new cookies. Confirmed after round 10. | -| Equal-second ties | A removal or opt-out wins a tie with an acquisition or clear. An ID re-acquired in that second is resolved again later. Confirmed after round 10. | +| Decision | Choice | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Scope | Framework and trait. Vendors get their own specs. | +| HEM input | Typed slot, always empty in v1. Intake is a later spec. | +| Browser vs server precedence | Configured per source through `owner`. The owner's valid value wins. | +| Cookie for server IDs | Separate `ts-eids-s`, server-owned and HttpOnly, sealed with AES-256-GCM so the origin and its logs never see server-only IDs. | +| Structure | Trait plus coordinator, with an optional passive-capture capability. | +| Trait shape | One required method, `descriptor`. Capabilities are default methods. Synchronous. Additive by construction. | +| Inputs | Declared in the descriptor and enforced: a provider can read only what it declared. | +| Page load for the one-write rule | A navigation plus every request it causes. Only the navigation writes. Strict. | +| New-visitor write | One create before routing, as today, carrying browser EIDs. No provider runs on the first navigation. Changed in rounds 3 and 4. | +| New-visitor recovery | Today's rotation. The cookie-proof recreation of revision 3 is removed with the deferred create. | +| Failed write | Not retried. Changed from one merge-retry once Fastly's per-key limit was confirmed. | +| Browser EIDs beyond the cookie | `/auction` saves the KV gap in `ts-eids-s`. The next navigation writes it. | +| Delay on body-less routes | Accepted: a browser ID first seen in an `/auction` body reaches KV one page view later. | +| SPA views (`/_ts/page-bids`) | Stagers. They never write KV. | +| Hold budget | `hold_budget_ms` defaults to 100 ms, reduced from 300 ms (section 4.7 gives the reasons). | +| Outbound cookies | TS's identity cookies are stripped from Prebid Server requests. The origin keeps `ts-ec` but loses `ts-eids`, `ts-eids-s` and `ts-eids-r`, to stay under header limits. | +| Deletion | A tombstone with the 24-hour `TOMBSTONE_TTL` replaces the hard delete. No extra writes. | +| GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | +| `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | +| Consent withdrawal | Synchronous tombstone on any request, one of two G4 exceptions with the removal overflow write. Proposed in review; not yet confirmed. | +| Removal overflow | A synchronous, removal-only KV write before send, the second G4 exception. It retries only a CAS conflict and falls back to new cookies. Confirmed after round 10. | +| Equal-second ties | A removal or opt-out wins a tie with an acquisition or clear. An ID re-acquired in that second is resolved again later. Confirmed after round 10. | +| Server cookie lifetime | #1260's handoff cookie lived five minutes, only filled absent sources and was never authoritative. `ts-eids-s` lives a year, revalidates against KV every `kv_revalidate_sec` and can carry owner writes, so staged entries survive failed flushes and long SPA sessions. | Round 7 proposed these, and they are applied in revision 8 pending the user's review of the spec: @@ -4011,6 +4258,15 @@ user's review of the spec: | Browser candidates | The owner's `ps.iv` and `ps.o` filter the browser's value for owned sources | | `/identify` consent | The request's visitor consent. No persisted visitor authorization | +Round 11 proposed these, and they are applied in revision 12 pending the +user's review of the spec: + +| Decision | Choice | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Real-browser gate | A non-browser capture stores no positive ID, stages removals only in its own `ts-eids-r`, and never makes the overflow write | +| Global revoke | `revoke_scope`, default `"provider"`. `"global"` runs the withdrawal tombstone, from real browsers only, under the existing exception | +| Cookie keys | Dedicated rotating keys from the secret store, `v1...`. Writers re-seal unapplied removal batches | + ## 17. Review log **Round 1** (2026-10-21) found 5 blocking issues, 16 significant gaps and 17 @@ -4267,3 +4523,27 @@ the declared capabilities rather than every provider shape, HEM intake and multi-step exchanges stay out of v1 (section 2.2), and the vendor examples are illustrations, to be checked against protocol fixtures in their own specs. + +**Round 11** (2026-10-23) folded in findings from +[#1246](https://github.com/IABTechLab/trusted-server/issues/1246) (Lockr IDs to +KV) and the review of +[#1260](https://github.com/IABTechLab/trusted-server/pull/1260) (server-side +identity foundation and Lockr capture). The Lockr ID5 EID lives in Prebid's +`ortb2` config, and capture sees it only at first issue. The first auction +can race the Lockr response. An empty GPP string must not block. Capture +should stay off until live acceptance. A replayed `ts-ec` could change a +row through the proxy, a revoke's scope needs an operator choice, and the +cookie keys should rotate apart from `ec.passphrase`. Revision 12 resolves +them: + +| Finding | Source | Resolution | +| --------------------------------------------------------------------------- | ------------ | -------------------------------------------------------------------------------------- | +| Lockr's ID5 EID is only in `ortb2`, which `getUserIdsAsEids()` misses | #1246 | Dependency on the EID fidelity collector stated; capture-once note, 12.2; G10 test, 13 | +| The first `/auction` can leave before the Lockr response sets `ts-eids-s` | #1260 review | Open question 3, 15; race metric in rollout step 1, 14 | +| A header-only GPP string (`DBAA`) blocked capture | #1260 review | Stated in 9.1; consent test, 13 | +| Capture defaulted to on | #1260 review | `capture_enabled`, default `false`, 4.7, 8, 10.1, 10.2, 14 | +| Owner, partner entry and pull sync prerequisites were implicit | #1260 review | Capture provider checklist, 14 | +| The handoff cookie's lifetime differed from `ts-eids-s` | #1260 | Decisions log row, 16 | +| A replayed `ts-ec` could enrich, strip or tombstone a row through the proxy | #1260 review | Real-browser gate on capture effects, G12, 4.7, 5.6, 6.2, 8, 13 | +| Global revoke had no operator gate | #1260 | `revoke_scope`, default `"provider"`, 5.1, 8, 10.1, 10.2, 12.2, 13 | +| Cookie keys came from `ec.passphrase` and could not rotate | #1260 | Dedicated rotating keys, re-seal, retention, 6.2, 9.3, 10.1, 10.2, 13, 14 | From e49916fb63510f7aec06ef31116b73ef63744a1f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 14:37:29 +0530 Subject: [PATCH 06/89] Revise the identity providers spec to revision 15 and update the plan Revision 15 drops the sealed identity cookies, the keyring and the removal overflow write. Removals reach KV on the request that carries them, KV moves to schema version 2, ts-eids-s becomes a client-only mirror written by the shim, and capture is revoke-only. The implementation plan is rewritten to match: tasks for the cookie keyring and the race metric are retired, Task 27 now holds the withdrawal tombstone retry, and the remaining tasks follow the new KV-only eligibility, mirror payload and capture removal write. Signed-off-by: dhruv8sh --- ...26-10-22-server-side-identity-providers.md | 6622 ++++++++++++----- ...0-server-side-identity-providers-design.md | 3404 ++++----- 2 files changed, 6150 insertions(+), 3876 deletions(-) diff --git a/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md b/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md index 4ce4338879..9581b8980a 100644 --- a/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md +++ b/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md @@ -2,25 +2,28 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Let Trusted Server call identity providers (ID5, LiveRamp ATS, Lockr, UID2 and generic pull partners) from the edge through one `IdentityProvider` trait and one coordinator. Every page load makes at most one browser-driven KV write. Provider IDs are cached first in a sealed first-party cookie (`__Host-ts-eids-s`) and then in KV, and removals always win. +**Goal:** Let Trusted Server call identity providers (ID5, LiveRamp ATS, Lockr, UID2 and generic pull partners) from the edge through one `IdentityProvider` trait and one coordinator. Every page load makes at most one KV write, apart from the removal writes, and removals reach KV on the request that carries them. A provider can return complete EIDs, with several entries and UIDs per source and their EID-level fields, and TS stores and sends them whole. Every path that predates this work keeps its one-UID behaviour (G13). -**Architecture:** New code lives in `crates/trusted-server-core/src/ec/identity/`. Providers are pure. They describe themselves with a `ProviderDescriptor`, build `ProviderRequest`s and parse buffered `ProviderResponse`s into `Resolution`s. The `IdentityCoordinator` owns eligibility, consent, transport limits, backoff and turning results into changes. Each request is either the **writer** (a real-browser top-level document navigation) or a **stager**. The writer adds every KV change to one request-scoped `IdentityMutation` and applies it with one conditional write after the response is sent. Stagers never write KV. They save positive changes in `__Host-ts-eids-s` and removals in per-batch `__Host-ts-eids-r-` cookies, and the next writer flushes them. Today's pull sync becomes `GenericPullProvider`. +**Architecture:** New code lives in `crates/trusted-server-core/src/ec/identity/`. Providers are pure. They describe themselves with a `ProviderDescriptor`, build `ProviderRequest`s and parse buffered `ProviderResponse`s into `Resolution`s. The `IdentityCoordinator` owns eligibility, consent, transport limits, backoff and turning results into changes. Each request is either the **writer** (a real-browser top-level document navigation) or a **stager**. Resolving providers run only on the writer of a returning EC. The writer adds every KV change to one request-scoped `IdentityMutation` and applies it with one conditional write after the response is sent. Stagers persist no positive ID: they read KV and serve from it. The two removal writes are the exceptions: a failed flush retries its removals alone, and a capture's revocation is written on the proxied request through the shared CAS loop in `removal.rs`. The server sets no identity cookie of its own beyond `ts-ec`. `/auction` and `/_ts/page-bids` return a `ts-eids-s` mirror payload that the shim writes, and the server never reads that cookie. KV schema version 2 stores every record as an entry set (`eids`), and reads version 1 rows as one-entry records. Today's pull sync becomes `GenericPullProvider`. -**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `trusted-server-adapter-fastly`, `error-stack`, `derive_more`, `serde`/`serde_json`, `hmac`/`sha2`, `base64`, `rand`, `flate2`/`brotli` (already dependencies), and a new `aes-gcm = "0.10"`. Fastly Compute (Viceroy for tests). +**Tech Stack:** Rust 1.95 (edition 2024), `trusted-server-core`, `trusted-server-adapter-fastly`, `error-stack`, `derive_more`, `serde`/`serde_json`, `hmac`/`sha2`, `base64`, `rand`, `flate2`/`brotli` (all already dependencies). The shim is TypeScript in `crates/trusted-server-js/lib`. Fastly Compute (Viceroy for tests). -**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md` (revision 11, branch `spec/1246-server-side-identity-providers`). Read it before starting. "Spec 5.4" means section 5.4 of that file. Where this plan and the spec disagree, the spec wins. Stop and report the disagreement instead of choosing one. The spec cites `file:line` at commit `c84a73da7`. Line numbers have drifted slightly since then, so find code by the function names given here, not by line number. +**Spec:** `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md` (revision 15 with the round 15 amendments, branch `spec/1246-server-side-identity-providers`). Read it before starting. "Spec 5.4" means section 5.4 of that file. Where this plan and the spec disagree, the spec wins. Stop and report the disagreement instead of choosing one. The spec cites `file:line` at commit `c84a73da7`. Line numbers have drifted slightly since then, so find code by the function names given here, not by line number. ## How this plan is split -The spec's rollout (spec 14) has three steps, and each step can ship on its own. This plan has one **Part** per step, and each Part is one pull request. Each PR's branch is stacked on the previous one: +The spec's rollout (spec 14) has two steps, and each step can ship on its own. This plan has one **Part** per step, and each Part is one pull request. Each PR's branch is stacked on the previous one: -| Part | Spec step | Branch | Base | Tasks | -| ---- | ------------------------ | ---------------------------------- | ------------------------------------------ | -------- | -| 0 | 0. Strips first | `feat/1246-identity-cookie-strips` | `spec/1246-server-side-identity-providers` | 1 to 4 | -| 1 | 1. One write, no vendors | `feat/1246-identity-single-write` | `feat/1246-identity-cookie-strips` | 5 to 31 | -| 2 | 2. Providers | `feat/1246-identity-providers` | `feat/1246-identity-single-write` | 32 to 41 | +| Part | Spec step | Branch | Base | Tasks | +| ---- | ------------------------ | --------------------------------- | ------------------------------------------ | ------------- | +| 1 | 1. One write, no vendors | `feat/1246-identity-single-write` | `spec/1246-server-side-identity-providers` | 1 to 31 | +| 2 | 2. Providers | `feat/1246-identity-providers` | `feat/1246-identity-single-write` | 32 to 41, 39b | -Part 0 is the rollback floor (spec 14). Do not start Part 1 until Part 0's PR is open. Do not start Part 2 until Part 1's one-write invariant tests (Task 30) pass. +A task with a letter suffix, such as Task 39b, was added by a later spec revision. It runs right after the task with the same number. Task numbers 4b, 16 and 31b are retired: revision 15 removed the cookie keyring, the sealed cookies and the first-auction race metric. Task 27 is reused: it held the removal overflow write, which revision 15 removed, and now holds the withdrawal tombstone's after-send attempt. Part 1 starts with the outbound cookie strips (Tasks 1 to 4), which spec 14 folds into rollout step 1. + +Do not start Part 2 until Part 1's one-write invariant tests (Task 30) pass. + +**Rolling back past Part 1 is unsupported** (spec 14). Part 1 writes KV schema version 2, and today's reader rejects every version but 1, so a build without Part 1 cannot read a row Part 1 has written. Part 1's PR description and changelog say so. ## Global Constraints @@ -33,16 +36,27 @@ Copied from the spec and `AGENTS.md`. Every task implicitly includes these. - Never log UIDs, envelopes, HEMs, provider state, consent strings, cookie values, or provider URLs with a query string. EC IDs go through `log_id`. Log provider errors with `current_context()` only, never `{:?}` of the report (spec 9.3). - Use only example or fictional data in code, tests, comments and docs (`example.com` hosts). Public protocol source domains (`id5-sync.com`, `liveramp.com`, `uidapi.com`, `sharedid.org`) may appear where the spec uses them. - **Fixed values from the spec. Do not change them:** - - `__Host-ts-eids-s`: sealed value at most **2,048** bytes. `__Host-ts-eids-r-<8 hex>`: sealed value at most **512** bytes, normally at most **4** cookies. Both: `Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=31536000`, with no `Domain`. - - Seal: AES-256-GCM, value `v1..`, base64url without padding, 12-byte nonce, the full EC ID as associated data. Key labels: `trusted-server/ts-eids-s/key/v1` and `trusted-server/ts-eids-r/key/v1`, each derived with HMAC-SHA256 from `ec.passphrase`. - - KV: `ext` at most **256** serialized bytes, `ps.st` at most **1,024** bytes of embedded JSON, at most **32** `ps` entries, at most **4** `ps.iv` hashes, `rv` is **16** hex digits (64 random bits). `SCHEMA_VERSION` stays **1**. + - `ts-eids-s` (`__Host-ts-eids-s`) is client-only: the shim writes it with `Path=/; Secure; SameSite=Lax`, no `Domain` and no `HttpOnly`, with the `Max-Age` the payload gives. The server never sets, opens, parses or trusts it. The mirror payload is `ext.ts_eids_s` on `/auction` and a `ts_eids_s` member of the `/_ts/page-bids` body: `{value, max_age}`, `{clear: true}`, or absent. `value` is base64url without padding of compact JSON `{"v":1,"eids":[...]}`, at most **2,048** bytes. `max_age` is at most **86,400** and never past the earliest `exp` in the mirrored set (spec 6.2). + - `TS_IDENTITY_COOKIE_NAMES` = `ts-ec`, `ts-eids`, `__Host-ts-eids-s`, stripped from Prebid Server requests. `TS_ORIGIN_STRIPPED_COOKIE_NAMES` = `ts-eids`, `__Host-ts-eids-s`, stripped from origin requests. Exact names only: there are no cookie prefixes. + - Hashes: `ps.iv` and `ps.ov` hold the first **16** hex digits of the unkeyed SHA-256 of each UID. The consent fingerprint `fp` is a plain bitmask written as hex. The only keyed value is the rate-limit visitor key, an HMAC of the full EC ID with a key derived from `ec.passphrase` (spec 9.3). Nothing stored in the row is keyed. + - KV: `SCHEMA_VERSION` is **2**. Every record is `{w?, rv?, at?, exp?, ra?, ru?, eids: [{uids: [{id, atype?, ext?}], inserter?, matcher?, mm?, ext?}]}`, and `eids` is never empty. A version 1 record (`{uid, ...}`) reads as one entry holding that UID, and the row's next write stores version 2. Every UID-level and EID-level `ext` is a JSON object of at most **2,048** bytes as compact JSON. `inserter` and `matcher` are not blank and at most **255** bytes. A source holds at most **8** entries and **32** UIDs, and its entry set is at most **4,096** bytes as compact JSON (`MAX_ENTRY_SET_BYTES`). `ps.st` is at most **1,024** bytes of embedded JSON, a row has at most **32** `ps` entries, and `ps.iv` and `ps.ov` are each an ordered list of at most **128** hashes (`MAX_INVALIDATED_UIDS`), oldest first, truncated from the front before validation. One function, `entry_set_bytes`, measures `MAX_ENTRY_SET_BYTES` on the stored `eids` array, for both the coordinator and `KvEntry::validate`. Expired means `now >= exp`. `rv` is **16** hex digits (64 random bits). On read, a record that breaks a limit is dropped from the row and logged with its source only; only `serialize_entry` rejects. - `ProviderRequest` call context: at most **1,024** bytes of JSON. `IdentityProviderId`: `^[a-z][a-z0-9_-]{0,62}$`. - - `[ec.identity]` defaults: `max_concurrency = 4`, `hold_budget_ms = 100`, `hold_retry_sec = 300`, `max_response_bytes = 65536`, `pending_grace_sec = 300`, `staged_max_age_sec = 604800`, `kv_revalidate_sec = 3600`. Per-provider defaults: `mode = "after_send"`, `timeout_ms` = `hold_budget_ms` in hold mode and `1000` after send, `refresh_after_sec = 86400`, `max_calls_per_minute = 1`, `breaker_failures_per_minute = 50`, `cookie_priority = 0`. - - Backoff (spec 8): a `no_match` waits `n = now + retry_in`, default 24 h and at most 7 d. An input-caused failure waits `min(1 h × 2^(f-1), 24 h)`. A transport, 429 or 5xx failure waits `min(5 min × 2^(f-1), 1 h)`. A cut-off hold call waits `hold_retry_sec`. A skip waits `refresh_after_sec`. A marked provider that was not dispatched gets `ps.n = max(ps.n, now + 60 s)`. -- **G4 is strict.** A page load makes at most one browser-driven KV write and never retries it. The only exceptions are the consent-withdrawal tombstone path (unchanged) and the removal overflow write (Task 27). -- **Removals win.** No filter (consent re-gate, staleness, ownership, `cf` mismatch, size cap) may drop a removal, an opt-out or an `iv` hash. Only the EC binding and, when present, the revision check apply to them (spec 5.6). + - `[ec.identity]` defaults: `max_concurrency = 4`, `hold_budget_ms = 100`, `hold_retry_sec = 300`, `max_response_bytes = 65536`. Per-provider defaults: `mode = "after_send"`, `timeout_ms` = `hold_budget_ms`, `after_send_timeout_ms = 1000`, `refresh_after_sec = 86400`, `max_calls_per_minute = 1`, `breaker_failures_per_minute = 50`, `cookie_priority = 0`, `sources` = every source the descriptor declares (required for a capture-only provider). For a provider that declares `captures`: `revoke_scope = "provider"`. + - Removed config keys, each rejected with an error that names its replacement (spec 10.2): `ec.pull_sync_concurrency` (`[ec.identity].max_concurrency`), and `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]` (`refresh_after_sec` and `max_calls_per_minute = ceil(N / 60)` in `[ec.identity.providers.pull-]`). + - Backoff (spec 8): a `no_match` waits `n = now + retry_in`, default 24 h and at most 7 d. An input-caused failure waits `min(1 h × 2^(f-1), 24 h)`. A transport, 429 or 5xx failure waits `min(5 min × 2^(f-1), 1 h)`. A cut-off hold call waits `hold_retry_sec`. A skip waits `refresh_after_sec`. A due provider that was not dispatched gets `ps.n = max(ps.n, now + 60 s)`. +- **G4 is strict.** A page load makes at most one KV write, never retried as a whole. The only exceptions (spec 5.1): + - the consent-withdrawal tombstone path (a CAS with up to five attempts, then its completion marker, per withdrawn EC ID), which also runs for a `revoke_scope = "global"` capture from a real browser, and gets one more attempt after send when it fails before send (Task 27) + - the removal-only retry of a failed flush (spec 5.2, phase 4, step 5; Task 29) + - the capture removal write (spec 5.3; Task 38) + + The last two share one CAS loop, `ec/identity/removal.rs` (Task 18): up to five attempts, retrying only a generation mismatch, and `removal_lost` when every attempt fails. + +- **Removals win.** No filter (consent re-gate, staleness, ownership, the provider no longer being enabled, size cap) may drop a removal, an opt-out or a clear. The one gate a removal faces is the real-browser gate on capture (spec 4.7, 5.6). - **Keep `ts-eids` and `x-ts-eids`.** Do not remove either (product requirement). -- **Feature gate for test support.** The spec names a `test-support` feature. `trusted-server-core` already has a `test-utils` feature for the same purpose (`crates/trusted-server-core/Cargo.toml`). Reuse `test-utils` and gate on `#[cfg(any(test, feature = "test-utils"))]`. This is the only intended deviation from the spec. Mention it in the PR description. +- **Behaviour that predates the spec does not change (G13).** G13 itself changes none of the paths that predate the spec. Browser ingest, `ts-eids`, batch sync, generic pull, `/auction` body forwarding and the `/identify` response keep the behaviour the rest of the spec gives them, and every record they write is one entry with one UID: one UID per registered source, picked by today's rule (`collect_eid_updates`, `ec/prebid_eids.rs:146-151`: for a repeated source, the first valid UID of its last entry, with the sharedId cookie applied last), and nothing for an unregistered source. Only a provider result writes more, and no Part 1 code writes more. A version 1 row goes out to bidders exactly as it does on `main`. Golden tests, named `golden_*` so `cargo test-fastly -- golden` runs them all, pin these paths (Tasks 6, 7, 28 and 30). +- **Feature gate for test support.** Gate test-only code on `#[cfg(any(test, feature = "test-utils"))]`, the feature `trusted-server-core` already has (`crates/trusted-server-core/Cargo.toml`) and spec 13 names. `ec::identity::test_support` is a `pub mod` under that gate, and every item in it is `pub`, from the task that creates it, as existing `test-utils` items are (`IntegrationRegistry::from_request_filters`). A `pub(crate)` item behind the feature is dead code in the non-test library build that `cargo clippy-fastly` lints with `--all-features`, so `-D warnings` fails it. The Fastly adapter's dev-dependencies already enable the feature (`crates/trusted-server-adapter-fastly/Cargo.toml`). +- **Adapter-facing API.** The Fastly adapter is a separate crate. It reaches identity state only through `pub struct IdentityRuntime` (Task 23) and `pub` opaque wrappers such as `BrowserSeed` (Task 25). Core types stay `pub(crate)`, and no `pub` item exposes a `pub(crate)` type: `private_interfaces` is an error under `-D warnings`. +- **Lints that bite plan code.** Backtick `OpenRTB`, CamelCase vendor names such as `LiveRamp` and `SharedID`, and other code-like words in doc comments (`clippy::doc_markdown`). `clippy::panic` is denied workspace-wide, tests included. On `wasm32-wasip1` a panic aborts the whole test binary under Viceroy and `#[should_panic]` tests are ignored, so a check that a test must see fail returns a `Result` instead of panicking. - Test discipline: 1. **Red first.** Run each new test against unchanged code and watch it fail for the expected reason before you implement. 2. **Pair absence with presence.** Each "X is gone" assertion sits next to a "control Y is still there" assertion on the same value. @@ -52,109 +66,79 @@ Copied from the spec and `AGENTS.md`. Every task implicitly includes these. ## Test commands -| What | Command | -| --------------------------------------- | --------------------------------------------------------------------------------------------- | -| Core and Fastly adapter (Viceroy) | `cargo test-fastly` | -| One core test by name | `cargo test-fastly -- ` | -| Fastly adapter with reusable sandbox | `cargo test-fastly-reuse` | -| Other adapters (Part 0 strips, Task 31) | `cargo test-axum && cargo test-cloudflare && cargo test-spin` | -| Lint, Fastly target | `cargo clippy-fastly` | -| Docs | `cargo doc --no-deps --all-features -p trusted-server-core` | -| Parity crate | `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` | -| CLI (config validation, Task 33) | `./scripts/test-cli.sh` | -| Full gate before each PR | `/check-ci` | +| What | Command | +| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| Core and Fastly adapter (Viceroy) | `cargo test-fastly` | +| One core test by name | `cargo test-fastly -- ` | +| Fastly adapter with reusable sandbox | `cargo test-fastly-reuse` | +| Other adapters (strips, Task 31) | `cargo test-axum && cargo test-cloudflare && cargo test-spin` | +| Lint, Fastly target | `cargo clippy-fastly` | +| Docs | `cargo doc --no-deps --all-features -p trusted-server-core` | +| Parity crate | `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` | +| CLI (config validation, Tasks 19, 33) | `./scripts/test-cli.sh` | +| JS shim (Tasks 26, 30, 31) | `cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run && mise exec node@24.12.0 -- npm run lint` | +| Full gate before each PR | `/check-ci` | `cargo test-fastly -- ` runs only the tests whose path contains ``. Every "Run" step below gives the exact filter. ## File map -| File | Responsibility | Tasks | -| ------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -------------------------- | -| `Cargo.toml` (workspace), `crates/trusted-server-core/Cargo.toml` | `aes-gcm` dependency | 5 | -| `crates/trusted-server-core/src/constants.rs`, `src/cookies.rs` | Cookie names, identity cookie lists, name-and-prefix strip | 1 | -| `crates/trusted-server-core/src/auction/routing.rs`, `src/integrations/prebid.rs` | Strip identity cookies from Prebid Server requests | 2 | -| `crates/trusted-server-core/src/publisher.rs` | Origin strip; `is_prefetch_request` move; page-bids stops ingest; navigation merge; hold mode | 3, 23, 26, 28, 36, 37 | -| `crates/trusted-server-core/src/cookies/template_cache_policy.rs` | New cookies refused as cache keys | 4 | -| `crates/trusted-server-core/src/ec/identity/mod.rs`, `ids.rs`, `resolution.rs` (new) | Module, ID newtypes, error type, results | 5, 10 | -| `crates/trusted-server-core/src/ec/kv_types.rs` | `KvPartnerId` metadata, `KvProviderState`, `KvEntry.ps`, validation | 6 | -| `crates/trusted-server-core/src/ec/kv.rs` | Constructors; `PartnerIdUpdate.acquired_at`; `compute_cluster_size`; removal overflow write; counting store | 6, 7, 22, 24, 27 | -| `crates/trusted-server-core/src/ec/admin.rs` | Redact `ps.st`; servability; operator opt-out clear | 6, 14, 36 | -| `crates/trusted-server-core/src/ec/identity/descriptor.rs` (new) | Descriptor, builder, `Resolves`, `Captures`, `ConsentPolicy`, `EidProvenance`, `Input`, `Access`, `ResolveKind`, `UsOptOut` | 5, 8 | -| `crates/trusted-server-core/src/ec/identity/input.rs` (new) | `ResolveInput`, `CaptureInput`, `StoredId`, `Region`, `HashedEmail`, `ProviderConsent`, randomness | 9 | -| `crates/trusted-server-core/src/ec/identity/exchange.rs` (new) | `ProviderRequest`, `ProviderResponse` | 10 | -| `crates/trusted-server-core/src/ec/identity/provider.rs` (new) | `IdentityProvider`, `IdentityProviderConfig` | 11 | -| `crates/trusted-server-core/src/ec/identity/policy.rs` (new), `src/consent/*` | Policy evaluation, GPP opt-outs, `stricter_of`, fingerprint, redacting `Debug` | 12, 34 | -| `crates/trusted-server-core/src/ec/identity/revision.rs` (new) | `IdentityKeys`, `Revision`, keyed hashes | 13 | -| `crates/trusted-server-core/src/ec/identity/catalog.rs`, `servable.rs` (new) | Provider catalog, ownership, `cf`; the servability check | 14, 22, 32, 33 | -| `crates/trusted-server-core/src/ec/eids.rs`, `ec/identify.rs` | Servability for KV IDs; `/identify` stops writing | 14, 24 | -| `crates/trusted-server-core/src/ec/identity/mutation.rs` (new) | `IdentityMutation` and its apply algorithm | 15 | -| `crates/trusted-server-core/src/ec/identity/cookie.rs` (new) | Seal and open, `ts-eids-s`, size cap, KV merge, `ts-eids-r` | 16, 17, 18 | -| `crates/trusted-server-core/src/ec/identity/config.rs` (new), `src/settings.rs`, `src/config.rs` | `[ec.identity]`, `owner`, deploy and runtime validation | 19, 33 | -| `crates/trusted-server-core/src/ec/identity/coordinator.rs` (new) | Eligibility, outcomes, dispatch, breaker, hold, capture, re-grants | 20, 21, 35, 36, 37, 38 | -| `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (new), `ec/pull_sync.rs` | `GenericPullProvider`; pull sync dispatcher removed | 22, 29 | -| `crates/trusted-server-core/src/ec/identity/test_support.rs` (new) | Shared fixtures, `ProviderHarness`, fake HTTP client and limiter | 14, 15, 20, 21, 22, 35, 39 | -| `crates/trusted-server-core/src/ec/identity/request.rs` (new), `src/ec/mod.rs`, `src/http_util.rs` | Writer decision, phase-1 captures, request identity state, read triggers | 23, 29 | -| `crates/trusted-server-core/src/ec/finalize.rs` | Mutation instead of writes; stager cookies; `ts-eids-r` on every path; metrics | 25, 26, 27, 29, 31 | -| `crates/trusted-server-core/src/auction/endpoints.rs` | KV-gap staging; server set in the merge; owner denials; hold mode | 26, 28, 36, 37 | -| `crates/trusted-server-core/src/ec/identity/flush.rs` (new) | Phase 4 `run_after_send` | 29 | -| `crates/trusted-server-adapter-fastly/src/app.rs`, `src/main.rs`, `src/rate_limiter.rs`, `tests/identity_writes.rs` | Writer decision, status capture, phase 4, `record`/`count`, invariant tests | 23, 25, 29, 30, 35 | -| `crates/trusted-server-core/src/integrations/registry.rs` | Provider registration, capture-path check, capture plumbing, `is_real_browser` | 23, 32, 38 | -| `crates/trusted-server-core/src/ec/rate_limiter.rs` | `record` and `count` | 35 | -| `crates/trusted-server-core/src/ec/batch_sync.rs` | Owned sources fill only when absent | 40 | -| `crates/trusted-server-core/src/ec/identity/examples.rs` (new, tests only) | Spec 4.6 reference providers | 39 | -| `crates/trusted-server-core/src/migration_guards.rs` | Every new `ec/identity/*.rs` file | 5 onward | -| `docs/guide/*.md`, `CHANGELOG.md`, `trusted-server.example.toml` | Docs | 4, 19, 31, 41 | - -------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ---------------------- | -| `Cargo.toml` (workspace), `crates/trusted-server-core/Cargo.toml` | `aes-gcm` dependency | 5 | -| `crates/trusted-server-core/src/constants.rs` | New cookie-name constants | 1 | -| `crates/trusted-server-core/src/cookies.rs` | Identity-cookie lists and the name-and-prefix strip helper | 1 | -| `crates/trusted-server-core/src/integrations/prebid.rs` | Strip identity cookies from Prebid Server requests | 2 | -| `crates/trusted-server-core/src/publisher.rs` | Origin strip; writer-request hooks; page-bids stops ingest; navigation merge and hold mode | 3, 22, 25, 27, 36 | -| `crates/trusted-server-core/src/cookies/template_cache_policy.rs` | New cookies refused as cache keys | 4 | -| `crates/trusted-server-core/src/ec/identity/mod.rs` (new) | Module docs, re-exports | 5 | -| `crates/trusted-server-core/src/ec/identity/ids.rs` (new) | `IdentityProviderId`, `SourceDomain`, `UnixTime` | 5 | -| `crates/trusted-server-core/src/ec/kv_types.rs` | `KvPartnerId` metadata, `KvProviderState`, `KvEntry.ps`, validation | 6 | -| `crates/trusted-server-core/src/ec/kv.rs` | Literals to constructors; `PartnerIdUpdate.acquired_at`; mutation flush; overflow write | 6, 7, 21, 26 | -| `crates/trusted-server-core/src/ec/admin.rs` | Redact `ps.st`; servability in the admin view | 6, 13 | -| `crates/trusted-server-core/src/ec/identity/descriptor.rs` (new) | `ProviderDescriptor`, builder, `Resolves`, `Captures`, `ConsentPolicy`, `EidProvenance`, `Input`, `Access`, `UsOptOut` | 8 | -| `crates/trusted-server-core/src/ec/identity/input.rs` (new) | `ResolveInput`, `CaptureInput`, `StoredId`, `Region`, `HashedEmail`, `ProviderConsent`, `ResolveKind` | 9 | -| `crates/trusted-server-core/src/ec/identity/exchange.rs` (new) | `ProviderRequest`, `ProviderResponse` | 10 | -| `crates/trusted-server-core/src/ec/identity/resolution.rs` (new) | `Resolution`, `ResolvedId`, `When`, `StateChange`, `IdentityProviderError` | 10 | -| `crates/trusted-server-core/src/ec/identity/provider.rs` (new) | `IdentityProvider`, `IdentityProviderConfig` | 11 | -| `crates/trusted-server-core/src/ec/identity/revision.rs` (new) | `Revision`, `rv` generation, keyed hashes (`pv` fallback, `iv`, visitor key, fingerprint) | 12 | -| `crates/trusted-server-core/src/ec/identity/servable.rs` (new) | The shared servability check | 13 | -| `crates/trusted-server-core/src/ec/eids.rs`, `ec/identify.rs` | Use the servability check; `/identify` stops writing | 13, 23 | -| `crates/trusted-server-core/src/ec/identity/mutation.rs` (new) | `IdentityMutation`, `PartnerIdChange`, `ProviderResult`, apply algorithm | 14 | -| `crates/trusted-server-core/src/ec/identity/cookie.rs` (new) | Seal and open, `ts-eids-s` payload, size cap, `ts-eids-r` batches, merge with KV | 15, 16, 17 | -| `crates/trusted-server-core/src/ec/identity/config.rs` (new) | `[ec.identity]` settings and validation | 18, 32 | -| `crates/trusted-server-core/src/settings.rs` | `Ec.identity`; `EcPartner.owner`; deprecate `pull_sync_concurrency` | 18, 32 | -| `crates/trusted-server-core/src/ec/identity/coordinator.rs` (new) | Eligibility, merged view, dispatch, outcomes, markers, breaker | 19, 20, 34, 35 | -| `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (new) | `GenericPullProvider` | 21 | -| `crates/trusted-server-core/src/ec/pull_sync.rs` | Reduced to helpers `GenericPullProvider` reuses; dispatch removed | 21, 28 | -| `crates/trusted-server-core/src/ec/identity/test_support.rs` (new) | `ProviderHarness`, `WriteCountingEcKv`, fake HTTP client | 11, 21, 40 | -| `crates/trusted-server-core/src/ec/mod.rs` | `EcContext` writer flag, phase-1 captures, identity cookie state, mutation | 22 | -| `crates/trusted-server-core/src/http_util.rs` | `is_prefetch_request` moves here; `is_document_navigation` | 22 | -| `crates/trusted-server-core/src/ec/finalize.rs` | Mutation instead of ingest writes; stager cookie saves; `ts-eids-r` on every path | 24 | -| `crates/trusted-server-core/src/auction/endpoints.rs` | KV-gap staging; server set in `merge_auction_eids`; hold mode | 25, 27, 36 | -| `crates/trusted-server-adapter-fastly/src/app.rs`, `src/main.rs` | Writer decision, status capture, phase 4 entry point | 22, 28 | -| `crates/trusted-server-core/src/integrations/registry.rs` | Identity provider registration; capture plumbing; capture-path check | 31, 37 | -| `crates/trusted-server-core/src/config.rs` | `validate_identity_integration`, ownership validation | 32 | -| `crates/trusted-server-core/src/consent/{gpp.rs,tcf.rs,types.rs,mod.rs}` | Sharing and targeted-advertising opt-outs, section 6, GPC and known-child, `has_vendor_consent`, `stricter_of`, redacting `Debug` | 33 | -| `crates/trusted-server-core/src/ec/identity/policy.rs` (new) | `ConsentPolicy` evaluation and the consent fingerprint | 33 | -| `crates/trusted-server-core/src/ec/rate_limiter.rs`, `adapter-fastly/src/rate_limiter.rs` | `record` and `count` | 34 | -| `crates/trusted-server-core/src/ec/batch_sync.rs` | Owned sources fill only when absent | 39 | -| `docs/guide/edge-cookies.md`, `configuration.md`, `api-reference.md`, `ec-setup-guide.md`, `integrations/prebid.md` | Docs | 4, 30, 41 | +| File | Responsibility | Tasks | +| ------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| `crates/trusted-server-core/src/constants.rs`, `src/cookies.rs` | Cookie names, identity cookie lists, name strip | 1 | +| `crates/trusted-server-core/src/auction/routing.rs`, `src/integrations/prebid.rs` | Strip identity cookies from Prebid Server requests | 2 | +| `crates/trusted-server-core/src/publisher.rs` | Origin strip; servability callers; `is_prefetch_request` move; finalize callers; page-bids stops ingest and returns the mirror; navigation merge and denials; preload decision; owned precedence; hold mode | 3, 14, 23, 25, 26, 28, 29, 36, 37 | +| `crates/trusted-server-core/src/cookies/template_cache_policy.rs` | `ts-eids-s` refused as a cache key | 4 | +| `crates/trusted-server-core/src/settings.rs`, `src/config.rs` | `[ec.identity]`: `IdentitySettings`, `ProviderSettings`, `RevokeScope`; removed keys; deploy and runtime validation; `ADMIN_ENDPOINTS` | 19, 22, 33, 36 | +| `crates/trusted-server-core/src/ec/identity/mod.rs`, `ids.rs`, `resolution.rs` (new) | Module, ID newtypes, `UnixTime`, error type, results (`ResolvedEid`, `ResolvedUid`, `entry_sets`) | 5, 10 | +| `crates/trusted-server-core/src/ec/kv_types.rs` | Schema version 2: `KvPartnerId` with `eids`, `KvEid`, `KvUid`, `entry_set_bytes`, `KvProviderState` with `iv` and `ov`, `KvEntry.ps`, validation, upgrade on read | 6 | +| `crates/trusted-server-core/src/ec/kv.rs` | Constructors; upgrade on read; `PartnerIdUpdate.acquired_at`; `create_if_absent` seed; `compute_cluster_size`; withdrawal retry after send; counting store | 6, 7, 22, 24, 25, 27 | +| `crates/trusted-server-core/src/ec/prebid_eids.rs` | Browser ingest records `at` and stays one UID | 7 | +| `crates/trusted-server-core/src/ec/admin.rs` | Redact `ps.st`; upgrade on read; servability; operator opt-out clear | 6, 14, 36 | +| `crates/trusted-server-core/src/ec/identity/descriptor.rs` (new) | Descriptor, builder, `Resolves`, `Captures`, `ConsentPolicy`, `EidProvenance`, `Input`, `Access`, `ResolveKind`, `UsOptOut` | 5, 8 | +| `crates/trusted-server-core/src/ec/identity/input.rs` (new) | `ResolveInput`, `CaptureInput`, `StoredId`, `Region`, `HashedEmail`, `ProviderConsent`, randomness | 9, 10 | +| `crates/trusted-server-core/src/ec/identity/exchange.rs` (new) | `ProviderRequest`, `ProviderResponse` | 10 | +| `crates/trusted-server-core/src/ec/identity/provider.rs` (new) | `IdentityProvider`, `IdentityProviderConfig` | 11 | +| `crates/trusted-server-core/src/ec/identity/policy.rs` (new), `src/consent/*` | Policy evaluation, GPP opt-outs, the plain consent fingerprint, redacting `Debug` | 12, 20, 34 | +| `crates/trusted-server-core/src/ec/identity/revision.rs` (new) | `Revision`, `RecordRevision`, `StateRevision`, the unkeyed UID hash, the visitor key | 13 | +| `crates/trusted-server-core/src/ec/identity/catalog.rs`, `servable.rs` (new) | Provider catalog, narrowed sources, ownership, `validate_identity_catalog`; the servability check | 14, 22, 32, 33 | +| `crates/trusted-server-core/src/ec/eids.rs`, `ec/identify.rs`, `src/openrtb.rs` | Servability for KV IDs; one EID per stored entry; provenance defaults; `/identify` stops writing | 14, 24, 28 | +| `crates/trusted-server-core/src/ec/identity/mutation.rs` (new) | `IdentityMutation` and its apply algorithm, including the removals-only apply | 15 | +| `crates/trusted-server-core/src/ec/identity/mirror.rs` (new) | The `ts-eids-s` mirror payload and its size cap | 17 | +| `crates/trusted-server-core/src/ec/identity/removal.rs` (new) | The shared removal write: CAS loop, after-send attempt, `removal_lost` | 18, 29, 38 | +| `crates/trusted-server-core/src/ec/identity/coordinator.rs` (new) | Eligibility, outcomes, dispatch, breaker, hold, capture, re-grants | 20, 21, 35, 36, 37, 38 | +| `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (new), `ec/pull_sync.rs`, `ec/pull_sync_marker.rs` | `GenericPullProvider`; dispatch order; pull sync dispatcher and the pull-complete marker removed | 21, 22, 29 | +| `crates/trusted-server-core/src/ec/identity/test_support.rs` (new, `pub` under `test-utils`) | Shared fixtures, counting store, fake HTTP client and limiter, `ProviderHarness`, `MultiEntryTestProvider` | 14, 15, 18, 20, 21, 22, 25, 35, 39, 39b | +| `crates/trusted-server-core/src/ec/identity/runtime.rs` (new) | `pub struct IdentityRuntime`: catalog and visitor key for the adapter | 23, 29 | +| `crates/trusted-server-core/src/ec/identity/request.rs` (new), `src/ec/mod.rs`, `src/http_util.rs` | Writer decision, phase-1 captures, request identity state, removal counters | 5, 23, 25, 29, 38 | +| `crates/trusted-server-core/src/ec/finalize.rs` | Mutation instead of writes; `browser_seed`; global revoke; withdrawal retry; metrics | 7, 25, 27, 29, 31, 38 | +| `crates/trusted-server-core/src/auction/endpoints.rs`, `src/openrtb.rs` | Servability callers; finalize test callers; mirror payload in `ext`; server set and denials in the merge; owned precedence; hold mode | 14, 25, 26, 28, 36, 37 | +| `crates/trusted-server-core/src/ec/identity/flush.rs` (new) | Phase 4 `run_after_send` | 29 | +| `crates/trusted-server-adapter-fastly/src/app.rs`, `src/main.rs`, `src/rate_limiter.rs` | `IdentityRuntime` in `AppState`; writer decision; status capture; testable post-route function and phase 4; `record`/`count`; request-level tests (`drive_request_for_test`); admin route | 23, 25, 27, 29, 30, 35, 36, 39b | +| `crates/trusted-server-adapter-axum/src/app.rs`, `crates/trusted-server-adapter-cloudflare/src/app.rs`, `crates/trusted-server-adapter-spin/src/app.rs` | Admin opt-out route parity | 36 | +| `crates/trusted-server-core/src/integrations/registry.rs` | `BrowserSeed` and `is_real_browser` on `ProxyDispatchInput`; provider registration; capture-path check; capture plumbing | 23, 25, 32, 38 | +| `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` | `is_prefetch_request` caller | 23 | +| `crates/trusted-server-core/src/integrations/lockr.rs` | Upstream log without the query string | 38 | +| `crates/trusted-server-js/lib/src/core/*.ts`, `src/integrations/prebid/*`, `test/**` | The mirror writer; registered-first `ts-eids` trim; `ortb2` G10 test | 26, 30, 31 | +| `crates/trusted-server-core/src/ec/rate_limiter.rs` | `record` and `count` | 35 | +| `crates/trusted-server-core/src/ec/batch_sync.rs` (and its adapter call site) | Owned sources fill only when absent | 40 | +| `crates/trusted-server-core/src/ec/identity/examples.rs` (new, tests only) | Spec 4.6 reference providers | 32, 33, 39, 39b | +| `crates/trusted-server-core/src/migration_guards.rs` | Every new `ec/identity/*.rs` file | 5 onward | +| `docs/guide/*.md`, `CHANGELOG.md`, `trusted-server.example.toml` | Docs, removed keys and the rollback note | 4, 19, 31, 41 | + +This layout is the one spec 4.1's file table lists, with one difference: `[ec.identity]` settings live in `settings.rs` with the rest of the settings, and their validation in `config.rs`, rather than in a new `ec/identity/config.rs`. `UnixTime` lives in `ids.rs`. --- -# Part 0: Strip the identity cookies first (spec 14, step 0) +# Part 1: One write, no new vendors (spec 14, step 1) -This part only removes cookies from outbound requests. It sets no new cookies and changes no KV behaviour. It ships one release before Part 1, so a rollback after Part 1 still keeps the new one-year cookies away from the origin and from Prebid Server. +Part 1 starts by removing TS's identity cookies from outbound requests (Tasks 1 to 4), then builds the provider framework without registering any vendor, moves KV to schema version 2, and makes a page load write KV once. Generic pull partners are its only providers. It reads, preserves, validates and serves complete entry sets, but nothing in it writes a set with more than one entry or UID, so a rollback from Part 2 to Part 1 keeps every set a provider wrote. No vendor provider is registered yet, and no provider runs in hold mode or captures. Those come in Part 2. -Before you start: create a worktree from the spec branch, `git worktree add -b feat/1246-identity-cookie-strips .claude/worktrees/1246-identity-providers spec/1246-server-side-identity-providers`. The implementation stacks on its spec branch, which already carries the Prebid EID-fidelity work this spec depends on. Build with the toolchain that branch pins in `rust-toolchain.toml`. +Tasks 1 to 4 strip the identity cookies from outbound requests. Tasks 5 to 22 build unit-tested modules. Most have no request wiring: Task 6 moves KV to schema version 2, Task 7 changes what finalize's browser ingest stores, and Task 14 moves the `/auction`, admin and `/identify` readers onto the servability check. Tasks 23 to 29 wire the rest into requests. Task 26 adds the `ts-eids-s` mirror payload and the shim code that writes it. Task 30 proves the one-write invariant, and Task 31 ships the metrics and the docs. -### Task 1: Identity cookie names and a name-and-prefix strip helper +Before you start: create a worktree from the spec branch, `git worktree add -b feat/1246-identity-single-write .claude/worktrees/1246-identity-providers spec/1246-server-side-identity-providers`. The implementation stacks on its spec branch, which already carries the Prebid EID-fidelity work this spec depends on. Build with the toolchain that branch pins in `rust-toolchain.toml`. + +### Task 1: Identity cookie names and a cookie strip helper **Files:** @@ -166,12 +150,12 @@ Before you start: create a worktree from the spec branch, `git worktree add -b f - Produces: - `pub const COOKIE_TS_EIDS_S: &str = "__Host-ts-eids-s";` - - `pub const COOKIE_TS_EIDS_R_PREFIX: &str = "__Host-ts-eids-r-";` - - `pub const TS_IDENTITY_COOKIE_NAMES: &[&str]` = `ts-ec`, `ts-eids`, `__Host-ts-eids-s`, `ts-ec-pull-complete` + - `pub const TS_IDENTITY_COOKIE_NAMES: &[&str]` = `ts-ec`, `ts-eids`, `__Host-ts-eids-s` (spec 6.2) - `pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str]` = `ts-eids`, `__Host-ts-eids-s` - - `pub const TS_IDENTITY_COOKIE_PREFIXES: &[&str]` = `__Host-ts-eids-r-` - - `pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str], prefixes: &[&str]) -> String` - - `pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str], prefixes: &[&str])` + - `pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str]) -> String` + - `pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str])` + +The lists match exact names only: no TS identity cookie is a family with a name prefix (spec 6.2). `ts-ec-pull-complete` is not on either list. Task 29 stops setting it, and a browser's copy expires within its one-hour `Max-Age`. - [ ] **Step 1: Write the failing tests** @@ -179,19 +163,14 @@ Append to `mod tests` in `cookies.rs`: ```rust #[test] - fn strip_cookie_pairs_removes_names_and_prefixes() { - let header = "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=v1.x.y; \ - __Host-ts-eids-r-0a1b2c3d=v1.p.q; other=1; ts-eids-extra=keep"; + fn strip_cookie_pairs_removes_exact_names_only() { + let header = "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ; other=1; ts-eids-extra=keep"; - let stripped = strip_cookie_pairs( - header, - TS_ORIGIN_STRIPPED_COOKIE_NAMES, - TS_IDENTITY_COOKIE_PREFIXES, - ); + let stripped = strip_cookie_pairs(header, TS_ORIGIN_STRIPPED_COOKIE_NAMES); assert_eq!( stripped, "ts-ec=abc; other=1; ts-eids-extra=keep", - "should drop the ID cookies and keep ts-ec and unrelated cookies" + "should drop the ID cookies and keep ts-ec and a cookie that only shares a prefix" ); } @@ -205,7 +184,7 @@ Append to `mod tests` in `cookies.rs`: ); headers.append(header::COOKIE, http::HeaderValue::from_static("other=1")); - strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES); + strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES); let values: Vec<_> = headers.get_all(header::COOKIE).iter().collect(); assert_eq!(values.len(), 1, "should collapse into one Cookie header"); @@ -217,11 +196,11 @@ Append to `mod tests` in `cookies.rs`: let mut headers = http::HeaderMap::new(); headers.insert( header::COOKIE, - http::HeaderValue::from_static("ts-ec=abc; __Host-ts-eids-r-00000000=v1.a.b"), + http::HeaderValue::from_static("ts-ec=abc; __Host-ts-eids-s=eyJ2IjoxfQ"), ); headers.insert(header::USER_AGENT, http::HeaderValue::from_static("ExampleBrowser/1.0")); - strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES); + strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES); assert!(headers.get(header::COOKIE).is_none(), "should remove the empty Cookie header"); assert!(headers.get(header::USER_AGENT).is_some(), "should leave other headers alone"); @@ -238,55 +217,44 @@ Expected: compile errors, because `strip_cookie_pairs`, `strip_cookie_headers` a In `constants.rs`, under `COOKIE_TS_EIDS`: ```rust -/// Server-owned, sealed cache of provider-collected EIDs (spec 6.2). +/// Client-only mirror of the servable server-side EIDs, written by the shim +/// and never read by the server (spec 6.2). pub const COOKIE_TS_EIDS_S: &str = "__Host-ts-eids-s"; -/// Name prefix of the per-batch removal cookies, `__Host-ts-eids-r-<8 hex>`. -pub const COOKIE_TS_EIDS_R_PREFIX: &str = "__Host-ts-eids-r-"; ``` -In `cookies.rs`, extend the `crate::constants` import with `COOKIE_SHAREDID` (if you use it later), `COOKIE_TS_EC`, `COOKIE_TS_EC_PULL_COMPLETE`, `COOKIE_TS_EIDS`, `COOKIE_TS_EIDS_R_PREFIX` and `COOKIE_TS_EIDS_S`. Then add, after `CONSENT_COOKIE_NAMES`: +In `cookies.rs`, extend the `crate::constants` import with `COOKIE_TS_EC`, `COOKIE_TS_EIDS` and `COOKIE_TS_EIDS_S`. Then add, after `CONSENT_COOKIE_NAMES`: ```rust /// Trusted Server identity cookies that never reach a third party. /// /// Stripped from Prebid Server requests and never added to proxy or provider -/// requests. The `__Host-ts-eids-r-` family is matched by -/// [`TS_IDENTITY_COOKIE_PREFIXES`]. -pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[ - COOKIE_TS_EC, - COOKIE_TS_EIDS, - COOKIE_TS_EIDS_S, - COOKIE_TS_EC_PULL_COMPLETE, -]; +/// requests. +pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; /// Identity cookies removed from publisher-origin requests. /// -/// `ts-ec` and `ts-ec-pull-complete` still reach the origin. The ID cookies +/// `ts-ec` still reaches the origin. The ID cookies /// are removed so that they cannot push the origin's `Cookie` header past /// common 8 KB header limits. pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; - -/// Name prefixes of identity cookie families, stripped wherever the named -/// identity cookies are. -pub const TS_IDENTITY_COOKIE_PREFIXES: &[&str] = &[COOKIE_TS_EIDS_R_PREFIX]; ``` After `strip_cookies`: ```rust -/// Strips cookies matching any exact name or any name prefix from a `Cookie` -/// header value. +/// Strips cookies whose name is exactly one of `names` from a `Cookie` header +/// value. /// /// Returns an empty string when nothing is left. #[must_use] -pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str], prefixes: &[&str]) -> String { +pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str]) -> String { cookie_header .split(';') .map(str::trim) .filter(|pair| !pair.is_empty()) .filter(|pair| { let name = pair.split('=').next().unwrap_or_default().trim(); - !names.contains(&name) && !prefixes.iter().any(|prefix| name.starts_with(prefix)) + !names.contains(&name) }) .collect::>() .join("; ") @@ -298,12 +266,12 @@ pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str], prefixes: &[&str] /// written back as one field. A field that is not valid UTF-8 is dropped /// rather than forwarded, because it cannot be checked. The header is removed /// when nothing is left. -pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str], prefixes: &[&str]) { +pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str]) { let retained = headers .get_all(header::COOKIE) .iter() .filter_map(|value| value.to_str().ok()) - .map(|value| strip_cookie_pairs(value, names, prefixes)) + .map(|value| strip_cookie_pairs(value, names)) .filter(|value| !value.is_empty()) .collect::>() .join("; "); @@ -332,7 +300,7 @@ Expected: PASS, including the existing `strip_cookies` tests. ```bash git add crates/trusted-server-core/src/constants.rs crates/trusted-server-core/src/cookies.rs -git commit --signoff -S -m "Add identity cookie lists and a name-and-prefix cookie strip" +git commit --signoff -S -m "Add identity cookie lists and a cookie strip by name" ``` ### Task 2: Strip identity cookies from Prebid Server requests in every mode @@ -345,7 +313,7 @@ git commit --signoff -S -m "Add identity cookie lists and a name-and-prefix cook **Interfaces:** -- Consumes: `strip_cookie_pairs`, `TS_IDENTITY_COOKIE_NAMES`, `TS_IDENTITY_COOKIE_PREFIXES`, `CONSENT_COOKIE_NAMES` (Task 1) +- Consumes: `strip_cookie_pairs`, `TS_IDENTITY_COOKIE_NAMES`, `CONSENT_COOKIE_NAMES` (Task 1) - Produces: `PrebidTransportHeaders::cookies(&self) -> &[HeaderValue]`, which replaces `cookie()`. `apply_prebid_header_values` takes `cookies: &[HeaderValue]` as its first argument. Today `apply_prebid_header_values` forwards the raw `Cookie` header unchanged whenever `consent_forwarding.strips_consent_cookies()` is false, and both callers pass only the first `Cookie` field. After this task, identity cookies are stripped in every mode, every field is read, and a non-UTF-8 field is dropped (spec 6.2, "Prebid Server"). @@ -368,8 +336,8 @@ Find the existing tests of `apply_prebid_header_values` with `grep -n "apply_pre fn prebid_headers_strip_identity_cookies_when_consent_cookies_are_forwarded() { let cookies = [ HeaderValue::from_static("ts-ec=abc; euconsent-v2=tcf; other=1"), - HeaderValue::from_static("__Host-ts-eids-s=v1.a.b; __Host-ts-eids-r-0a1b2c3d=v1.c.d"), - HeaderValue::from_static("ts-eids=W10; ts-ec-pull-complete=v1; second=2"), + HeaderValue::from_static("__Host-ts-eids-s=eyJ2IjoxfQ; partner=p"), + HeaderValue::from_static("ts-eids=W10; second=2"), ]; let mut to = http::Request::new(EdgeBody::empty()); @@ -379,13 +347,13 @@ Find the existing tests of `apply_prebid_header_values` with `grep -n "apply_pre None, None, &mut to, - ConsentForwardingMode::CookiesAndBody, + ConsentForwardingMode::Both, None, ); assert_eq!( cookie_values(&to), - vec!["euconsent-v2=tcf; other=1; second=2".to_owned()], + vec!["euconsent-v2=tcf; other=1; partner=p; second=2".to_owned()], "should drop identity cookies, keep consent and unrelated cookies, and read every field" ); } @@ -422,7 +390,7 @@ Find the existing tests of `apply_prebid_header_values` with `grep -n "apply_pre None, None, &mut to, - ConsentForwardingMode::CookiesAndBody, + ConsentForwardingMode::Both, None, ); @@ -465,7 +433,7 @@ and in `snapshot`: Fix every other use of `cookie:` or `.cookie()` in `routing.rs` that the compiler reports, including the tests at about lines 1300-1315 that append two `Cookie` fields. Those tests should now see both fields. -In `integrations/prebid.rs`, change the import to `use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES, strip_cookie_pairs};`. Then replace the cookie half of `apply_prebid_header_values`: +In `integrations/prebid.rs`, change the import to `use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES, strip_cookie_pairs};`. Then replace the cookie half of `apply_prebid_header_values`: ```rust fn apply_prebid_header_values( @@ -489,8 +457,8 @@ fn apply_prebid_header_values( let retained = cookies .iter() .filter_map(|value| value.to_str().ok()) - .map(|value| strip_cookie_pairs(value, TS_IDENTITY_COOKIE_NAMES, TS_IDENTITY_COOKIE_PREFIXES)) - .map(|value| strip_cookie_pairs(&value, consent_names, &[])) + .map(|value| strip_cookie_pairs(value, TS_IDENTITY_COOKIE_NAMES)) + .map(|value| strip_cookie_pairs(&value, consent_names)) .filter(|value| !value.is_empty()) .collect::>() .join("; "); @@ -525,7 +493,7 @@ git commit --signoff -S -m "Strip identity cookies from Prebid Server requests i **Interfaces:** -- Consumes: `strip_cookie_headers`, `TS_ORIGIN_STRIPPED_COOKIE_NAMES`, `TS_IDENTITY_COOKIE_PREFIXES` (Task 1) +- Consumes: `strip_cookie_headers`, `TS_ORIGIN_STRIPPED_COOKIE_NAMES` (Task 1) - [ ] **Step 1: Write the failing test** @@ -539,7 +507,7 @@ Add to `publisher.rs` tests, next to any existing `rewrite_origin_request` test .header(header::COOKIE, "ts-ec=abc; ts-eids=W10; session=1") .header( header::COOKIE, - "__Host-ts-eids-s=v1.a.b; __Host-ts-eids-r-0a1b2c3d=v1.c.d; ts-ec-pull-complete=v1", + "__Host-ts-eids-s=eyJ2IjoxfQ; ts-ec-pull-complete=v1", ) .body(EdgeBody::empty()) .expect("should build request"); @@ -563,7 +531,7 @@ Add to `publisher.rs` tests, next to any existing `rewrite_origin_request` test fn rewrite_origin_request_drops_cookie_header_when_only_id_cookies() { let mut req = Request::builder() .uri("https://www.example.com/") - .header(header::COOKIE, "ts-eids=W10; __Host-ts-eids-s=v1.a.b") + .header(header::COOKIE, "ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ") .body(EdgeBody::empty()) .expect("should build request"); @@ -591,14 +559,10 @@ In `rewrite_origin_request`, after `req.headers_mut().remove("fastly-ssl");`: ```rust // The publisher origin keeps `ts-ec` but never receives the ID cookies, // which can push its `Cookie` header past common 8 KB limits (spec 6.2). - strip_cookie_headers( - req.headers_mut(), - TS_ORIGIN_STRIPPED_COOKIE_NAMES, - TS_IDENTITY_COOKIE_PREFIXES, - ); + strip_cookie_headers(req.headers_mut(), TS_ORIGIN_STRIPPED_COOKIE_NAMES); ``` -Add `use crate::cookies::{TS_IDENTITY_COOKIE_PREFIXES, TS_ORIGIN_STRIPPED_COOKIE_NAMES, strip_cookie_headers};` to the imports at the top of `publisher.rs`. +Add `use crate::cookies::{TS_ORIGIN_STRIPPED_COOKIE_NAMES, strip_cookie_headers};` to the imports at the top of `publisher.rs`. - [ ] **Step 4: Run and see it pass, then run the other adapters** @@ -609,10 +573,10 @@ Expected: PASS. The strip lives in core, so every adapter gets it (spec 11). If ```bash git add crates/trusted-server-core/src/publisher.rs -git commit --signoff -S -m "Strip ID cookies from publisher origin requests" -m "The origin keeps ts-ec and the pull-complete marker. ts-eids, __Host-ts-eids-s and the __Host-ts-eids-r- cookies no longer reach it. An origin that read ts-eids must read x-ts-eids instead." +git commit --signoff -S -m "Strip ID cookies from publisher origin requests" -m "The origin keeps ts-ec and every unrelated cookie. ts-eids and __Host-ts-eids-s no longer reach it. An origin that read ts-eids must read x-ts-eids instead." ``` -### Task 4: Refuse the new cookies as template-cache keys, and document Part 0 +### Task 4: Refuse the mirror cookie as a template-cache key, and document the strips **Files:** @@ -626,15 +590,13 @@ Find the test that rejects `ts-eids` as a key (`grep -n "identity cookie" crates ```rust #[test] - fn template_cache_key_rejects_sealed_identity_cookies() { - for name in ["__Host-ts-eids-s", "__Host-ts-eids-r-0a1b2c3d"] { - let error = validate_names_for_test(&[name.to_owned()], &[]) - .expect_err("should reject identity cookie as a cache key"); - assert!( - error.contains("must not key on Trusted Server identity cookie"), - "should explain the rejection for {name}, got {error}" - ); - } + fn template_cache_key_rejects_the_mirror_cookie() { + let error = validate_names_for_test(&["__Host-ts-eids-s".to_owned()], &[]) + .expect_err("should reject the mirror cookie as a cache key"); + assert!( + error.contains("must not key on Trusted Server identity cookie"), + "should explain the rejection, got {error}" + ); validate_names_for_test(&["session".to_owned()], &[]) .expect("should still accept an unrelated cookie"); } @@ -644,8 +606,8 @@ Find the test that rejects `ts-eids` as a key (`grep -n "identity cookie" crates - [ ] **Step 2: Run and see it fail** -Run: `cargo test-fastly -- template_cache_policy::tests::template_cache_key_rejects_sealed` -Expected: FAIL, because both names are accepted today. +Run: `cargo test-fastly -- template_cache_policy::tests::template_cache_key_rejects_the_mirror` +Expected: FAIL, because the name is accepted today. - [ ] **Step 3: Implement** @@ -653,13 +615,11 @@ Replace the condition at line 35: ```rust if field == "template_cache_key_cookies" - && ([COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S, COOKIE_SHAREDID] - .contains(&name.as_str()) - || name.starts_with(COOKIE_TS_EIDS_R_PREFIX)) + && [COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S, COOKIE_SHAREDID].contains(&name.as_str()) { ``` -and extend the `crate::constants` import with `COOKIE_TS_EIDS_R_PREFIX` and `COOKIE_TS_EIDS_S`. +and extend the `crate::constants` import with `COOKIE_TS_EIDS_S`. - [ ] **Step 4: Run and see it pass** @@ -668,12 +628,12 @@ Expected: PASS. - [ ] **Step 5: Docs** -In `docs/guide/edge-cookies.md`, in the section that lists TS cookies, add a short "Cookies forwarded to the origin" paragraph. Say that `ts-ec` and `ts-ec-pull-complete` still reach the publisher origin, that `ts-eids` and the server-owned `__Host-ts-eids-s` and `__Host-ts-eids-r-*` cookies do not, and that origin code that read `ts-eids` should read the `x-ts-eids` response header instead. Also say that Prebid Server never receives any TS identity cookie. +In `docs/guide/edge-cookies.md`, in the section that lists TS cookies, add a short "Cookies forwarded to the origin" paragraph. Say that `ts-ec` still reaches the publisher origin, that `ts-eids` and `__Host-ts-eids-s` do not, and that origin code that read `ts-eids` should read the `x-ts-eids` response header instead. Also say that Prebid Server never receives any TS identity cookie. In `CHANGELOG.md` under `## [Unreleased]` / `### Changed`, add: ```markdown -- **Breaking:** The publisher origin no longer receives the `ts-eids` cookie, and Prebid Server no longer receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, `ts-ec-pull-complete`, and the server-owned `__Host-ts-eids-s` and `__Host-ts-eids-r-*` cookies that a later release introduces). The origin still receives `ts-ec`. Origin code that read `ts-eids` should read the `x-ts-eids` response header instead. This release is the rollback floor for the server-side identity provider work. +- **Breaking:** The publisher origin no longer receives the `ts-eids` cookie, and Prebid Server no longer receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, and the `__Host-ts-eids-s` mirror that this release's shim writes). The origin still receives `ts-ec`. Origin code that read `ts-eids` should read the `x-ts-eids` response header instead. ``` Run: `cd docs && npm run format && cd ..` and `docs/node_modules/.bin/prettier --config docs/.prettierrc --write CHANGELOG.md` @@ -682,28 +642,13 @@ Run: `cd docs && npm run format && cd ..` and `docs/node_modules/.bin/prettier - ```bash git add crates/trusted-server-core/src/cookies/template_cache_policy.rs docs/guide/edge-cookies.md CHANGELOG.md -git commit --signoff -S -m "Refuse the sealed identity cookies as template cache keys" +git commit --signoff -S -m "Refuse the identity mirror cookie as a template cache key" ``` -- [ ] **Step 7: Part 0 gate and PR** - -Run `/check-ci`. Every gate must pass. Open the PR against `spec/1246-server-side-identity-providers` with the `pr-creator` agent. Title: "Strip identity cookies from origin and Prebid Server requests". The description must say this is spec step 0 and the rollback floor for the later parts, and must name the origin `ts-eids` behaviour change. - ---- - -# Part 1: One write per page load, no new vendors (spec 14, step 1) - -Part 1 builds the trait, descriptor, harness and coordinator, moves every browser-driven KV write into the writer request's single flush, adds `__Host-ts-eids-s` and `__Host-ts-eids-r-*`, and replaces pull sync with `GenericPullProvider`. No vendor provider is registered yet, and no provider runs in hold mode or captures. Those come in Part 2. - -Before you start: `git switch feat/1246-identity-cookie-strips && git switch -c feat/1246-identity-single-write`. - -Tasks 5 to 22 build pure, unit-tested modules with no request wiring. Tasks 23 to 29 wire them into requests. Task 30 proves the one-write invariant, and Task 31 ships. - ### Task 5: The `ec::identity` module, its ID newtypes and its error type **Files:** -- Modify: `Cargo.toml` (workspace `[workspace.dependencies]`, alphabetical), `crates/trusted-server-core/Cargo.toml` - Create: `crates/trusted-server-core/src/ec/identity/mod.rs` - Create: `crates/trusted-server-core/src/ec/identity/ids.rs` - Create: `crates/trusted-server-core/src/ec/identity/resolution.rs` (error type only in this task, extended in Task 10) @@ -718,24 +663,7 @@ Tasks 5 to 22 build pure, unit-tested modules with no request wiring. Tasks 23 t - `UnixTime`: `from_secs(u64)`, `from_millis(u64)`, `as_secs(self) -> u64`, `Add`. Derives `Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord`. - `IdentityProviderError` (all seven variants from spec 4.5) with `config`, `rejected`, `malformed` and `not_implemented` constructors. `Access` and `Input` are referenced by two variants, so this task adds a minimal `descriptor.rs` with just those two enums. Task 8 fills in the rest of the file. -- [ ] **Step 1: Add the dependency** - -In the workspace `Cargo.toml`, under `[workspace.dependencies]` in alphabetical order: - -```toml -aes-gcm = "0.10" -``` - -In `crates/trusted-server-core/Cargo.toml` `[dependencies]`, alphabetical: - -```toml -aes-gcm = { workspace = true } -``` - -Run: `cargo check-fastly && cargo check-cloudflare` -Expected: both build. `aes-gcm` 0.10 depends on `getrandom` only through its optional `getrandom` feature, which stays off. Nonces come from `rand::thread_rng` (Task 16). If `cargo check-cloudflare` fails on `getrandom`, set `default-features = false, features = ["aes", "alloc"]` in the workspace entry and run it again. - -- [ ] **Step 2: Write the failing tests** +- [ ] **Step 1: Write the failing tests** Create `ec/identity/ids.rs` with only this test module for now: @@ -747,8 +675,7 @@ mod tests { #[test] fn provider_id_accepts_the_documented_grammar() { for id in ["id5", "pull-ssp-example-com", "lockr_aim", "a"] { - id.parse::() - .unwrap_or_else(|_| panic!("should accept `{id}`")); + assert!(id.parse::().is_ok(), "should accept `{id}`"); } let longest = format!("a{}", "b".repeat(62)); longest.parse::().expect("should accept 63 characters"); @@ -803,7 +730,7 @@ Create `ec/identity/mod.rs`: //! Server-side identity providers. //! //! An [`IdentityProvider`](provider::IdentityProvider) turns request signals -//! into OpenRTB EIDs. Providers only build requests and parse buffered +//! into `OpenRTB` EIDs. Providers only build requests and parse buffered //! responses. The coordinator owns eligibility, consent, transport limits and //! KV write-back, so every page load makes at most one browser-driven KV //! write. See `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. @@ -819,12 +746,12 @@ pub use resolution::IdentityProviderError; Add `pub mod identity;` to the module list in `ec/mod.rs`, in alphabetical order after `pub mod identify;`. -- [ ] **Step 3: Run and see the tests fail** +- [ ] **Step 2: Run and see the tests fail** Run: `cargo test-fastly -- ec::identity::ids` Expected: compile errors, because the types do not exist. -- [ ] **Step 4: Implement `resolution.rs` (error only) and the minimal `descriptor.rs`** +- [ ] **Step 3: Implement `resolution.rs` (error only) and the minimal `descriptor.rs`** `ec/identity/descriptor.rs` for now: @@ -966,7 +893,7 @@ impl IdentityProviderError { } ``` -- [ ] **Step 5: Implement `ids.rs`** +- [ ] **Step 4: Implement `ids.rs`** Put this above the test module: @@ -1092,7 +1019,7 @@ impl UnixTime { Self(secs) } - /// From Unix milliseconds, for vendors such as Lockr and UID2. Floors to + /// From Unix milliseconds, for vendors such as Lockr and `UID2`. Floors to /// whole seconds. #[must_use] pub fn from_millis(millis: u64) -> Self { @@ -1128,76 +1055,185 @@ Add both new files to `checked_sources()` in `migration_guards.rs`, in alphabeti Every later task that creates a file under `ec/identity/` adds it to this list the same way. -- [ ] **Step 6: Run and see the tests pass** +- [ ] **Step 5: Run and see the tests pass** Run: `cargo test-fastly -- ec::identity` and `cargo test-fastly -- migration_guards` Expected: PASS. -- [ ] **Step 7: Commit** +- [ ] **Step 6: Commit** ```bash -git add Cargo.toml Cargo.lock crates/trusted-server-core/Cargo.toml crates/trusted-server-core/src/ec/mod.rs crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs -git commit --signoff -S -m "Add the identity module with provider and source ID newtypes" -m "Adds aes-gcm, which the sealed identity cookies use." +git add crates/trusted-server-core/src/ec/mod.rs crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs +git commit --signoff -S -m "Add the identity module with provider and source ID newtypes" ``` -### Task 6: KV schema: record metadata, provider state, `Default` and constructors +### Task 6: KV schema version 2: one record shape, provider state, `Default` and constructors **Files:** -- Modify: `crates/trusted-server-core/src/ec/kv_types.rs` (`KvEntry`, `KvConsent`, `KvGeo`, `KvPartnerId`, `validate`, constructors) -- Modify: every file with a `KvPartnerId {` literal (find them with `grep -rn "KvPartnerId {" crates/`) -- Modify: `crates/trusted-server-core/src/ec/admin.rs` (raw lookup response, around `admin.rs:394`) -- Test: `crates/trusted-server-core/src/ec/kv_types.rs`, `crates/trusted-server-core/src/ec/admin.rs` +- Modify: `crates/trusted-server-core/src/ec/kv_types.rs` (`SCHEMA_VERSION`, `KvEntry`, `KvConsent`, `KvGeo`, `KvPartnerId`, `validate`, `upgrade_for_read`, constructors, the size estimate in the `KvEntry` doc comment) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (`deserialize_entry`, every production `.uid` reader, and the tests at about lines 1736 and 1758) +- Modify: every file with a `KvPartnerId {` literal or a `.uid` read of a `KvPartnerId` (find them with `grep -rn "KvPartnerId {" crates/` and the compiler) +- Modify: `crates/trusted-server-core/src/ec/admin.rs` (`build_lookup_response`, around `admin.rs:375-415`, and the `.uid` readers at about lines 490 and 616) +- Modify: `crates/trusted-server-core/src/ec/eids.rs` (the `.uid` readers at about lines 40, 53 and 71; Task 14 rewrites this file's output path) +- Test: `crates/trusted-server-core/src/ec/kv_types.rs`, `crates/trusted-server-core/src/ec/kv.rs`, `crates/trusted-server-core/src/ec/admin.rs` **Interfaces:** - Produces: - - `KvPartnerId { uid, w: Option, rv: Option, at: Option, exp: Option, ra: Option, ru: Option, atype: Option, ext: Option }`, deriving `Default`, with `KvPartnerId::new(uid: impl Into) -> Self` and `KvPartnerId::browser(uid: impl Into, acquired_at: Option) -> Self` - - `KvProviderState { n: Option, f: u32, o: Option, oc: Option, iv: Vec, st: Option }`, deriving `Default` - - `KvOptOut { at: u64, fp: String }` + - `pub const SCHEMA_VERSION: u8 = 2;`. `KvMetadata.v` mirrors it on every write. + - `KvPartnerId { w: Option, rv: Option, at: Option, exp: Option, ra: Option, ru: Option, eids: Vec }`, deriving `Debug, Clone, Default, PartialEq, Serialize, Deserialize`, with `#[serde(from = "KvPartnerIdWire")]`. It has no `uid`, `atype` or `ext` field. `eids` is always serialized and never empty in a valid record. Each optional field is skipped when `None`. + - `KvPartnerId::new(uid: impl Into) -> Self` (one entry with one UID and no metadata), `KvPartnerId::browser(uid: impl Into, acquired_at: Option) -> Self`, and `KvPartnerId::first_uid(&self) -> Option<&str>` (the first UID of the first entry) + - `KvEid { uids: Vec, inserter: Option, matcher: Option, mm: Option, ext: Option }` (no `source`: the `ids` key holds it) and `KvUid { id: String, atype: Option, ext: Option }`, both deriving `Debug, Clone, Default, PartialEq, Serialize, Deserialize`, with every optional field skipped when `None`, and neither denying unknown fields + - `KvProviderState { n: Option, f: u32, o: Option, oc: Option, iv: Vec, ov: Vec, st: Option }`, deriving `Default` + - `KvOptOut { at: u64, fp: String }`. `fp` is the consent bitmask of spec 8 as lowercase hex, at most 16 digits (Task 20 defines the bits). - `KvEntry.ps: BTreeMap`, and `KvEntry` derives `Default` - - Constants `MAX_PARTNER_EXT_BYTES = 256`, `MAX_PROVIDER_STATE_BYTES = 1024`, `MAX_PROVIDER_STATES = 32`, `MAX_INVALIDATED_UIDS = 4`, `RECORD_REVISION_HEX_LEN = 16` + - Constants `MAX_PARTNER_EXT_BYTES = 2048` (UID-level and EID-level `ext`), `MAX_EID_FIELD_BYTES = 255` (`inserter`, `matcher`), `MAX_ENTRIES_PER_SOURCE = 8`, `MAX_UIDS_PER_SOURCE = 32`, `MAX_ENTRY_SET_BYTES = 4096` (compact JSON of the stored set), `MAX_PROVIDER_STATE_BYTES = 1024`, `MAX_PROVIDER_STATES = 32`, `MAX_INVALIDATED_UIDS = 128` (the cap of `ps.iv` and of `ps.ov`, each an ordered list, oldest first), `RECORD_REVISION_HEX_LEN = 16` + - `pub(crate) fn entry_set_bytes(set: &[KvEid]) -> usize`: the length of `serde_json::to_vec(set)`, the stored array without `source`, after the field rules and without provenance defaults. `KvEntry::validate` and the coordinator (Tasks 10 and 20) both measure `MAX_ENTRY_SET_BYTES` with it, so they never disagree. + - `pub(crate) fn KvEntry::upgrade_for_read(&mut self) -> Result, String>`: accepts `v` of 1 or 2 and sets it to 2, rejecting any other version as today's check does. It then drops each record that breaks a rule of `validate_record`, and returns one message per dropped record, naming the source and the rule, never a UID (spec 6.1). `deserialize_entry` calls it before `validate` and logs each message at warn. Only `serialize_entry` rejects a bad record. + +Spec 6.1: every record stores its source's entry set in `eids`. A record that browser ingest, generic pull or batch sync writes is one entry with one UID and no EID-level fields. Only a provider result writes more, and nothing in Part 1 does. This task makes larger sets read, preserved and validated, so a rollback from Part 2 to Part 1 keeps every set a provider wrote readable. Such a set is not served while its writer is not configured (spec 14). + +Every row on `main` is version 1, whose records hold one `uid`. The reader accepts both versions: a version 1 record reads as one entry holding that `uid`, with no `w`, no `rv` and no timing beyond `at`, and the row's next write stores version 2. Today's reader rejects every version but its own (`ec/kv_types.rs:376-381`), so a build without this task cannot read a row this task has written. Rolling back past Part 1 is unsupported (spec 14), and Task 31's changelog says so. + +`validate` runs on every read (`ec/kv.rs:366`, `deserialize_entry`) as well as on every write. One bad record must not make its whole row unreadable, because nothing could then write the row again. So the read path first drops such a record, and only the write path rejects it. - [ ] **Step 1: Write the failing tests** -Add to `kv_types.rs` tests: +Add to `kv_types.rs` tests. The test module has `use super::*;` only, so also add `use serde_json::json;`: ```rust + fn kv_uid(id: &str, atype: Option) -> KvUid { + KvUid { + id: id.to_owned(), + atype, + ext: None, + } + } + + fn kv_eid(uids: Vec) -> KvEid { + KvEid { + uids, + inserter: None, + matcher: None, + mm: None, + ext: None, + } + } + + /// The set of spec 6.1's `example-ids.example.com` record. + fn example_full_set() -> Vec { + vec![ + KvEid { + uids: vec![ + kv_uid("example-a", Some(1)), + KvUid { + id: "example-b".to_owned(), + atype: Some(3), + ext: Some(json!({ "stype": "ppuid" })), + }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + KvEid { + uids: vec![kv_uid("example-c", Some(1))], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ] + } + + fn with_set(set: Vec) -> KvEntry { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ids.get_mut("example.com").expect("should exist").eids = set; + entry + } + #[test] - fn legacy_partner_record_reads_and_writes_unchanged() { - let legacy = json!({ "uid": "example-uid" }); - let record: KvPartnerId = serde_json::from_value(legacy.clone()).expect("should read legacy"); - assert_eq!(record, KvPartnerId::new("example-uid"), "should fill every new field with None"); - assert_eq!(serde_json::to_value(&record).expect("should write"), legacy, "should add no fields"); + fn golden_version_1_row_reads_as_one_entry_records_and_writes_version_2() { + let main_row = json!({ + "v": 1, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { + "id5-sync.com": { "uid": "ID5*example" }, + "sharedid.org": { "uid": "example-shared-id" } + } + }); + + let mut entry: KvEntry = serde_json::from_value(main_row).expect("should read a row from main"); + let messages = entry.upgrade_for_read().expect("should accept version 1"); + entry.validate().expect("should be valid once upgraded"); + + assert!(messages.is_empty(), "should drop nothing from a valid version 1 row"); + assert_eq!(entry.ids["id5-sync.com"], KvPartnerId::new("ID5*example"), "should read uid as one entry"); + assert_eq!( + serde_json::to_value(&entry).expect("should write"), + json!({ + "v": 2, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { + "id5-sync.com": { "eids": [{ "uids": [{ "id": "ID5*example" }] }] }, + "sharedid.org": { "eids": [{ "uids": [{ "id": "example-shared-id" }] }] } + } + }), + "should write version 2, with no uid key and no ps" + ); } #[test] - fn provider_record_and_state_round_trip() { + fn version_2_row_and_provider_state_round_trip() { let stored = json!({ - "v": 1, + "v": 2, "created": 1_760_000_000, "consent": { "ok": true, "updated": 1_760_000_000 }, "geo": { "country": "US" }, "ids": { "id5-sync.com": { - "uid": "ID5*example", "w": "id5", "rv": "9c1d04b7e6a83f2a", "at": 1_760_000_000, "exp": 1_767_776_000, "ra": 1_760_086_400, "ru": 1_770_000_000, - "atype": 1, - "ext": { "linkType": 2 } + "eids": [{ "uids": [{ "id": "ID5*example", "atype": 1, "ext": { "linkType": 2 } }] }] + }, + "example-ids.example.com": { + "w": "exampleid", + "rv": "4be71f0c2d9a6e35", + "at": 1_760_000_000, + "eids": [ + { + "uids": [ + { "id": "example-a", "atype": 1 }, + { "id": "example-b", "atype": 3, "ext": { "stype": "ppuid" } } + ], + "inserter": "example.com", + "mm": 3 + }, + { + "uids": [{ "id": "example-c", "atype": 1 }], + "matcher": "partner.example.com" + } + ] } }, "ps": { "id5": { "n": 1_760_003_600, "f": 1, - "o": { "at": 1_759_000_000, "fp": "0123456789abcdef" }, + "o": { "at": 1_759_000_000, "fp": "1a" }, "oc": 1_759_500_000, "iv": ["aaaaaaaaaaaaaaaa"], + "ov": ["bbbbbbbbbbbbbbbb"], "st": { "signature": "example-opaque-state" } } } @@ -1206,21 +1242,163 @@ Add to `kv_types.rs` tests: let entry: KvEntry = serde_json::from_value(stored.clone()).expect("should read"); entry.validate().expect("should be valid"); assert_eq!(serde_json::to_value(&entry).expect("should write"), stored, "should round-trip exactly"); + assert_eq!( + entry.ids["example-ids.example.com"].eids, + example_full_set(), + "should read the set whole and in order" + ); } #[test] - fn validate_rejects_oversized_ext_state_and_too_many_providers() { - let mut entry = KvEntry::minimal("example.com", "example-uid", 1); - entry.ids.get_mut("example.com").expect("should exist").ext = - Some(json!({ "pad": "x".repeat(MAX_PARTNER_EXT_BYTES) })); - assert!(entry.validate().expect_err("should reject ext").contains("ext"), "should name ext"); + fn a_record_reads_its_eids_and_its_version_1_uid_only_without_them() { + let both: KvPartnerId = serde_json::from_value(json!({ + "uid": "stale-copy", + "eids": [{ "uids": [{ "id": "example-a" }] }] + })) + .expect("should read a record with both keys"); + assert_eq!(both.first_uid(), Some("example-a"), "should take eids over a uid copy"); - let mut entry = KvEntry::minimal("example.com", "example-uid", 1); - entry.ps.insert( - "id5".to_owned(), - KvProviderState { st: Some(json!("x".repeat(MAX_PROVIDER_STATE_BYTES))), ..KvProviderState::default() }, + let neither: KvPartnerId = serde_json::from_value(json!({ "w": "exampleid" })).expect("should read"); + assert!(neither.eids.is_empty(), "should read no entry from a record with neither key"); + assert_eq!(neither.first_uid(), None, "should have no first UID"); + with_set(Vec::new()).validate().expect_err("should reject a record with an empty eids on write"); + } + + #[test] + fn upgrade_for_read_drops_bad_records_and_names_the_source_only() { + let mut set = example_full_set(); + set[1].matcher = Some(" ".to_owned()); + let mut entry = with_set(set); + entry.ids.insert("sharedid.org".to_owned(), KvPartnerId::new("example-shared-id")); + entry.v = 1; + + let messages = entry.upgrade_for_read().expect("should accept the row"); + + assert_eq!(entry.v, SCHEMA_VERSION, "should upgrade the version"); + assert_eq!(messages.len(), 1, "should report one dropped record"); + assert!(messages[0].contains("example.com"), "should name the source"); + assert!(!messages[0].contains("example-a"), "should never name a UID"); + assert!(!entry.ids.contains_key("example.com"), "should drop the bad record"); + assert_eq!( + entry.ids["sharedid.org"], + KvPartnerId::new("example-shared-id"), + "should leave a valid record alone" + ); + entry.validate().expect("should validate once upgraded"); + + let mut future = KvEntry::minimal("example.com", "example-uid", 1); + future.v = SCHEMA_VERSION + 1; + future.upgrade_for_read().expect_err("should reject a version it does not know"); + } + + #[test] + fn entry_set_bytes_measures_the_stored_array() { + let set = example_full_set(); + let bytes = serde_json::to_vec(&set).expect("should serialize the set"); + + assert_eq!(entry_set_bytes(&set), bytes.len(), "should measure the stored array"); + assert!( + !String::from_utf8(bytes).expect("should be UTF-8").contains("\"source\""), + "should measure entries without source" ); - assert!(entry.validate().expect_err("should reject st").contains("st"), "should name st"); + } + + #[test] + fn validate_rejects_entry_set_limits() { + let rejects = |entry: &KvEntry, field: &str| { + let error = entry.validate().expect_err(&format!("should reject {field}")); + assert!(error.contains(field), "should name {field}, got {error}"); + }; + + rejects(&with_set(Vec::new()), "eids"); + + let mut set = example_full_set(); + set[0].ext = Some(json!({ "pad": "x".repeat(MAX_PARTNER_EXT_BYTES) })); + rejects(&with_set(set), "ext"); + + let mut set = example_full_set(); + set[0].uids[0].ext = Some(json!("not an object")); + rejects(&with_set(set), "ext"); + + let mut set = example_full_set(); + set[0].inserter = Some(" ".to_owned()); + rejects(&with_set(set), "inserter"); + + let mut set = example_full_set(); + set[1].matcher = Some("m".repeat(MAX_EID_FIELD_BYTES + 1)); + rejects(&with_set(set), "matcher"); + + let mut set = example_full_set(); + set[0].mm = Some(-1); + rejects(&with_set(set), "mm"); + + let mut set = example_full_set(); + set[0].uids[1].atype = Some(-1); + rejects(&with_set(set), "atype"); + + let mut set = example_full_set(); + set[1].uids[0].id = " ".to_owned(); + rejects(&with_set(set), "uid"); + + let mut set = example_full_set(); + set[1].uids.clear(); + rejects(&with_set(set), "uid"); + + let nine_entries = (0..=MAX_ENTRIES_PER_SOURCE) + .map(|index| kv_eid(vec![kv_uid(&format!("example-{index}"), None)])) + .collect(); + rejects(&with_set(nine_entries), "entries"); + + let thirty_three_uids = (0..=MAX_UIDS_PER_SOURCE) + .map(|index| kv_uid(&format!("example-{index}"), None)) + .collect(); + rejects(&with_set(vec![kv_eid(thirty_three_uids)]), "uids"); + + let oversized = (0..4) + .map(|index| KvEid { + ext: Some(json!({ "pad": "x".repeat(1_100) })), + ..kv_eid(vec![kv_uid(&format!("example-{index}"), None)]) + }) + .collect(); + rejects(&with_set(oversized), "entry set"); + + let mut old = with_set(example_full_set()); + old.v = 1; + rejects(&old, "version"); + + with_set(example_full_set()).validate().expect("should accept the control set"); + } + + #[test] + fn validate_rejects_oversized_state_and_too_many_providers() { + let with_state = |state: KvProviderState| { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ps.insert("id5".to_owned(), state); + entry + }; + let hashes = |count: usize| vec!["a".repeat(16); count]; + + let error = with_state(KvProviderState { st: Some(json!("x".repeat(MAX_PROVIDER_STATE_BYTES))), ..KvProviderState::default() }) + .validate() + .expect_err("should reject st"); + assert!(error.contains("st"), "should name st"); + + for (name, state) in [ + ("iv", KvProviderState { iv: hashes(MAX_INVALIDATED_UIDS + 1), ..KvProviderState::default() }), + ("ov", KvProviderState { ov: hashes(MAX_INVALIDATED_UIDS + 1), ..KvProviderState::default() }), + ("fp", KvProviderState { o: Some(KvOptOut { at: 1, fp: "XYZ".to_owned() }), ..KvProviderState::default() }), + ] { + let error = with_state(state).validate().expect_err(&format!("should reject {name}")); + assert!(error.contains(name), "should name {name}, got {error}"); + } + with_state(KvProviderState { + iv: hashes(MAX_INVALIDATED_UIDS), + ov: hashes(MAX_INVALIDATED_UIDS), + o: Some(KvOptOut { at: 1, fp: "1a".to_owned() }), + ..KvProviderState::default() + }) + .validate() + .expect("should accept 128 hashes in each list and a hex fingerprint"); let mut entry = KvEntry::minimal("example.com", "example-uid", 1); for index in 0..=MAX_PROVIDER_STATES { @@ -1228,22 +1406,20 @@ Add to `kv_types.rs` tests: } assert!(entry.validate().expect_err("should reject ps size").contains("ps"), "should name ps"); - let mut entry = KvEntry::minimal("example.com", "example-uid", 1); - entry.ps.insert( - "id5".to_owned(), - KvProviderState { iv: vec!["a".repeat(16); MAX_INVALIDATED_UIDS + 1], ..KvProviderState::default() }, - ); - assert!(entry.validate().expect_err("should reject iv").contains("iv"), "should name iv"); - let mut entry = KvEntry::minimal("example.com", "example-uid", 1); entry.ids.get_mut("example.com").expect("should exist").rv = Some("not-hex".to_owned()); assert!(entry.validate().expect_err("should reject rv").contains("rv"), "should name rv"); - KvEntry::minimal("example.com", "example-uid", 1).validate().expect("control entry should be valid"); + KvEntry::minimal("example.com", "example-uid", 1).validate().expect("should accept the control entry"); } ``` -The validation test builds `KvProviderState` with `..KvProviderState::default()` because it tests one field at a time. It is not an exhaustiveness test. +The validation tests build `KvProviderState` with `..KvProviderState::default()` and `KvEid` with struct update syntax because they test one field at a time. They are not exhaustiveness tests. The oversized-set case keeps every `ext` under 2,048 bytes and every other limit intact, so only the 4,096-byte set limit applies. + +In `kv.rs` tests, rewrite the two tests that pin today's version and UID checks: + +- `deserialize_entry_rejects_invalid_legacy_values` (about line 1736) becomes `deserialize_entry_drops_an_invalid_record`: a body whose record for one source has a UID over `MAX_UID_LENGTH` deserializes, the row reads without that source, and a second, valid source is kept. A row-level fault, such as a malformed `consent`, still fails the read as a whole, as today. +- The test that sets `SCHEMA_VERSION + 1` (about line 1758) keeps its expectation: the read fails. Add a sibling, `deserialize_entry_reads_a_version_1_body`, whose body is a row from `main`. Add to `admin.rs` tests, next to the existing raw-lookup test (`grep -n "lookup_raw\|fn .*lookup" crates/trusted-server-core/src/ec/admin.rs`). Use the same setup that test uses to put an entry in the in-memory graph and call the handler: @@ -1257,41 +1433,57 @@ Add to `admin.rs` tests, next to the existing raw-lookup test (`grep -n "lookup_ ); let body = admin_lookup_body_for_test(&entry); - assert_eq!(body["ps"]["id5"]["st"], json!("[redacted]"), "should redact the state"); - assert_eq!(body["ps"]["id5"]["n"], json!(5), "should keep the rest of ps"); - assert_eq!(body["ids"]["example.com"]["uid"], json!("example-uid"), "should keep the ids"); + assert_eq!(body["entry"]["ps"]["id5"]["st"], json!("[redacted]"), "should redact the state"); + assert_eq!(body["entry"]["ps"]["id5"]["n"], json!(5), "should keep the rest of ps"); + assert_eq!( + body["entry"]["ids"]["example.com"]["eids"][0]["uids"][0]["id"], + json!("example-uid"), + "should keep the ids" + ); } ``` -`admin_lookup_body_for_test` stands for whatever the existing raw-lookup test does to get the JSON body: insert the row, call the handler and parse the body. Write it as a small helper in the test module that reuses that test's setup. +`admin_lookup_body_for_test` stands for whatever the existing lookup tests do to get the JSON body: insert the row, call the handler and parse the body. Write it as a small helper in the test module that reuses that setup. The stored row sits under `"entry"` in the response, as the existing assertions such as `json["entry"]["ids"]["bidstream.example"]["uid"]` (`admin.rs:998`) show. Those existing assertions read `uid`; move each to `eids[0].uids[0].id`. - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- ec::kv_types::tests` and `cargo test-fastly -- ec::admin::tests::admin_lookup_redacts` -Expected: compile errors (`ps`, `KvProviderState` and `KvPartnerId::new` do not exist). +Run: `cargo test-fastly -- ec::kv_types::tests`, `cargo test-fastly -- ec::kv::tests::deserialize_entry` and `cargo test-fastly -- ec::admin::tests::admin_lookup_redacts` +Expected: compile errors (`ps`, `KvProviderState`, `KvEid`, `KvUid`, `KvPartnerId::new`, `first_uid`, `entry_set_bytes` and `upgrade_for_read` do not exist). - [ ] **Step 3: Implement the schema** -In `kv_types.rs`, add the constants after `MAX_UID_LENGTH`: +In `kv_types.rs`, set `SCHEMA_VERSION` to 2 and add the constants after `MAX_UID_LENGTH`: ```rust -/// Maximum serialized bytes of a stored UID-level `ext`. -pub const MAX_PARTNER_EXT_BYTES: usize = 256; +/// Maximum serialized bytes of a stored UID-level or EID-level `ext`. +pub const MAX_PARTNER_EXT_BYTES: usize = 2048; +/// Maximum bytes of a stored `inserter` or `matcher`. +pub const MAX_EID_FIELD_BYTES: usize = 255; +/// Maximum number of EID entries stored for one source. +pub const MAX_ENTRIES_PER_SOURCE: usize = 8; +/// Maximum number of UIDs stored for one source, across its entries. +pub const MAX_UIDS_PER_SOURCE: usize = 32; +/// Maximum bytes of one source's stored entry set, as measured by +/// [`entry_set_bytes`]. +pub const MAX_ENTRY_SET_BYTES: usize = 4096; /// Maximum serialized bytes of one provider's private state (`ps.st`). pub const MAX_PROVIDER_STATE_BYTES: usize = 1024; /// Maximum number of providers with a `ps` entry in one row. pub const MAX_PROVIDER_STATES: usize = 32; -/// Maximum number of invalidated-UID hashes kept per provider (`ps.iv`). -pub const MAX_INVALIDATED_UIDS: usize = 4; +/// Maximum number of UID hashes kept in each of a provider's `ps.iv` and +/// `ps.ov` lists, enough for four whole 32-UID sets. +pub const MAX_INVALIDATED_UIDS: usize = 128; /// Length of a record revision (`rv`) in lowercase hex digits. pub const RECORD_REVISION_HEX_LEN: usize = 16; ``` +Change the size estimate in the `KvEntry` doc comment from `(max ~5KB)` to the typical and at-every-cap figures spec 6.1 gives, citing spec 6.1. + Add `Default` to the derives of `KvEntry`, `KvConsent` and `KvGeo`. Add the `ps` field at the end of `KvEntry`: ```rust - /// Per-provider scheduling, opt-out and private state, keyed by - /// identity provider ID. Never sent to bidders or cookies. + /// Per-provider scheduling, opt-out, removal and private state, keyed by + /// identity provider ID. Never sent to bidders. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub ps: BTreeMap, ``` @@ -1299,15 +1491,15 @@ Add `Default` to the derives of `KvEntry`, `KvConsent` and `KvGeo`. Add the `ps` Replace `KvPartnerId`: ```rust -/// A partner user ID within a KV entry. +/// A source's record within a KV entry (spec 6.1). /// -/// Every field but `uid` is optional and omitted when absent, so legacy rows -/// read and write unchanged and `SCHEMA_VERSION` stays 1 (spec 6.1). +/// Every record stores its source's entry set in `eids`. A version 1 record, +/// which holds one `uid`, reads as one entry holding that UID. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(from = "KvPartnerIdWire")] pub struct KvPartnerId { - /// The partner's user identifier. - pub uid: String, - /// Writer: an identity provider ID. Absent for browser ingest and legacy rows. + /// Writer: an identity provider ID. Absent for browser ingest and + /// version 1 records. #[serde(default, skip_serializing_if = "Option::is_none")] pub w: Option, /// Record revision: 16 lowercase hex digits, random on every write. @@ -1316,7 +1508,7 @@ pub struct KvPartnerId { /// Unix seconds when the ID was acquired or last refreshed. #[serde(default, skip_serializing_if = "Option::is_none")] pub at: Option, - /// Unix seconds after which the ID must not be sent. + /// Unix seconds at or after which the ID must not be sent. #[serde(default, skip_serializing_if = "Option::is_none")] pub exp: Option, /// Unix seconds after which the ID is due for refresh. @@ -1325,26 +1517,102 @@ pub struct KvPartnerId { /// Unix seconds after which the ID can no longer be refreshed. #[serde(default, skip_serializing_if = "Option::is_none")] pub ru: Option, - /// Provider-supplied `atype`. Absent: use the partner's `openrtb_atype`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub atype: Option, - /// UID-level `ext`, at most [`MAX_PARTNER_EXT_BYTES`] serialized bytes. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub ext: Option, + /// The source's entry set, in order. Never empty in a valid record. + pub eids: Vec, +} + +/// The stored shape of [`KvPartnerId`], which also reads version 1's `uid`. +#[derive(Deserialize)] +struct KvPartnerIdWire { + #[serde(default)] + uid: Option, + #[serde(default)] + w: Option, + #[serde(default)] + rv: Option, + #[serde(default)] + at: Option, + #[serde(default)] + exp: Option, + #[serde(default)] + ra: Option, + #[serde(default)] + ru: Option, + #[serde(default)] + eids: Vec, +} + +impl From for KvPartnerId { + fn from(wire: KvPartnerIdWire) -> Self { + let eids = match (wire.eids.is_empty(), wire.uid) { + (true, Some(uid)) => vec![KvEid::one_uid(uid)], + (_, _) => wire.eids, + }; + Self { w: wire.w, rv: wire.rv, at: wire.at, exp: wire.exp, ra: wire.ra, ru: wire.ru, eids } + } } impl KvPartnerId { - /// A bare record holding only a UID. + /// A record holding one entry with one UID, and no metadata. #[must_use] pub fn new(uid: impl Into) -> Self { - Self { uid: uid.into(), ..Self::default() } + Self { eids: vec![KvEid::one_uid(uid.into())], ..Self::default() } } /// A browser-ingested record: no writer, with its acquisition time. #[must_use] pub fn browser(uid: impl Into, acquired_at: Option) -> Self { - Self { uid: uid.into(), at: acquired_at, ..Self::default() } + Self { at: acquired_at, ..Self::new(uid) } } + + /// The first UID of the first entry, which `/identify` returns and + /// today's one-UID readers use. + #[must_use] + pub fn first_uid(&self) -> Option<&str> { + self.eids.first().and_then(|entry| entry.uids.first()).map(|uid| uid.id.as_str()) + } +} + +/// One EID entry of a record (spec 6.1). +/// +/// The `OpenRTB` EID object without `source`, which the `ids` key holds. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvEid { + /// The entry's UIDs, in order. Never empty in a valid record. + pub uids: Vec, + /// `eid.inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// `eid.matcher`. Absent means `source`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// `eid.mm`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// EID-level `ext`, at most [`MAX_PARTNER_EXT_BYTES`] serialized bytes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +impl KvEid { + /// One entry holding one UID and nothing else: the shape browser + /// ingest, generic pull and batch sync write. + fn one_uid(id: String) -> Self { + Self { uids: vec![KvUid { id, atype: None, ext: None }], ..Self::default() } + } +} + +/// One UID of a stored entry. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + /// The user identifier. + pub id: String, + /// `uid.atype`. Absent: use the partner's `openrtb_atype`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub atype: Option, + /// UID-level `ext`, at most [`MAX_PARTNER_EXT_BYTES`] serialized bytes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, } /// A provider's sticky opt-out (spec 8). @@ -1352,7 +1620,8 @@ impl KvPartnerId { pub struct KvOptOut { /// Unix seconds when the opt-out was reported. pub at: u64, - /// Consent fingerprint at that time: 16 lowercase hex digits. + /// The consent bits the provider's policy read then, as a lowercase hex + /// bitmask of at most 16 digits. Plain, not keyed (spec 8). pub fp: String, } @@ -1371,9 +1640,14 @@ pub struct KvProviderState { /// When the provider's opt-out was last cleared. #[serde(default, skip_serializing_if = "Option::is_none")] pub oc: Option, - /// Keyed hashes of UIDs the owner invalidated, newest last. + /// Unkeyed hashes of UIDs the owner invalidated, oldest first, at most + /// [`MAX_INVALIDATED_UIDS`]. Never emptied (spec 5.4). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub iv: Vec, + /// Unkeyed hashes of UIDs the provider's opt-out removed, oldest first, + /// at most [`MAX_INVALIDATED_UIDS`]. Emptied when the opt-out clears. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ov: Vec, /// The provider's private state as embedded JSON. KV only. #[serde(default, skip_serializing_if = "Option::is_none")] pub st: Option, @@ -1382,35 +1656,37 @@ pub struct KvProviderState { fn is_zero(value: &u32) -> bool { *value == 0 } -``` -The listed constructors (`KvEntry::new`, `minimal`, `tombstone`) gain `ps: BTreeMap::new()`. `minimal` uses `KvPartnerId::new(uid)`. +/// Bytes of a stored entry set as compact JSON: the array of entries without +/// `source`, as `eids` stores it. Both `validate` and the coordinator measure +/// [`MAX_ENTRY_SET_BYTES`] with this function. +pub(crate) fn entry_set_bytes(set: &[KvEid]) -> usize { + serde_json::to_vec(set).map_or(usize::MAX, |bytes| bytes.len()) +} +``` -Extend `validate()`. Inside the existing `for (partner_id, partner_uid) in &self.ids` loop, after the UID length check: +`KvPartnerId`, `KvEid` and `KvUid` do not deny unknown fields, so a field a later build adds never makes this build fail to read the row (spec 6.1). -```rust - if let Some(ext) = &partner_uid.ext - && serde_json::to_vec(ext).map_or(true, |bytes| bytes.len() > MAX_PARTNER_EXT_BYTES) - { - return Err(format!("partner ID '{partner_id}' ext exceeds {MAX_PARTNER_EXT_BYTES} bytes")); - } - if let Some(rv) = &partner_uid.rv - && (rv.len() != RECORD_REVISION_HEX_LEN - || !rv.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))) - { - return Err(format!("partner ID '{partner_id}' rv is not {RECORD_REVISION_HEX_LEN} hex digits")); - } -``` +The listed constructors (`KvEntry::new`, `minimal`, `tombstone`) gain `ps: BTreeMap::new()` and write `v: SCHEMA_VERSION`, which is now 2. `minimal` uses `KvPartnerId::new(uid)`. `KvMetadata.v` takes the entry's `v`, so metadata written from now on says 2. Find every reader of the metadata version (`grep -rn "\.v\b\|SCHEMA_VERSION" crates/trusted-server-core/src/ec/`) and make each accept 1 and 2 where it reads, and write 2. -After the loop: +Extend `validate()`. It still rejects `v != SCHEMA_VERSION`, now 2, with a message containing `version`. Inside the existing `for (partner_id, partner_uid) in &self.ids` loop, replace the UID length check with `validate_record(partner_id, partner_uid)?;`. After the loop: ```rust if self.ps.len() > MAX_PROVIDER_STATES { return Err(format!("ps exceeds MAX_PROVIDER_STATES ({})", self.ps.len())); } for (provider, state) in &self.ps { - if state.iv.len() > MAX_INVALIDATED_UIDS { - return Err(format!("ps '{provider}' iv exceeds {MAX_INVALIDATED_UIDS} hashes")); + for (name, hashes) in [("iv", &state.iv), ("ov", &state.ov)] { + if hashes.len() > MAX_INVALIDATED_UIDS { + return Err(format!("ps '{provider}' {name} exceeds {MAX_INVALIDATED_UIDS} hashes")); + } + } + if let Some(opt_out) = &state.o + && (opt_out.fp.is_empty() + || opt_out.fp.len() > 16 + || !opt_out.fp.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))) + { + return Err(format!("ps '{provider}' o.fp is not a hex bitmask")); } if let Some(st) = &state.st && serde_json::to_vec(st).map_or(true, |bytes| bytes.len() > MAX_PROVIDER_STATE_BYTES) @@ -1420,60 +1696,172 @@ After the loop: } ``` -- [ ] **Step 4: Move every literal to the constructor** +Add the helpers after `impl KvEntry`: + +```rust +/// Checks one record against the limits of spec 4.5 and 6.1. +fn validate_record(partner_id: &str, record: &KvPartnerId) -> Result<(), String> { + if record.eids.is_empty() { + return Err(format!("partner ID '{partner_id}' has an empty eids")); + } + if let Some(rv) = &record.rv + && (rv.len() != RECORD_REVISION_HEX_LEN + || !rv.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))) + { + return Err(format!("partner ID '{partner_id}' rv is not {RECORD_REVISION_HEX_LEN} hex digits")); + } + if record.eids.len() > MAX_ENTRIES_PER_SOURCE { + return Err(format!("partner ID '{partner_id}' has more than {MAX_ENTRIES_PER_SOURCE} entries")); + } + let mut uid_count = 0; + for entry in &record.eids { + if entry.uids.is_empty() { + return Err(format!("partner ID '{partner_id}' has an entry without a uid")); + } + for (name, value) in [("inserter", &entry.inserter), ("matcher", &entry.matcher)] { + if let Some(value) = value + && (value.trim().is_empty() || value.len() > MAX_EID_FIELD_BYTES) + { + return Err(format!("partner ID '{partner_id}' {name} is blank or over {MAX_EID_FIELD_BYTES} bytes")); + } + } + if entry.mm.is_some_and(|mm| mm < 0) { + return Err(format!("partner ID '{partner_id}' mm is negative")); + } + validate_ext(partner_id, entry.ext.as_ref())?; + for uid in &entry.uids { + if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + return Err(format!("partner ID '{partner_id}' has a blank or oversized uid")); + } + if uid.atype.is_some_and(|atype| atype < 0) { + return Err(format!("partner ID '{partner_id}' atype is negative")); + } + validate_ext(partner_id, uid.ext.as_ref())?; + } + uid_count += entry.uids.len(); + } + if uid_count > MAX_UIDS_PER_SOURCE { + return Err(format!("partner ID '{partner_id}' has more than {MAX_UIDS_PER_SOURCE} uids")); + } + if entry_set_bytes(&record.eids) > MAX_ENTRY_SET_BYTES { + return Err(format!("partner ID '{partner_id}' entry set exceeds {MAX_ENTRY_SET_BYTES} bytes")); + } + Ok(()) +} + +/// Checks a UID-level or EID-level `ext`: a JSON object of at most +/// [`MAX_PARTNER_EXT_BYTES`] bytes as compact JSON. +fn validate_ext(partner_id: &str, ext: Option<&serde_json::Value>) -> Result<(), String> { + let Some(ext) = ext else { + return Ok(()); + }; + let oversized = serde_json::to_vec(ext).map_or(true, |bytes| bytes.len() > MAX_PARTNER_EXT_BYTES); + if !ext.is_object() || oversized { + return Err(format!("partner ID '{partner_id}' ext is not an object of at most {MAX_PARTNER_EXT_BYTES} bytes")); + } + Ok(()) +} +``` + +Today's UID check (`kv_types.rs:375`) decides whether a version 1 `uid` is valid. Read it before writing `validate_record`, and keep accepting every `uid` it accepts, so no row that reads on `main` loses a record here. If it accepts a blank `uid`, accept it too and say so in the commit body; the spec's non-blank rule then applies only to UIDs a provider writes (Task 10). + +Add `upgrade_for_read` to `impl KvEntry`: + +```rust + /// Brings a row read from KV to the current schema (spec 6.1). + /// + /// Accepts schema versions 1 and 2 and sets the version to 2. Drops each + /// record that breaks a rule of spec 4.5 or 6.1, and returns one message + /// per dropped record, naming the source and the rule, never a UID. + /// Called on read only, before [`Self::validate`], so one bad record + /// never makes the row unreadable. + /// + /// # Errors + /// + /// Returns an error naming the version when the row's version is + /// neither 1 nor 2. + pub(crate) fn upgrade_for_read(&mut self) -> Result, String> { + if !(1..=SCHEMA_VERSION).contains(&self.v) { + return Err(format!("unsupported schema version {}", self.v)); + } + self.v = SCHEMA_VERSION; + let mut messages = Vec::new(); + self.ids.retain(|partner_id, record| match validate_record(partner_id, record) { + Ok(()) => true, + Err(message) => { + messages.push(message); + false + } + }); + Ok(messages) + } +``` + +Every `validate_record` message names the partner ID and the rule and never a UID, so it is safe to log. In `ec/kv.rs` `deserialize_entry` (`kv.rs:355`), make `entry` mutable and call `entry.upgrade_for_read()` right before `entry.validate()`. Map its `Err` to the same error today's version check gives, and log each message with `log::warn!("Dropping an invalid record on read for key '{}': {message}", log_id(ec_id))`. `serialize_entry` keeps calling only `validate`, so a bad record is never written. In `ec/admin.rs` `build_lookup_response`, call `upgrade_for_read` on the parsed `KvEntry` before its `validate()` too, so the auction view matches what the read path serves, and when it returns messages set `payload.entry_error` to `dropped on read: ` followed by the messages joined with `; `. -Run `grep -rn "KvPartnerId {" crates/` and change each `KvPartnerId { uid: X }` to `KvPartnerId::new(X)`. That covers about 30 sites in 8 files, including `ec/kv.rs` (`apply_cookie_partner_id_updates`, `apply_partner_id_updates` and others), the batch sync and pull sync writers, the admin tests and `kv_types.rs`. Run `grep -rn "KvEntry {" crates/` too, and add `ps: BTreeMap::new()` to any full literal the compiler flags. +- [ ] **Step 4: Move every literal and `.uid` reader** + +Run `grep -rn "KvPartnerId {" crates/` and change each `KvPartnerId { uid: X }` to `KvPartnerId::new(X)`. That covers about 30 sites in 8 files, including `ec/kv.rs` (`apply_cookie_partner_id_updates`, `apply_partner_id_updates` and others), the batch sync and pull sync writers, the admin tests and `kv_types.rs`. None of them copies an existing record, so none loses metadata or entries. If one does, clone the record instead of rebuilding it. Run `grep -rn "KvEntry {" crates/` too, and add `ps: BTreeMap::new()` to any full literal the compiler flags. + +Then let the compiler list every read of the removed `uid` field. Each production reader moves to `first_uid()`, keeping today's meaning, because every record a Part 1 path reads that way is one entry with one UID: + +- `ec/kv.rs`, about lines 147, 186, 201 and 916, and the "is it the same UID" check at about line 988, which becomes `existing.first_uid() == Some(uid)` +- `ec/admin.rs`, about lines 490 and 616 +- `ec/eids.rs`, about lines 40, 53 and 71 (Task 14 replaces these with the stored set) + +Test readers in `ec/kv.rs`, `auction/endpoints.rs`, `ec/finalize.rs` and `ec/pull_sync.rs` move to `first_uid()` too, or compare whole records against `KvPartnerId::new(..)`. Do not change what they assert. - [ ] **Step 5: Redact `ps.st` in the admin lookup** -In `ec/admin.rs`, find where the raw lookup body is returned verbatim (`grep -n "lookup_raw" crates/trusted-server-core/src/ec/admin.rs`). Before it is written to the response, parse it, redact and re-serialize: +`build_lookup_response` (`ec/admin.rs:375`) already parses the body into a `JsonValue`, adds the `*_iso` companions and returns it as `payload.entry`. Redact that value in place, right after the parse and before `add_iso_timestamp_companions`: ```rust /// Replaces every `ps..st` with a marker. Provider state is never /// returned by the admin lookup (spec 6.1, 9.3). -fn redact_provider_state(body: &[u8]) -> Vec { - let Ok(mut value) = serde_json::from_slice::(body) else { - return body.to_vec(); +fn redact_provider_state(entry: &mut JsonValue) { + let Some(states) = entry.get_mut("ps").and_then(JsonValue::as_object_mut) else { + return; }; - if let Some(states) = value.get_mut("ps").and_then(serde_json::Value::as_object_mut) { - for state in states.values_mut() { - if let Some(st) = state.get_mut("st") { - *st = serde_json::Value::String("[redacted]".to_owned()); - } + for state in states.values_mut() { + if let Some(st) = state.get_mut("st") { + *st = JsonValue::String("[redacted]".to_owned()); } } - serde_json::to_vec(&value).unwrap_or_else(|_| body.to_vec()) } ``` -Call it on `lookup.body` where the handler builds its response. A body that is not JSON is returned unchanged, as today, because the lookup exists to inspect corrupt rows. +A body that is not JSON goes into `raw_body` unchanged, as today, because the lookup exists to inspect corrupt rows. Check that `raw_body` is only set for a body that does not parse, so it can never carry `ps.st`. The stored JSON is returned as stored, so a version 1 row shows `uid` until its next write. - [ ] **Step 6: Run and see the tests pass** Run: `cargo test-fastly -- ec::` and then `cargo clippy-fastly` -Expected: PASS. Every existing KV, batch sync and pull sync test must still pass unchanged. +Expected: PASS. Every existing KV, batch sync and pull sync test passes once its readers moved in Step 4, with its assertions unchanged. - [ ] **Step 7: Commit** ```bash -git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Add provider metadata and provider state to the KV schema" -m "New fields are optional and omitted when absent, so legacy rows read and write unchanged and the schema version stays 1. KvPartnerId and KvEntry derive Default, and literals move to constructors. The admin lookup redacts provider state." +git add -A crates/ +git commit --signoff -S -m "Move the KV schema to version 2 with one record shape" -m "Every record stores its source's entry set in eids, with optional writer, revision and timing fields, and the row gains per-provider state. Version 1 rows read as one-entry records and are written as version 2. A build without this change cannot read a version 2 row, so rolling back past it is unsupported." -m "A record that breaks a limit is dropped on read and logged with its source only, so one bad record never makes a row unreadable; only the write path rejects it." -m "KvPartnerId and KvEntry derive Default, and literals move to constructors. The admin lookup redacts provider state." ``` -### Task 7: Browser ingest records when it acquired an ID +### Task 7: Browser ingest records when it acquired an ID, and stores one UID **Files:** - Modify: `crates/trusted-server-core/src/ec/kv.rs` (`PartnerIdUpdate`, `apply_cookie_partner_id_updates`, `apply_partner_id_updates`) -- Modify: `crates/trusted-server-core/src/ec/prebid_eids.rs` (`collect_eid_updates`, `collect_sharedid_update`, `collect_prebid_eid_updates_from_eids`) -- Modify: every `PartnerIdUpdate::new` caller the compiler reports -- Test: `crates/trusted-server-core/src/ec/kv.rs` +- Modify: `crates/trusted-server-core/src/ec/prebid_eids.rs` (`collect_eid_updates`, `dedupe_partner_updates`) +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (both `collect_eid_updates` callers), and every other `PartnerIdUpdate::new` or `collect_eid_updates` caller the compiler reports +- Test: `crates/trusted-server-core/src/ec/kv.rs`, `crates/trusted-server-core/src/ec/prebid_eids.rs` **Interfaces:** -- Produces: `PartnerIdUpdate { partner_id, uid, acquired_at: Option }`, `PartnerIdUpdate::new(partner_id, uid)` (`acquired_at = None`), `PartnerIdUpdate::with_acquired_at(self, at: u64) -> Self`. A record that browser ingest inserts now carries `at`. +- Produces: `PartnerIdUpdate { partner_id, uid, acquired_at: Option }`, `PartnerIdUpdate::new(partner_id, uid)` (`acquired_at = None`), `PartnerIdUpdate::with_acquired_at(self, at: u64) -> Self`. `collect_eid_updates` gains a last parameter `now: Option`. A record that browser ingest inserts now carries `at`. -- [ ] **Step 1: Write the failing test** +What browser ingest stores does not change (G13): one UID per registered source, as one entry with one UID and no `atype`, `ext` or EID-level field, picked by today's rule, and nothing for an unregistered source, whatever extra UIDs or EID-level fields the browser sent. Today's rule is in the `collect_eid_updates` doc comment (`ec/prebid_eids.rs:146-151`): for a repeated source, the first valid UID of its last entry wins, and the sharedId cookie is applied last, so it wins over a Prebid EID for the same partner. The golden test pins that for `ts-eids` and the sharedId cookie, the only input browser ingest keeps once Task 26 stops `/auction` and `/_ts/page-bids` from ingesting their bodies (spec G10). Until then the body input still goes through `collect_eid_updates` unchanged. + +- [ ] **Step 1: Write the failing tests** + +In `kv.rs` tests: ```rust #[test] @@ -1493,10 +1881,55 @@ git commit --signoff -S -m "Add provider metadata and provider state to the KV s } ``` -- [ ] **Step 2: Run and see it fail** +In `prebid_eids.rs` tests, next to `collect_eid_updates_merges_prebid_and_sharedid_without_kv` (`make_registry` and `encode_json` already exist there). Add `use crate::ec::kv::apply_partner_id_updates;` and `use crate::ec::kv_types::KvEntry;` to the test module's imports if they are not there: + +```rust + #[test] + fn golden_browser_ingest_stores_one_uid_per_registered_source() { + let registry = make_registry(vec![("id5", "id5-sync.com"), ("sharedid", "sharedid.org")]); + let eids_cookie = encode_json(&json!([ + { "source": "id5-sync.com", "uids": [{ "id": "ID5_earlier_entry", "atype": 1 }] }, + { + "source": "id5-sync.com", + "uids": [ + { "id": " ", "atype": 1 }, + { "id": "ID5_first", "atype": 1, "ext": { "linkType": 2 } }, + { "id": "ID5_second", "atype": 1 } + ], + "inserter": "example.com", + "matcher": "partner.example.com", + "mm": 3, + "ext": { "segments": ["example-segment"] } + }, + { "source": "sharedid.org", "uids": [{ "id": "prebid-shared-id", "atype": 1 }] }, + { "source": "unregistered.example.com", "uids": [{ "id": "other", "atype": 1 }] } + ])); + let updates = + collect_eid_updates(Some(&eids_cookie), Some(" cookie-shared-id "), None, ®istry, Some(1_760_000_000)); + let mut entry = KvEntry::minimal("example.com", "existing", 1); + apply_partner_id_updates(&mut entry, &updates); + let stored = serde_json::to_value(&entry.ids).expect("should serialize ids"); + + assert_eq!( + stored, + json!({ + "example.com": { "eids": [{ "uids": [{ "id": "existing" }] }] }, + "id5-sync.com": { "at": 1_760_000_000, "eids": [{ "uids": [{ "id": "ID5_first" }] }] }, + "sharedid.org": { "at": 1_760_000_000, "eids": [{ "uids": [{ "id": "cookie-shared-id" }] }] } + }), + "should store one UID per registered source by today's rule, with no EID-level field, atype or ext" + ); + } +``` + +The expected values are today's rule, not a new one. Before Step 3, check them against the code as Task 6 leaves it: call the four-argument `collect_eid_updates`, drop every `"at"` from the expectation, and the assertion must pass. If it does not, the expectation is wrong, not the code: fix the expectation to what `main` stores and record the difference in the PR description. -Run: `cargo test-fastly -- ec::kv::tests::browser_ingest_records_acquisition_time` -Expected: compile error, because `with_acquired_at` does not exist. +Update the existing `collect_eid_updates(...)` calls in `prebid_eids.rs` tests to pass `None` as the new last argument. Their expectations do not change. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::kv::tests::browser_ingest_records_acquisition_time` and `cargo test-fastly -- ec::prebid_eids::tests::golden_browser_ingest` +Expected: compile errors, because `with_acquired_at` and the `now` parameter do not exist. - [ ] **Step 3: Implement** @@ -1526,18 +1959,31 @@ impl PartnerIdUpdate { In both `apply_*` functions, the de-duplicating map now keeps the whole update: `BTreeMap<&str, &PartnerIdUpdate>`. Insert `KvPartnerId::browser(update.uid.clone(), update.acquired_at)`. Equality checks still compare only `uid`, so an unchanged UID is not rewritten just to bump `at`. -In `prebid_eids.rs`, give `collect_eid_updates` a `now: Option` parameter. Call `.with_acquired_at(now)` on each update when it is `Some`, and pass `checked_current_timestamp()` from the callers in `finalize.rs`. Leave batch sync and pull sync without `acquired_at`. They get it through the mutation in Tasks 15 and 22. +In `prebid_eids.rs`, add the `now: Option` parameter last in `collect_eid_updates`. When it is `Some`, call `.with_acquired_at(now)` on each update before `dedupe_partner_updates`. `dedupe_partner_updates` rebuilds every update from its `partner_id` and `uid` today, which would drop `acquired_at`, so make it keep the whole update: -- [ ] **Step 4: Run and see it pass** +```rust +pub(crate) fn dedupe_partner_updates(updates: Vec) -> Vec { + let mut latest = std::collections::BTreeMap::new(); + for update in updates { + latest.insert(update.partner_id.clone(), update); + } + + latest.into_values().collect() +} +``` + +The later update for a partner still wins, and the output is still ordered by partner ID. Pass `checked_current_timestamp()` as `now` from both callers in `finalize.rs`. Leave batch sync and pull sync without `acquired_at`. They get it through the mutation in Tasks 15 and 22. + +- [ ] **Step 4: Run and see them pass** Run: `cargo test-fastly -- ec::` -Expected: PASS. +Expected: PASS, including every existing `prebid_eids` and `finalize` test unchanged apart from the extra `None` argument. - [ ] **Step 5: Commit** ```bash git add -A crates/trusted-server-core/src/ec -git commit --signoff -S -m "Record when browser ingest acquired each ID" +git commit --signoff -S -m "Record when browser ingest acquired each ID" -m "Ingest still stores one UID per registered source, by today's rule: the first valid UID of a repeated source's last entry, with the sharedId cookie applied last. A golden test pins that extra UIDs and EID-level fields from the browser are not stored." ``` ### Task 8: The provider descriptor @@ -1555,7 +2001,7 @@ git commit --signoff -S -m "Record when browser ingest acquired each ID" - `ProviderDescriptor` and its getters: `builder`, `id`, `sources`, `primary_source`, `source_named`, `resolves`, `endpoint`, `captures`, `consent_policy`, `provenance`, `is_expiring` - `ProviderDescriptorBuilder`: `with_source`, `with_resolves`, `with_captures`, `with_consent_policy`, `with_expiring_ids`, `with_expiring_source`, `with_provenance`, `build` - `Resolves`: `new`, `with_endpoint(&str)`, `with_host(&str)`, `with_needs`, `with_needs_for`, `with_wants`, `with_kinds`, `with_private_state`, plus `pub(in crate::ec::identity) fn with_ec_id(self)` and crate-visible getters `hosts()`, `needs(kind)`, `wants()`, `kinds()`, `has_private_state()`, `gets_ec_id()` - - `Captures`: `new`, `with_path(Method, &str)`, `with_path_prefix(&str)`, `with_request_consent(pointers)`, plus crate-visible `matches(&Method, &str) -> bool`, `paths()`, `prefixes()`, `consent_pointers()` + - `Captures`: `new`, `with_path(Method, &str)`, `with_path_prefix(&str)`, plus crate-visible `matches(&Method, &str) -> bool`, `paths()`, `prefixes()`. There is no body-consent rule: capture is gated by the request's own consent, from its cookies and headers (spec 5.5). - `EidProvenance`: `new`, `with_inserter`, `with_matcher`, `with_mm`, `inserter`, `matcher`, `mm` - `ConsentPolicy`: `strict`, `with_tcf_purposes`, `with_tcf_vendor`, `allowing_dnt`, `allowing_gpc`, `allowing_us_opt_out`, plus `pub(in crate::ec::identity) fn eid_persistence_only()` for generic pull and crate-visible getters - `UsOptOut { Sale, Sharing, TargetedAdvertising }` @@ -1616,7 +2062,7 @@ mod tests { assert_eq!( descriptor.consent_policy().tcf_purposes().iter().copied().collect::>(), vec![1, 4], - "strict should require Purposes 1 and 4" + "should require Purposes 1 and 4 under strict" ); } @@ -1642,10 +2088,43 @@ mod tests { .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) .with_expiring_source(&source("other.example.com")), ), - ]; - for (name, builder) in cases { - let error = builder.build().expect_err(name); - assert!( + ( + "blank provenance inserter", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance(&source("example.com"), EidProvenance::new().with_inserter(" ")), + ), + ( + "blank provenance matcher", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance(&source("example.com"), EidProvenance::new().with_matcher("")), + ), + ( + "oversized provenance matcher", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("example.com"), + EidProvenance::new().with_matcher(&"m".repeat(MAX_EID_FIELD_BYTES + 1)), + ), + ), + ( + "negative provenance mm", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance(&source("example.com"), EidProvenance::new().with_mm(-1)), + ), + ]; + for (name, builder) in cases { + let error = builder + .build() + .expect_err(&format!("should reject a descriptor with {name}")); + assert!( matches!(error.current_context(), IdentityProviderError::Config { .. }), "should be a Config error: {name}" ); @@ -1665,13 +2144,13 @@ mod tests { .build() .expect("should build"); - assert!(!descriptor.is_expiring(&a), "a should not expire"); - assert!(descriptor.is_expiring(&b), "b should expire"); - let provenance = descriptor.provenance(&a).expect("a should have provenance"); + assert!(!descriptor.is_expiring(&a), "should not mark a as expiring"); + assert!(descriptor.is_expiring(&b), "should mark b as expiring"); + let provenance = descriptor.provenance(&a).expect("should have provenance for a"); assert_eq!(provenance.inserter(), Some("example-inserter"), "should keep the inserter"); assert_eq!(provenance.matcher(), None, "should leave the matcher unset"); assert_eq!(provenance.mm(), Some(2), "should keep mm"); - assert!(descriptor.provenance(&b).is_none(), "b should have none"); + assert!(descriptor.provenance(&b).is_none(), "should have no provenance for b"); assert_eq!(descriptor.source_named("A.EXAMPLE.COM"), Some(&a), "should find a by any spelling"); } @@ -1694,8 +2173,11 @@ mod tests { .with_path_prefix("/integrations/p/stream/"); assert!(captures.matches(&Method::POST, "/integrations/p/api/token"), "should match the exact path"); assert!(!captures.matches(&Method::GET, "/integrations/p/api/token"), "should require the method"); - assert!(!captures.matches(&Method::POST, "/integrations/p/api/token/x"), "exact means exact"); - assert!(captures.matches(&Method::GET, "/integrations/p/stream/1"), "prefix matches any method"); + assert!( + !captures.matches(&Method::POST, "/integrations/p/api/token/x"), + "should not treat an exact path as a prefix" + ); + assert!(captures.matches(&Method::GET, "/integrations/p/stream/1"), "should match a prefix for any method"); assert!(!captures.matches(&Method::GET, "/integrations/p/settings"), "should not match others"); } @@ -1708,12 +2190,29 @@ mod tests { assert_eq!( resolves.needs(ResolveKind::Initial), [Input::ClientIp, Input::HashedEmail].into_iter().collect(), - "Initial needs both" + "should need both inputs for Initial" ); assert_eq!( resolves.needs(ResolveKind::Refresh), [Input::ClientIp].into_iter().collect(), - "Refresh needs only the IP" + "should need only the IP for Refresh" + ); + } + + #[test] + fn valid_provenance_builds() { + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("example.com"), + EidProvenance::new().with_inserter("example-inserter").with_matcher("example-matcher").with_mm(0), + ) + .build() + .expect("should accept non-blank provenance and mm 0"); + assert!( + descriptor.provenance(&source("example.com")).is_some(), + "should keep the provenance" ); } } @@ -1737,6 +2236,7 @@ use url::Url; use super::ids::{IdentityProviderId, SourceDomain}; use super::resolution::IdentityProviderError; +use crate::ec::kv_types::MAX_EID_FIELD_BYTES; use crate::ec::partner::normalize_partner_source_domain; /// What a provider is, may write, needs and must respect. @@ -1819,7 +2319,7 @@ impl ProviderDescriptor { &self.consent_policy } - /// EID-level fields for `source`, if declared. + /// Default EID-level fields for `source`, if declared. #[must_use] pub fn provenance(&self, source: &SourceDomain) -> Option<&EidProvenance> { self.provenance.get(source) @@ -1890,8 +2390,12 @@ impl ProviderDescriptorBuilder { self } - /// EID-level fields for a declared source, added to every server-side - /// value of that source this provider wrote. No KV field stores them. + /// Default EID-level fields for a declared source. At output, the + /// coordinator applies them to a stored entry of that source, written by + /// this provider, only when the entry sets none of `inserter`, `matcher` + /// and `mm`. An entry that sets any of them is the vendor's complete + /// statement, so vendor and config provenance never mix in one entry. + /// The defaults are not stored, so a config change applies at once. #[must_use] pub fn with_provenance(mut self, source: &SourceDomain, provenance: EidProvenance) -> Self { self.provenance.insert(source.clone(), provenance); @@ -1904,8 +2408,9 @@ impl ProviderDescriptorBuilder { /// /// Returns `Config` when the descriptor has no source, no capability, a /// resolver with no endpoint or host, an endpoint that is not an `https` - /// URL, a host that is not a bare hostname, or provenance or an expiring - /// mark for a source it does not declare. + /// URL, a host that is not a bare hostname, provenance or an expiring + /// mark for a source it does not declare, a provenance `inserter` or + /// `matcher` that is blank or over 255 bytes, or a provenance `mm` below 0. pub fn build(self) -> Result> { let id = self.id; let fail = |what: &str| IdentityProviderError::config(format!("provider `{id}`: {what}")); @@ -1921,6 +2426,21 @@ impl ProviderDescriptorBuilder { return Err(fail(&format!("names undeclared source `{source}`"))); } } + for (source, provenance) in &self.provenance { + let fields = [("inserter", provenance.inserter()), ("matcher", provenance.matcher())]; + for (name, value) in fields { + if let Some(value) = value + && (value.trim().is_empty() || value.len() > MAX_EID_FIELD_BYTES) + { + return Err(fail(&format!( + "provenance `{name}` for `{source}` is blank or over {MAX_EID_FIELD_BYTES} bytes" + ))); + } + } + if provenance.mm().is_some_and(|mm| mm < 0) { + return Err(fail(&format!("provenance `mm` for `{source}` is below 0"))); + } + } let mut endpoint = None; let mut resolves = self.resolves; @@ -2059,8 +2579,8 @@ impl Resolves { self } - /// This provider keeps private state, so it runs only on the writer - /// request, where state can be stored (spec 5.3). + /// This provider keeps private state in KV (`ps.st`, spec 6.1). It can + /// then read the stored state and return a state change. #[must_use] pub fn with_private_state(mut self) -> Self { self.private_state = true; @@ -2126,12 +2646,13 @@ impl Resolves { } } -/// Passive capture from responses TS proxies for this provider. +/// Revoke-only capture from responses TS proxies for this provider: it +/// can remove IDs, opt out or clear an opt-out, and never stores a positive +/// ID (spec 5.5). #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Captures { paths: Vec<(Method, String)>, prefixes: Vec, - consent_pointers: Vec<&'static str>, } impl Captures { @@ -2156,15 +2677,6 @@ impl Captures { self } - /// JSON pointers to consent strings in the proxied request's JSON body. - /// The coordinator extracts only these fields and gates capture by the - /// stricter of cookie and body consent. The provider never sees the body. - #[must_use] - pub fn with_request_consent(mut self, pointers: impl IntoIterator) -> Self { - self.consent_pointers.extend(pointers); - self - } - /// Whether a proxied request matches a rule. #[must_use] pub(crate) fn matches(&self, method: &Method, path: &str) -> bool { @@ -2183,15 +2695,12 @@ impl Captures { pub(crate) fn prefixes(&self) -> &[String] { &self.prefixes } - - /// Declared body consent pointers. - #[must_use] - pub(crate) fn consent_pointers(&self) -> &[&'static str] { - &self.consent_pointers - } } -/// EID-level fields (OpenRTB 2.6 §3.2.27) that bidders read. +/// Default EID-level fields (`OpenRTB` 2.6 §3.2.27) that bidders read, such +/// as who inserted an ID and how it was matched. They apply only to an entry +/// that sets none of the three. A provider whose vendor returns these per +/// response sets them on each `ResolvedEid` instead (spec 4.5). #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct EidProvenance { inserter: Option, @@ -2370,7 +2879,9 @@ impl ConsentPolicy { } ``` -`normalize_partner_source_domain` doubles as a hostname check: it accepts exactly a bare, lowercase-normalized hostname. Comparing its output to the input also rejects uppercase. Update `mod.rs`: +`normalize_partner_source_domain` doubles as a hostname check: it accepts exactly a bare, lowercase-normalized hostname. Comparing its output to the input also rejects uppercase. + +The descriptor only holds the provenance defaults. Task 28 applies them at output to entries the provider wrote, and only to an entry that sets none of `inserter`, `matcher` and `mm` (all or nothing, spec 4.3). Nothing writes them to KV. `build()` reuses Task 6's `MAX_EID_FIELD_BYTES` (255), the same cap the coordinator applies to a `ResolvedEid`'s fields. Update `mod.rs`: ```rust pub use descriptor::{ @@ -2396,21 +2907,22 @@ git commit --signoff -S -m "Add the identity provider descriptor and its builder **Files:** - Create: `crates/trusted-server-core/src/ec/identity/input.rs` -- Modify: `crates/trusted-server-core/src/ec/identity/mod.rs`, `migration_guards.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/descriptor.rs` (`Access::EcId`), `mod.rs`, `migration_guards.rs` - Test: `crates/trusted-server-core/src/ec/identity/input.rs` **Interfaces:** -- Consumes: `ProviderDescriptor`, `Resolves`, `Input`, `Access`, `ResolveKind` (Task 8), and `ConsentContext` (`consent/types.rs`) +- Consumes: `ProviderDescriptor`, `Resolves`, `Input`, `Access`, `ResolveKind` (Task 8), `KvPartnerId`, `KvEid` and `KvUid` (Task 6), `crate::openrtb::{Eid, Uid}`, and `ConsentContext` (`consent/types.rs`) - Produces: - `ProviderConsent` with `gdpr_applies`, `raw_tc_string`, `raw_gpp_string`, `gpp_section_ids`, `raw_us_privacy`, and `pub(crate) fn from_context(&ConsentContext) -> Self` - `Region` with `country`, `subdivision`, `is`, and `pub(crate) fn from_geo(country: &str, region: Option<&str>) -> Option`, which normalizes to upper case and strips an `XX-` prefix - - `HashedEmail` (32 bytes) with `as_bytes`, `to_hex`, `to_base64`, and a redacting `Debug` - - `StoredId` with `uid`, `atype`, `ext`, `writer`, `acquired_at`, `expires_at`, `refresh_at`, `is_expired`, and `pub(crate) fn from_record(&KvPartnerId) -> Self` - - `ResolveSignals<'a>`: a crate-private struct holding every signal the coordinator has (IP, UA, page URL, region, HEM, consent, publisher domain, now, random seed source, EC ID, stored IDs, raw state JSON, dry-run flag) - - `ResolveInput<'a>` with the spec 4.4 accessors, `pub(crate) fn new(descriptor: &'a ProviderDescriptor, kinds: BTreeMap, signals: &'a ResolveSignals<'a>) -> Self`, `pub(crate) fn recorded_errors(&self) -> Vec`, `pub(crate) fn state_decode_failed(&self) -> bool`, and `pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report>` - - `CaptureInput<'a>` with the spec 4.4 accessors and `pub(crate) fn new(...)` - - `RandomSource`: a trait `fn fill(&self, bytes: &mut [u8])`, with `ThreadRandom` for production and `SeededRandom` (a ChaCha-free xorshift from a `u64` seed, test support only) + - `HashedEmail` (32 bytes) with `as_bytes`, `to_hex`, `to_base64`, a redacting `Debug`, and `pub fn from_bytes([u8; 32]) -> Self` behind `#[cfg(any(test, feature = "test-utils"))]`. It is the only constructor in v1, and Task 39's harness reuses it instead of adding a second one + - `StoredId` with `eids`, `uid`, `atype`, `ext`, `writer`, `acquired_at`, `expires_at`, `refresh_at`, `is_expired`, and `pub(crate) fn from_record(source: &SourceDomain, record: &KvPartnerId) -> Option`, `None` for a record with an empty `eids`. `eids()` is the record's whole entry set (spec 4.4), one `Eid` per stored entry with `source` filled in; a record that browser ingest, generic pull or batch sync wrote is one entry with one UID. `uid`, `atype` and `ext` describe the first UID of the first entry, so a single-UID provider such as LiveRamp reads what it reads today + - `ResolveSignals<'a>`: a crate-private struct holding every signal the coordinator has (IP, UA, page URL, region, HEM, consent, publisher domain, now, random seed source, EC ID, stored IDs, raw state JSON). There is no dry-run flag: phase 4 calls `request` once (spec 5.2). + - `ResolveInput<'a>` with the spec 4.4 accessors, `pub(crate) fn new(descriptor: &'a ProviderDescriptor, kinds: BTreeMap, signals: &'a ResolveSignals<'a>) -> Self`, `pub(crate) fn recorded_errors(&self) -> Vec`, `pub(crate) fn state_decode_failed(&self) -> bool`, and `pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report>`. Only the inputs of the kinds in `kinds` are available (spec 4.4): an input the descriptor declares only for a kind this call does not hold reads as `InputUnavailable`, not as an undeclared read + - `Access::EcId`, a new variant in `descriptor.rs` (`Access` is `#[non_exhaustive]`): an EC ID read by a resolver not declared `with_ec_id()` + - `RandomSource`: a crate-private trait `fn fill(&self, bytes: &mut [u8])`, with `ThreadRandom` for production and `pub struct SeededRandom` (an xorshift generator from a `u64` seed, `pub fn new(seed: u64)`) behind `#[cfg(any(test, feature = "test-utils"))]`. Test-support items are `pub` under that gate, as the crate's existing `test-utils` items are, so the non-test library build with `--all-features` has no dead code. Task 14's `test_support` module re-exports `SeededRandom` + - `CaptureInput` moves to Task 10, which creates the `ProviderResponse` it holds - [ ] **Step 1: Write the failing tests** @@ -2421,11 +2933,12 @@ mod tests { use std::net::IpAddr; use super::*; - use crate::ec::identity::descriptor::{Captures, ProviderDescriptor, Resolves}; + use crate::ec::identity::descriptor::{ProviderDescriptor, Resolves}; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; fn descriptor(resolves: Resolves) -> ProviderDescriptor { - ProviderDescriptor::builder("p".parse().expect("should parse")) - .with_source("example.com".parse().expect("should parse")) + ProviderDescriptor::builder("p".parse().expect("should parse the provider id")) + .with_source("example.com".parse().expect("should parse the source")) .with_resolves(resolves) .build() .expect("should build") @@ -2445,7 +2958,6 @@ mod tests { ec_id: Some("a".repeat(64) + ".ABC123"), stored: BTreeMap::new(), state_json: None, - dry_run: false, } } @@ -2460,16 +2972,46 @@ mod tests { let signals = signals(); let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Initial), &signals); - assert_eq!(input.client_ip().expect("should read ip"), "192.0.2.10".parse::().expect("ip"), "ip"); + assert_eq!( + input.client_ip().expect("should read the ip"), + "192.0.2.10".parse::().expect("should parse the ip"), + "should return the client ip" + ); let region = input.region().expect("should read region"); assert_eq!((region.country(), region.subdivision()), ("US", Some("CA")), "should normalize the region"); assert!(region.is("US", "CA"), "should test country and subdivision"); let error = input.page_url().expect_err("should be unavailable"); assert!( matches!(error.current_context(), IdentityProviderError::InputUnavailable { input: Input::PageUrl }), - "a declared but missing input is InputUnavailable" + "should report a declared but missing input as InputUnavailable" + ); + assert!(input.recorded_errors().is_empty(), "should record nothing for declared reads"); + } + + #[test] + fn input_declared_only_for_an_absent_kind_is_unavailable() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_needs([Input::ClientIp]) + .with_needs_for(ResolveKind::Refresh, [Input::UserAgent]), + ); + let signals = signals(); + + let initial = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Initial), &signals); + let refresh = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &signals); + + let error = initial.user_agent().expect_err("should hide a Refresh-only input on an Initial call"); + assert!( + matches!(error.current_context(), IdentityProviderError::InputUnavailable { input: Input::UserAgent }), + "should report InputUnavailable, not an undeclared read" + ); + assert!(initial.recorded_errors().is_empty(), "should record no undeclared read"); + assert_eq!( + refresh.user_agent().expect("should read the user agent on a Refresh call"), + "ExampleBrowser/1.0", + "should return the user agent for the kind that declares it" ); - assert!(input.recorded_errors().is_empty(), "declared reads record nothing"); } #[test] @@ -2482,13 +3024,13 @@ mod tests { let state = input.state::(); assert!(ignored.is_none(), "should not return an undeclared input"); - assert!(state.is_err(), "state needs with_private_state"); + assert!(state.is_err(), "should need with_private_state for state"); assert_eq!( input.recorded_errors(), vec![Access::Input(Input::UserAgent), Access::PrivateState], "should record both undeclared reads" ); - input.endpoint().expect("the declared endpoint is always available"); + input.endpoint().expect("should always return the declared endpoint"); } #[test] @@ -2497,33 +3039,24 @@ mod tests { let mut good = signals(); good.state_json = Some(r#"{"signature":"s"}"#.to_owned()); let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &good); - let state: serde_json::Value = input.state().expect("should decode").expect("should be present"); + let state: serde_json::Value = input + .state() + .expect("should decode the state") + .expect("should have a state"); assert_eq!(state["signature"], "s", "should decode the state"); - assert!(!input.state_decode_failed(), "a good state is not a failure"); + assert!(!input.state_decode_failed(), "should not record a good state as a failure"); - #[derive(serde::Deserialize)] - struct Typed { - #[allow(dead_code)] - signature: String, - } let mut bad = signals(); bad.state_json = Some(r#""a bare string""#.to_owned()); let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &bad); - let error = input.state::().expect_err("should not decode"); - assert!(matches!(error.current_context(), IdentityProviderError::InvalidState), "InvalidState"); + let error = input.state::().expect_err("should not decode a bare string"); + assert!( + matches!(error.current_context(), IdentityProviderError::InvalidState), + "should report InvalidState" + ); assert!(input.state_decode_failed(), "should record the failure"); } - #[test] - fn dry_run_is_never_true_for_stateful_providers() { - let stateful = descriptor(Resolves::new().with_endpoint("https://a.example.com").with_private_state()); - let stateless = descriptor(Resolves::new().with_endpoint("https://a.example.com")); - let mut dry = signals(); - dry.dry_run = true; - assert!(!ResolveInput::new(&stateful, kinds(&stateful, ResolveKind::Initial), &dry).is_dry_run(), "stateful"); - assert!(ResolveInput::new(&stateless, kinds(&stateless, ResolveKind::Initial), &dry).is_dry_run(), "stateless"); - } - #[test] fn ec_id_is_only_for_providers_declared_with_ec_id() { let generic = descriptor(Resolves::new().with_host("a.example.com").with_ec_id()); @@ -2531,36 +3064,124 @@ mod tests { let signals = signals(); ResolveInput::new(&generic, kinds(&generic, ResolveKind::Initial), &signals) .ec_id() - .expect("generic pull reads the EC ID"); + .expect("should let generic pull read the EC ID"); let input = ResolveInput::new(&other, kinds(&other, ResolveKind::Initial), &signals); - input.ec_id().expect_err("others may not"); - assert_eq!(input.recorded_errors().len(), 1, "should record the read"); + input.ec_id().expect_err("should refuse the EC ID to other providers"); + assert_eq!(input.recorded_errors(), vec![Access::EcId], "should record an EC ID read"); + + let mut no_ec = signals(); + no_ec.ec_id = None; + let error = ResolveInput::new(&generic, kinds(&generic, ResolveKind::Initial), &no_ec) + .ec_id() + .expect_err("should fail without an EC ID"); + assert!( + matches!(error.current_context(), IdentityProviderError::Config { .. }), + "should report a missing EC ID as Config, not as a missing input" + ); } #[test] fn stored_ids_and_expiry() { - let record = crate::ec::kv_types::KvPartnerId { - uid: "u".to_owned(), + let record = KvPartnerId { w: Some("p".to_owned()), rv: Some("0123456789abcdef".to_owned()), at: Some(10), exp: Some(20), ra: Some(15), ru: None, - atype: Some(3), - ext: None, + eids: vec![KvEid { + uids: vec![KvUid { id: "u".to_owned(), atype: Some(3), ext: None }], + inserter: None, + matcher: None, + mm: None, + ext: None, + }], + }; + let source: SourceDomain = "example.com".parse().expect("should parse the source"); + let stored = StoredId::from_record(&source, &record).expect("should read a record with an entry"); + assert_eq!(stored.writer().map(IdentityProviderId::as_str), Some("p"), "should parse the writer"); + assert!(!stored.is_expired(UnixTime::from_secs(19)), "should not be expired before exp"); + assert!(stored.is_expired(UnixTime::from_secs(20)), "should be expired at exp"); + assert_eq!(stored.refresh_at(), Some(UnixTime::from_secs(15)), "should read ra"); + assert_eq!(stored.eids().len(), 1, "should expose a one-UID record as one entry"); + let eid = &stored.eids()[0]; + assert_eq!(eid.source, "example.com", "should fill in the source"); + assert_eq!(eid.uids.len(), 1, "should expose its one UID"); + assert_eq!(eid.uids[0].id, "u", "should keep the UID"); + assert_eq!(eid.uids[0].atype, Some(3), "should keep the atype"); + assert_eq!( + (eid.inserter.as_deref(), eid.matcher.as_deref(), eid.mm, eid.ext.as_ref()), + (None, None, None, None), + "should expose no EID-level fields when none are stored" + ); + assert_eq!((stored.uid(), stored.atype()), ("u", Some(3)), "should describe the first UID"); + + let empty = KvPartnerId { eids: Vec::new(), ..record }; + assert!(StoredId::from_record(&source, &empty).is_none(), "should not read a record with no entry"); + } + + #[test] + fn a_full_record_exposes_every_entry_and_its_first_uid() { + let record = KvPartnerId { + w: Some("p".to_owned()), + rv: Some("0123456789abcdef".to_owned()), + at: Some(10), + exp: None, + ra: None, + ru: None, + eids: vec![ + KvEid { + uids: vec![ + KvUid { id: "first".to_owned(), atype: Some(1), ext: Some(serde_json::json!({ "rtiPartner": "TDID" })) }, + KvUid { id: "second".to_owned(), atype: Some(3), ext: None }, + ], + inserter: Some("inserter.example.com".to_owned()), + matcher: None, + mm: Some(2), + ext: Some(serde_json::json!({ "segments": ["a"] })), + }, + KvEid { + uids: vec![KvUid { id: "third".to_owned(), atype: None, ext: None }], + inserter: None, + matcher: Some("matcher.example.com".to_owned()), + mm: None, + ext: None, + }, + ], }; - let stored = StoredId::from_record(&record); - assert_eq!(stored.writer().map(IdentityProviderId::as_str), Some("p"), "writer"); - assert!(!stored.is_expired(UnixTime::from_secs(19)), "not expired before exp"); - assert!(stored.is_expired(UnixTime::from_secs(20)), "expired at exp"); - assert_eq!(stored.refresh_at(), Some(UnixTime::from_secs(15)), "ra"); + let source: SourceDomain = "example.com".parse().expect("should parse the source"); + + let stored = StoredId::from_record(&source, &record).expect("should read the record"); + + let ids: Vec> = stored + .eids() + .iter() + .map(|eid| eid.uids.iter().map(|uid| uid.id.as_str()).collect()) + .collect(); + assert_eq!(ids, vec![vec!["first", "second"], vec!["third"]], "should keep every entry in stored order"); + assert!(stored.eids().iter().all(|eid| eid.source == "example.com"), "should fill in the source"); + assert_eq!(stored.eids()[0].inserter.as_deref(), Some("inserter.example.com"), "should keep the inserter"); + assert_eq!(stored.eids()[0].mm, Some(2), "should keep mm"); + assert_eq!( + stored.eids()[0].ext, + Some(serde_json::json!({ "segments": ["a"] })), + "should keep the EID-level ext" + ); + assert_eq!(stored.eids()[1].matcher.as_deref(), Some("matcher.example.com"), "should keep the matcher"); + assert_eq!(stored.uid(), "first", "should return the first UID of the first entry"); + assert_eq!(stored.atype(), Some(1), "should return that UID's atype"); + assert_eq!( + stored.ext(), + Some(&serde_json::json!({ "rtiPartner": "TDID" })), + "should return that UID's ext" + ); + assert!(!format!("{stored:?}").contains("second"), "should redact every UID in Debug"); } #[test] fn hashed_email_debug_is_redacted() { let email = HashedEmail::from_bytes([0xab; 32]); - assert_eq!(email.to_hex(), "ab".repeat(32), "hex"); + assert_eq!(email.to_hex(), "ab".repeat(32), "should encode lowercase hex"); assert_eq!(format!("{email:?}"), "HashedEmail([redacted])", "should redact Debug"); } @@ -2568,32 +3189,14 @@ mod tests { descriptor.sources().iter().map(|source| (source.clone(), kind)).collect() } - #[test] - fn capture_input_exposes_path_and_never_the_request_body() { - let descriptor = ProviderDescriptor::builder("c".parse().expect("id")) - .with_source("example.com".parse().expect("source")) - .with_captures(Captures::new().with_path(http::Method::POST, "/integrations/c/api")) - .build() - .expect("should build"); - let response = crate::ec::identity::exchange::ProviderResponse::for_test(http::StatusCode::OK, b"{}"); - let consent = ProviderConsent::default(); - let input = CaptureInput::new( - &descriptor, - "/integrations/c/api", - &response, - &BTreeMap::new(), - &consent, - "www.example.com", - UnixTime::from_secs(1), - ); - assert_eq!(input.path(), "/integrations/c/api", "path"); - assert_eq!(input.primary_source().as_str(), "example.com", "source"); + #[derive(serde::Deserialize)] + struct TypedState { + #[allow(dead_code)] + signature: String, } } ``` -`ProviderResponse::for_test` arrives in Task 10. Until then, comment out the last test and enable it in Task 10, Step 4. - - [ ] **Step 2: Run and see them fail** Run: `cargo test-fastly -- ec::identity::input` @@ -2617,12 +3220,11 @@ use serde::de::DeserializeOwned; use url::Url; use super::descriptor::{Access, Input, ProviderDescriptor, ResolveKind}; -use super::exchange::ProviderResponse; use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; use super::resolution::IdentityProviderError; use crate::consent::ConsentContext; use crate::ec::kv_types::KvPartnerId; - +use crate::openrtb::{Eid, Uid}; /// The raw consent strings a provider forwards, and `gdpr_applies`. /// @@ -2755,9 +3357,11 @@ impl fmt::Debug for HashedEmail { } impl HashedEmail { - /// From digest bytes. No code in v1 calls this outside tests. + /// From digest bytes. Test support only: v1 has no source of hashed + /// emails, so nothing else builds one. + #[cfg(any(test, feature = "test-utils"))] #[must_use] - pub(crate) fn from_bytes(bytes: [u8; 32]) -> Self { + pub fn from_bytes(bytes: [u8; 32]) -> Self { Self(bytes) } @@ -2767,13 +3371,13 @@ impl HashedEmail { &self.0 } - /// Lowercase hex, as LiveRamp takes it. + /// Lowercase hex, as `LiveRamp` takes it. #[must_use] pub fn to_hex(&self) -> String { hex::encode(self.0) } - /// Standard base64, as UID2 takes it. + /// Standard base64, as `UID2` takes it. #[must_use] pub fn to_base64(&self) -> String { STANDARD.encode(self.0) @@ -2781,11 +3385,12 @@ impl HashedEmail { } /// A stored record for one of this provider's sources, valid or expired. +/// +/// It holds the source's whole entry set, which is never empty. #[derive(Clone, PartialEq)] pub struct StoredId { + eids: Vec, uid: String, - atype: Option, - ext: Option, writer: Option, acquired_at: Option, expires_at: Option, @@ -2796,6 +3401,7 @@ impl fmt::Debug for StoredId { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter .debug_struct("StoredId") + .field("entries", &self.eids.len()) .field("uid", &"[redacted]") .field("writer", &self.writer) .field("acquired_at", &self.acquired_at) @@ -2806,36 +3412,68 @@ impl fmt::Debug for StoredId { } impl StoredId { - /// From a KV record. A writer that does not parse is treated as none. - #[must_use] - pub(crate) fn from_record(record: &KvPartnerId) -> Self { - Self { - uid: record.uid.clone(), - atype: record.atype, - ext: record.ext.clone(), + /// From the KV record of `source`. A writer that does not parse is + /// treated as none. `None` for a record with no entry, which + /// `KvEntry::validate` never lets through. + #[must_use] + pub(crate) fn from_record(source: &SourceDomain, record: &KvPartnerId) -> Option { + let uid = record.first_uid()?.to_owned(); + let eids = record + .eids + .iter() + .map(|entry| Eid { + source: source.as_str().to_owned(), + uids: entry + .uids + .iter() + .map(|uid| Uid { id: uid.id.clone(), atype: uid.atype, ext: uid.ext.clone() }) + .collect(), + inserter: entry.inserter.clone(), + matcher: entry.matcher.clone(), + mm: entry.mm, + ext: entry.ext.clone(), + }) + .collect(); + Some(Self { + eids, + uid, writer: record.w.as_deref().and_then(|writer| writer.parse().ok()), acquired_at: record.at.map(UnixTime::from_secs), expires_at: record.exp.map(UnixTime::from_secs), refresh_at: record.ra.map(UnixTime::from_secs), - } + }) } - /// The UID. + fn first_uid(&self) -> Option<&Uid> { + self.eids.first().and_then(|eid| eid.uids.first()) + } + + /// Every stored entry for the source, in stored order, with its UIDs and + /// EID-level fields. A record that browser ingest, generic pull, batch + /// sync or a version 1 row wrote is one entry with one UID. + #[must_use] + pub fn eids(&self) -> &[Eid] { + &self.eids + } + + /// The first UID of the first entry. What a single-UID refresh, such as + /// `LiveRamp`'s, sends back. #[must_use] pub fn uid(&self) -> &str { &self.uid } - /// Provider-supplied `atype`. + /// The `atype` of that first UID. #[must_use] pub fn atype(&self) -> Option { - self.atype + self.first_uid().and_then(|uid| uid.atype) } - /// UID-level ext, as bidders read it in `uids[].ext`. + /// The UID-level ext of that first UID, as bidders read it in + /// `uids[].ext`. #[must_use] pub fn ext(&self) -> Option<&serde_json::Value> { - self.ext.as_ref() + self.first_uid().and_then(|uid| uid.ext.as_ref()) } /// The provider that wrote it. `None` for browser ingest and legacy rows. @@ -2886,11 +3524,15 @@ impl RandomSource for ThreadRandom { /// Deterministic bytes for tests: xorshift64* from a seed. #[cfg(any(test, feature = "test-utils"))] -pub(crate) struct SeededRandom(std::sync::Mutex); +#[derive(Debug)] +pub struct SeededRandom(std::sync::Mutex); #[cfg(any(test, feature = "test-utils"))] impl SeededRandom { - pub(crate) fn new(seed: u64) -> Self { + /// A generator that yields the same bytes for the same seed. A zero + /// seed is treated as one, since xorshift never leaves zero. + #[must_use] + pub fn new(seed: u64) -> Self { Self(std::sync::Mutex::new(seed.max(1))) } } @@ -2903,7 +3545,7 @@ impl RandomSource for SeededRandom { *state ^= *state >> 12; *state ^= *state << 25; *state ^= *state >> 27; - *byte = (state.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 56) as u8; + *byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D).to_be_bytes()[0]; } } } @@ -2923,7 +3565,6 @@ pub(crate) struct ResolveSignals<'a> { pub(crate) ec_id: Option, pub(crate) stored: BTreeMap, pub(crate) state_json: Option, - pub(crate) dry_run: bool, } /// What one provider may read for one call. @@ -2931,6 +3572,7 @@ pub struct ResolveInput<'a> { descriptor: &'a ProviderDescriptor, kinds: BTreeMap, signals: &'a ResolveSignals<'a>, + declared: BTreeSet, available: BTreeSet, undeclared: RefCell>, state_failed: RefCell, @@ -2948,20 +3590,28 @@ impl fmt::Debug for ResolveInput<'_> { impl<'a> ResolveInput<'a> { /// The input for one call. `kinds` holds only the sources to resolve. + /// The inputs of every kind present are available, plus the wanted ones. + /// An input declared only for a kind this call does not hold is + /// declared, so reading it is not an error of the provider, but it is + /// unavailable (spec 4.4). #[must_use] pub(crate) fn new( descriptor: &'a ProviderDescriptor, kinds: BTreeMap, signals: &'a ResolveSignals<'a>, ) -> Self { - let available = descriptor - .resolves() - .map(|resolves| resolves.declared_inputs()) - .unwrap_or_default(); + let (declared, available) = descriptor.resolves().map_or_else(Default::default, |resolves| { + let mut available = resolves.wants().clone(); + for kind in kinds.values().copied().collect::>() { + available.extend(resolves.needs(kind)); + } + (resolves.declared_inputs(), available) + }); Self { descriptor, kinds, signals, + declared, available, undeclared: RefCell::new(Vec::new()), state_failed: RefCell::new(false), @@ -2974,9 +3624,12 @@ impl<'a> ResolveInput<'a> { } fn declared(&self, input: Input, value: Option) -> Result> { - if !self.available.contains(&input) { + if !self.declared.contains(&input) { return Err(self.record(Access::Input(input))); } + if !self.available.contains(&input) { + return Err(Report::new(IdentityProviderError::InputUnavailable { input })); + } value.ok_or_else(|| Report::new(IdentityProviderError::InputUnavailable { input })) } @@ -3042,14 +3695,6 @@ impl<'a> ResolveInput<'a> { }) } - /// True during the dry run (spec 5.2). Always false for a provider - /// declared `with_private_state()`. - #[must_use] - pub fn is_dry_run(&self) -> bool { - self.signals.dry_run - && !self.descriptor.resolves().is_some_and(|resolves| resolves.has_private_state()) - } - /// The endpoint declared with `Resolves::with_endpoint`. /// /// # Errors @@ -3134,110 +3779,36 @@ impl<'a> ResolveInput<'a> { /// /// # Errors /// - /// `UndeclaredInput` otherwise. `InputUnavailable` (as `ClientIp`, the - /// nearest input) when the request has no EC ID. + /// `UndeclaredInput { access: EcId }` otherwise. `Config` when the + /// request has no EC ID, which the coordinator never allows: it does not + /// dispatch generic pull without one. pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report> { if !self.descriptor.resolves().is_some_and(|resolves| resolves.gets_ec_id()) { - return Err(self.record(Access::PrivateState)); + return Err(self.record(Access::EcId)); } self.signals .ec_id .as_deref() - .ok_or_else(|| Report::new(IdentityProviderError::InputUnavailable { input: Input::ClientIp })) - } -} - -/// What a capture may read from one proxied response. -pub struct CaptureInput<'a> { - descriptor: &'a ProviderDescriptor, - path: &'a str, - response: &'a ProviderResponse, - stored: &'a BTreeMap, - consent: &'a ProviderConsent, - publisher_domain: &'a str, - now: UnixTime, -} - -impl fmt::Debug for CaptureInput<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("CaptureInput") - .field("provider", self.descriptor.id()) - .field("path", &self.path) - .finish_non_exhaustive() + .ok_or_else(|| IdentityProviderError::config("generic pull was called without an EC ID")) } } +``` -impl<'a> CaptureInput<'a> { - /// The input for one capture. - #[must_use] - pub(crate) fn new( - descriptor: &'a ProviderDescriptor, - path: &'a str, - response: &'a ProviderResponse, - stored: &'a BTreeMap, - consent: &'a ProviderConsent, - publisher_domain: &'a str, - now: UnixTime, - ) -> Self { - Self { descriptor, path, response, stored, consent, publisher_domain, now } - } - - /// The proxied request path, without query string. - #[must_use] - pub fn path(&self) -> &str { - self.path - } - - /// The proxied response, decoded. - #[must_use] - pub fn response(&self) -> &ProviderResponse { - self.response - } - - /// The descriptor's primary source. - #[must_use] - pub fn primary_source(&self) -> &SourceDomain { - self.descriptor.primary_source() - } - - /// Finds a declared source by a vendor's spelling. - #[must_use] - pub fn source_named(&self, name: &str) -> Option<&SourceDomain> { - self.descriptor.source_named(name) - } - - /// The stored record for a source. - #[must_use] - pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId> { - self.stored.get(source) - } - - /// The stricter of the cookie consent and the request-body consent. - #[must_use] - pub fn consent(&self) -> &ProviderConsent { - self.consent - } +`StoredId::uid` returns the first UID of the first entry, the value `KvPartnerId::first_uid` (Task 6) reads, so `uid()` and `eids()[0].uids[0].id` always agree. The coordinator builds the `stored` map with `StoredId::from_record(source, record)` for each of the provider's sources in the row, skipping a `None`. - /// The publisher's domain. - #[must_use] - pub fn publisher_domain(&self) -> &str { - self.publisher_domain - } +`hex` is already a dependency of core (check with `grep -n '^hex' crates/trusted-server-core/Cargo.toml`). If it is only a build dependency, add `hex = { workspace = true }` to `[dependencies]`. `ResolveKind` stays in `descriptor.rs` and is re-exported from `mod.rs`. Update `mod.rs`: - /// The request's time. - #[must_use] - pub fn now(&self) -> UnixTime { - self.now - } -} +```rust +pub mod input; +pub use input::{HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; ``` -`CaptureInput::new` takes 7 arguments, which is the maximum `AGENTS.md` allows. Do not add an eighth. Use a struct if you need one. `hex` is already a dependency of core (check with `grep -n '^hex' crates/trusted-server-core/Cargo.toml`). If it is only a build dependency, add `hex = { workspace = true }` to `[dependencies]`. `ResolveKind` stays in `descriptor.rs` and is re-exported from `mod.rs`. Update `mod.rs`: +In `descriptor.rs`, add the variant to `Access` (Task 5), after `PrivateState`: ```rust -pub mod input; -pub use input::{CaptureInput, HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; + /// The full EC ID, without `with_ec_id()`. Only generic pull declares it. + #[display("ec_id")] + EcId, ``` Add `ec/identity/input.rs` to `migration_guards.rs`. @@ -3245,7 +3816,7 @@ Add `ec/identity/input.rs` to `migration_guards.rs`. - [ ] **Step 4: Run and see them pass** Run: `cargo test-fastly -- ec::identity::input` -Expected: PASS for every test except the `capture_input_...` test, which waits for Task 10. +Expected: PASS. - [ ] **Step 5: Commit** @@ -3259,19 +3830,25 @@ git commit --signoff -S -m "Add declared-input enforcement for identity provider **Files:** - Create: `crates/trusted-server-core/src/ec/identity/exchange.rs` -- Modify: `crates/trusted-server-core/src/ec/identity/resolution.rs` (add `Resolution`, `ResolvedId`, `When`, `StateChange`) +- Modify: `crates/trusted-server-core/src/ec/identity/resolution.rs` (add `Resolution`, `ResolvedEid`, `ResolvedUid`, `When`, `StateChange`, and the crate-private entry-set check) +- Modify: `crates/trusted-server-core/src/ec/identity/input.rs` (add `CaptureInput`, moved here from Task 9 because it holds a `ProviderResponse`) - Modify: `mod.rs`, `migration_guards.rs` -- Test: both files +- Test: all three files **Interfaces:** +- Consumes: `KvEid`, `KvUid`, `entry_set_bytes(&[KvEid]) -> usize` and the limits `MAX_UID_LENGTH`, `MAX_PARTNER_EXT_BYTES` (2,048), `MAX_EID_FIELD_BYTES` (255), `MAX_ENTRIES_PER_SOURCE` (8), `MAX_UIDS_PER_SOURCE` (32), `MAX_ENTRY_SET_BYTES` (4,096) and `MAX_PROVIDER_STATE_BYTES` (1,024) from `ec/kv_types.rs` (Task 6). `entry_set_bytes` is the one measure of a set: the coordinator and `KvEntry::validate` both call it, so a set one accepts the other never rejects - Produces: everything listed in spec 4.5, plus: - `ProviderRequest::call_context_json(&self) -> Option<&str>` (crate-private) - `ProviderRequest::has_cookie_header(&self) -> bool` (crate-private) - `ProviderResponse::new(status: StatusCode, headers: HeaderMap, body: Vec, call_context: Option) -> Self` (crate-private) - `ProviderResponse::for_test(status, body: &[u8]) -> Self` (test support) - - `Resolution::found_ids_by_source(&self)`: the first ID per source, plus the count of duplicates dropped (crate-private) - - `MAX_CALL_CONTEXT_BYTES = 1024`, `MAX_UID_EXT_BYTES = 256` + - `pub(crate) fn entry_sets(eids: &[ResolvedEid], now: UnixTime) -> EntrySets`: groups a result's entries by source, in order, applies every limit of spec 4.5 ("From entries to an entry set") and resolves each set's timing to the earliest its entries set + - `pub(crate) struct EntrySet { entries: Vec, expires_at, refresh_at, refresh_until: Option }`, `pub(crate) struct EntrySets { sets: BTreeMap, drops: Vec, oversized: Vec }`, `pub(crate) struct EntryDrop { source, part: DroppedPart }` and `pub(crate) enum DroppedPart`, all with `pub(crate)` fields + - `EntrySets::all_discarded(&self) -> bool`: the result named at least one source and every set was discarded. Task 20 handles such a result as `Malformed` (spec 4.5, 8): `f += 1` with its backoff, removals still applied. A source whose set is discarded is treated as not named, so its stored set stays as it is + - `CaptureInput<'a>` (in `input.rs`) with the spec 4.4 accessors and `pub(crate) fn new(...)` + - `Resolution` and its private `StateOp` have a `Debug` that never prints provider state (spec 9.3) + - `MAX_CALL_CONTEXT_BYTES = 1024` - [ ] **Step 1: Write the failing tests** @@ -3291,10 +3868,10 @@ mod tests { #[test] fn post_json_sets_body_and_content_type() { - let request = ProviderRequest::post_json(&url(), &json!({ "a": 1 })).expect("should build"); - assert_eq!(request.method(), Method::POST, "method"); - assert_eq!(request.headers()["content-type"], "application/json", "content type"); - assert_eq!(request.body(), br#"{"a":1}"#, "body"); + let request = ProviderRequest::post_json(&url(), &json!({ "a": 1 })).expect("should build a JSON request"); + assert_eq!(request.method(), Method::POST, "should use POST"); + assert_eq!(request.headers()["content-type"], "application/json", "should set the JSON content type"); + assert_eq!(request.body(), br#"{"a":1}"#, "should serialize the body"); } #[test] @@ -3305,8 +3882,8 @@ mod tests { .try_with_header("x-example", "v") .expect("should accept a valid header"); assert_eq!(request.url().as_str(), "https://api.example.com/v1/fetch?pid=1+2", "should encode the query"); - assert_eq!(request.headers()["authorization"], "Bearer token", "bearer"); - assert_eq!(request.headers()["x-example"], "v", "header"); + assert_eq!(request.headers()["authorization"], "Bearer token", "should set the bearer token"); + assert_eq!(request.headers()["x-example"], "v", "should set the header"); ProviderRequest::get(&url()).try_with_header("bad header", "v").expect_err("should reject a bad name"); ProviderRequest::get(&url()).try_with_header("x", "bad\nvalue").expect_err("should reject a bad value"); } @@ -3315,47 +3892,64 @@ mod tests { fn cookie_headers_are_flagged_from_either_helper() { let typed = ProviderRequest::get(&url()) .with_header(HeaderName::from_static("cookie"), HeaderValue::from_static("a=1")); - let dynamic = ProviderRequest::get(&url()).try_with_header("Cookie", "a=1").expect("should build"); - assert!(typed.has_cookie_header(), "typed helper"); - assert!(dynamic.has_cookie_header(), "dynamic helper"); - assert!(!ProviderRequest::get(&url()).has_cookie_header(), "control"); + let dynamic = ProviderRequest::get(&url()) + .try_with_header("Cookie", "a=1") + .expect("should accept a Cookie header from strings"); + assert!(typed.has_cookie_header(), "should flag a Cookie header from the typed helper"); + assert!(dynamic.has_cookie_header(), "should flag a Cookie header from the string helper"); + assert!(!ProviderRequest::get(&url()).has_cookie_header(), "should not flag a request without one"); } #[test] fn call_context_round_trips_and_is_capped() { - let request = ProviderRequest::get(&url()).try_with_call_context(&json!({ "nonce": [1, 2] })).expect("ok"); + let request = ProviderRequest::get(&url()) + .try_with_call_context(&json!({ "nonce": [1, 2] })) + .expect("should accept a small call context"); let response = ProviderResponse::new( StatusCode::OK, http::HeaderMap::new(), Vec::new(), request.call_context_json().map(str::to_owned), ); - let context: serde_json::Value = response.call_context().expect("decode").expect("present"); + let context: serde_json::Value = response + .call_context() + .expect("should decode the call context") + .expect("should carry the call context"); assert_eq!(context, json!({ "nonce": [1, 2] }), "should round-trip"); let error = ProviderRequest::get(&url()) .try_with_call_context(&"x".repeat(MAX_CALL_CONTEXT_BYTES)) - .expect_err("should reject oversize"); - assert!(matches!(error.current_context(), IdentityProviderError::Config { .. }), "Config"); + .expect_err("should reject an oversized call context"); + assert!( + matches!(error.current_context(), IdentityProviderError::Config { .. }), + "should report an oversized call context as Config" + ); } #[test] fn success_helpers_check_status_and_encoding() { let ok = ProviderResponse::for_test(StatusCode::OK, br#"{"uid":"u"}"#); - let value: serde_json::Value = ok.success_json().expect("should parse"); - assert_eq!(value["uid"], "u", "uid"); + let value: serde_json::Value = ok.success_json().expect("should parse a 2xx JSON body"); + assert_eq!(value["uid"], "u", "should read the uid"); let rejected = ProviderResponse::for_test(StatusCode::FORBIDDEN, br#"{}"#); - let error = rejected.success_json::().expect_err("should reject"); + let error = rejected + .success_json::() + .expect_err("should reject a non-2xx status"); assert!( matches!(error.current_context(), IdentityProviderError::Rejected { status } if *status == StatusCode::FORBIDDEN), - "Rejected with the status" + "should report Rejected with the status" ); - rejected.json::().expect("json ignores the status"); + rejected + .json::() + .expect("should parse JSON whatever the status"); let binary = ProviderResponse::for_test(StatusCode::OK, &[0xff, 0xfe]); let error = binary.success_text().expect_err("should reject non-UTF-8"); - assert!(matches!(error.current_context(), IdentityProviderError::Malformed), "Malformed"); + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should report non-UTF-8 as Malformed" + ); } } ``` @@ -3372,68 +3966,366 @@ mod tests { use super::*; fn source(name: &str) -> SourceDomain { - name.parse().expect("should parse") + name.parse().expect("should parse the source") + } + + fn uid(id: &str) -> ResolvedUid { + ResolvedUid::new(id) + } + + fn kv_uid(id: &str) -> KvUid { + KvUid { id: id.to_owned(), atype: None, ext: None } + } + + fn now() -> UnixTime { + UnixTime::from_secs(1_000) + } + + fn parts(sets: &EntrySets) -> Vec { + sets.drops.iter().map(|drop| drop.part).collect() } #[test] fn empty_found_is_no_match() { let resolution = Resolution::found([]); assert!(resolution.is_no_match(), "should be no_match"); - assert!(resolution.ids().is_empty(), "no ids"); + assert!(resolution.eids().is_empty(), "should hold no entries"); } #[test] - fn found_keeps_the_first_id_per_source() { + fn a_one_uid_entry_becomes_a_one_entry_set_without_eid_level_fields() { let a = source("a.example.com"); - let resolution = Resolution::found([ResolvedId::new(&a, "first"), ResolvedId::new(&a, "second")]); - let (kept, dropped) = resolution.found_ids_by_source(); - assert_eq!(kept.len(), 1, "one per source"); - assert_eq!(kept[0].uid(), "first", "should keep the first"); - assert_eq!(dropped, 1, "should count the duplicate"); + let resolution = Resolution::found([ResolvedEid::new(&a, uid("u").with_atype(3))]); + + let sets = entry_sets(resolution.eids(), now()); + + assert_eq!( + sets.sets[&a].entries, + vec![KvEid { + uids: vec![KvUid { id: "u".to_owned(), atype: Some(3), ext: None }], + inserter: None, + matcher: None, + mm: None, + ext: None, + }], + "should produce the one-UID shape browser ingest stores" + ); + assert!(sets.drops.is_empty() && sets.oversized.is_empty(), "should drop nothing"); } #[test] - fn invalidations_chain_and_state_changes_are_reported() { + fn entries_group_by_source_and_keep_every_uid_and_field_in_order() { let a = source("a.example.com"); let b = source("b.example.com"); - let resolution = Resolution::found([ResolvedId::new(&a, "u")]) - .also_invalidate(&b) - .with_retry_in(Duration::from_secs(60)) - .replacing_state(&json!({ "signature": "s" })); - assert_eq!(resolution.invalidated(), &[b], "should invalidate b"); - assert_eq!(resolution.retry_in(), Some(Duration::from_secs(60)), "retry"); - assert_eq!(resolution.state_change(), StateChange::Replace(r#"{"signature":"s"}"#), "state"); - assert_eq!(Resolution::no_match().clearing_state().state_change(), StateChange::Clear, "clear"); - assert_eq!(Resolution::no_match().state_change(), StateChange::Keep, "keep"); + let first = ResolvedEid::new(&a, uid("a1").with_atype(1).with_ext(json!({ "rtiPartner": "TDID" }))) + .with_uid(uid("a2")) + .with_inserter("inserter.example.com") + .with_matcher("matcher.example.com") + .with_mm(2) + .with_ext(json!({ "segments": ["s"] })); + let other = ResolvedEid::new(&b, uid("b1")); + let second = ResolvedEid::new(&a, uid("a3")); + + let sets = entry_sets(&[first, other, second], now()); + + assert_eq!( + sets.sets[&a].entries, + vec![ + KvEid { + uids: vec![ + KvUid { id: "a1".to_owned(), atype: Some(1), ext: Some(json!({ "rtiPartner": "TDID" })) }, + kv_uid("a2"), + ], + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(2), + ext: Some(json!({ "segments": ["s"] })), + }, + KvEid { uids: vec![kv_uid("a3")], inserter: None, matcher: None, mm: None, ext: None }, + ], + "should keep a's entries in the order returned" + ); + assert_eq!(sets.sets[&b].entries.len(), 1, "should group b on its own"); + assert!(sets.drops.is_empty(), "should drop nothing"); } #[test] - fn oversized_state_is_marked_invalid_not_dropped_silently() { - let resolution = Resolution::found([ResolvedId::new(&source("a.example.com"), "u")]) - .replacing_state(&"x".repeat(2000)); - assert!(resolution.state_rejected(), "should flag the state as rejected"); + fn bad_fields_are_dropped_and_their_entries_kept() { + let a = source("a.example.com"); + let entry = ResolvedEid::new(&a, uid("u").with_atype(-1).with_ext(json!(["not", "an", "object"]))) + .with_inserter(" ") + .with_matcher(&"m".repeat(MAX_EID_FIELD_BYTES + 1)) + .with_mm(-2) + .with_ext(json!("not an object")); + + let sets = entry_sets(&[entry], now()); + + assert_eq!( + sets.sets[&a].entries, + vec![KvEid { uids: vec![kv_uid("u")], inserter: None, matcher: None, mm: None, ext: None }], + "should keep the entry without its bad fields" + ); + assert_eq!( + parts(&sets), + vec![ + DroppedPart::Atype, + DroppedPart::UidExt, + DroppedPart::Inserter, + DroppedPart::Matcher, + DroppedPart::Mm, + DroppedPart::EidExt, + ], + "should name each dropped field" + ); + } + + #[test] + fn bad_uids_are_dropped_and_an_entry_without_uids_is_discarded() { + let a = source("a.example.com"); + let mixed = ResolvedEid::new(&a, uid(" ")).with_uid(uid("kept")); + let empty = ResolvedEid::new(&a, uid(&"x".repeat(MAX_UID_LENGTH + 1))).with_inserter("inserter.example.com"); + + let sets = entry_sets(&[mixed, empty], now()); + + assert_eq!(sets.sets[&a].entries.len(), 1, "should discard the entry left with no UID"); + assert_eq!(sets.sets[&a].entries[0].uids, vec![kv_uid("kept")], "should keep the valid UID"); + assert_eq!( + parts(&sets), + vec![DroppedPart::Uid, DroppedPart::Uid, DroppedPart::EmptyEntry], + "should name the dropped UIDs and the discarded entry" + ); + } + + #[test] + fn a_source_with_no_kept_entry_is_not_named() { + let a = source("a.example.com"); + + let sets = entry_sets(&[ResolvedEid::new(&a, uid(""))], now()); + + assert!(!sets.sets.contains_key(&a), "should leave the stored set as it is"); + assert_eq!( + parts(&sets), + vec![DroppedPart::Uid, DroppedPart::EmptyEntry], + "should record the dropped UID and the discarded entry" + ); + assert!(sets.all_discarded(), "should report that every named set was discarded"); + } + + #[test] + fn all_discarded_only_when_nothing_was_kept() { + let a = source("a.example.com"); + let b = source("b.example.com"); + + let mixed = entry_sets(&[ResolvedEid::new(&a, uid("")), ResolvedEid::new(&b, uid("kept"))], now()); + let empty = entry_sets(&[], now()); + + assert!(!mixed.all_discarded(), "should not report a result that kept one set"); + assert!(!empty.all_discarded(), "should not report a result that named nothing"); + } + + #[test] + fn duplicates_and_entries_and_uids_over_the_limits_are_dropped() { + let a = source("a.example.com"); + let mut eids = vec![ + ResolvedEid::new(&a, uid("e0")), + ResolvedEid::new(&a, uid("e0")), + ]; + // 7 more entries of 4 UIDs: 1 + 28 = 29 UIDs, then a ninth entry. + for entry in 1..=7 { + let mut eid = ResolvedEid::new(&a, uid(&format!("e{entry}u0"))); + for index in 1..4 { + eid = eid.with_uid(uid(&format!("e{entry}u{index}"))); + } + eids.push(eid); + } + eids.push(ResolvedEid::new(&a, uid("ninth"))); + + let sets = entry_sets(&eids, now()); + + let set = &sets.sets[&a]; + assert_eq!(set.entries.len(), MAX_ENTRIES_PER_SOURCE, "should keep 8 entries"); + let uid_count: usize = set.entries.iter().map(|entry| entry.uids.len()).sum(); + assert_eq!(uid_count, 29, "should stay under the UID cap in this control"); + assert_eq!( + parts(&sets), + vec![DroppedPart::DuplicateEntry, DroppedPart::ExtraEntry], + "should drop the duplicate and the ninth entry" + ); + + let mut wide = ResolvedEid::new(&a, uid("u0")); + for index in 1..=MAX_UIDS_PER_SOURCE { + wide = wide.with_uid(uid(&format!("u{index}"))); + } + let sets = entry_sets(&[wide, ResolvedEid::new(&a, uid("later"))], now()); + assert_eq!(sets.sets[&a].entries.len(), 1, "should discard the entry left with no room"); + assert_eq!(sets.sets[&a].entries[0].uids.len(), MAX_UIDS_PER_SOURCE, "should keep 32 UIDs"); + assert_eq!( + parts(&sets), + vec![DroppedPart::ExtraUid, DroppedPart::ExtraUid, DroppedPart::ExtraEntry], + "should drop the 33rd UID and the entry with no room" + ); + } + + #[test] + fn an_oversized_set_is_discarded_whole_and_others_are_kept() { + let a = source("a.example.com"); + let b = source("b.example.com"); + + let sets = entry_sets(&[oversized_entry(&a), ResolvedEid::new(&b, uid("small"))], now()); + + assert!(!sets.sets.contains_key(&a), "should never trim an oversized set"); + assert_eq!(sets.oversized, vec![a.clone()], "should report the oversized source"); + assert!(sets.sets.contains_key(&b), "should keep the other source"); + assert!(!sets.all_discarded(), "should not report the result as all discarded"); + + let only = entry_sets(&[oversized_entry(&a)], now()); + assert!(only.all_discarded(), "should report a result whose only set is oversized"); + } + + fn oversized_entry(source: &SourceDomain) -> ResolvedEid { + let mut big = ResolvedEid::new(source, uid(&"x".repeat(500))); + for index in 0..8 { + big = big.with_uid(uid(&format!("{index}{}", "x".repeat(499)))); + } + big + } + + #[test] + fn a_set_takes_the_earliest_times_its_entries_set() { + let a = source("a.example.com"); + let first = ResolvedEid::new(&a, uid("one")) + .with_expires_in(Duration::from_secs(100)) + .with_refresh_at(UnixTime::from_secs(1_080)) + .with_refresh_until(UnixTime::from_secs(5_000)); + let second = ResolvedEid::new(&a, uid("two")) + .with_expires_at(UnixTime::from_secs(1_050)) + .with_refresh_in(Duration::from_secs(90)) + .with_refresh_until(UnixTime::from_secs(4_000)); + let third = ResolvedEid::new(&a, uid("three")); + + let set = entry_sets(&[first, second, third], now()).sets.remove(&a).expect("should keep a"); + + assert_eq!(set.expires_at, Some(UnixTime::from_secs(1_050)), "should take the earliest expiry"); + assert_eq!(set.refresh_at, Some(UnixTime::from_secs(1_080)), "should take the earliest refresh"); + assert_eq!(set.refresh_until, Some(UnixTime::from_secs(4_000)), "should take the earliest refresh limit"); + } + + #[test] + fn dropped_entries_do_not_count_toward_timing() { + let a = source("a.example.com"); + let kept = ResolvedEid::new(&a, uid("one")).with_expires_at(UnixTime::from_secs(2_000)); + let duplicate = ResolvedEid::new(&a, uid("one")).with_expires_at(UnixTime::from_secs(1_100)); + let empty = ResolvedEid::new(&a, uid("")).with_refresh_at(UnixTime::from_secs(1_010)); + + let set = entry_sets(&[kept, duplicate, empty], now()).sets.remove(&a).expect("should keep a"); + + assert_eq!( + set.expires_at, + Some(UnixTime::from_secs(2_000)), + "should ignore the expiry of a dropped duplicate" + ); + assert_eq!(set.refresh_at, None, "should ignore the refresh time of a discarded entry"); + } + + #[test] + fn invalidations_chain_and_state_changes_are_reported() { + let a = source("a.example.com"); + let b = source("b.example.com"); + let resolution = Resolution::found([ResolvedEid::new(&a, uid("u"))]) + .also_invalidate(&b) + .with_retry_in(Duration::from_secs(60)) + .replacing_state(&json!({ "signature": "s" })); + assert_eq!(resolution.invalidated(), &[b], "should invalidate b"); + assert_eq!(resolution.retry_in(), Some(Duration::from_secs(60)), "should keep the retry delay"); + assert_eq!( + resolution.state_change(), + StateChange::Replace(r#"{"signature":"s"}"#), + "should replace the state" + ); + assert_eq!( + Resolution::no_match().clearing_state().state_change(), + StateChange::Clear, + "should clear the state" + ); + assert_eq!(Resolution::no_match().state_change(), StateChange::Keep, "should keep the state by default"); + } + + #[test] + fn resolution_debug_redacts_state() { + let resolution = Resolution::no_match().replacing_state(&json!({ "signature": "secret-state" })); + + let debug = format!("{resolution:?}"); + + assert!(!debug.contains("secret-state"), "should not print provider state"); + assert!(debug.contains("Replace([redacted])"), "should show that state is replaced"); + } + + #[test] + fn oversized_state_is_marked_invalid_not_dropped_silently() { + let resolution = Resolution::found([ResolvedEid::new(&source("a.example.com"), uid("u"))]) + .replacing_state(&"x".repeat(2000)); + assert!(resolution.state_rejected(), "should flag the state as rejected"); + } + + #[test] + fn oversized_ext_is_dropped_and_the_uid_and_entry_kept() { + let pad = json!({ "pad": "x".repeat(MAX_PARTNER_EXT_BYTES) }); + let resolved = uid("u").with_ext(pad.clone()); + assert_eq!(resolved.id(), "u", "should keep the UID"); + assert!(resolved.ext().is_none(), "should drop the UID-level ext"); + let small = uid("u").with_ext(json!({ "linkType": 2 })); + assert_eq!(small.ext(), Some(&json!({ "linkType": 2 })), "should keep a small ext"); + + let eid = ResolvedEid::new(&source("a.example.com"), uid("u")).with_ext(pad); + assert_eq!(eid.uids().len(), 1, "should keep the entry"); + assert!(eid.ext().is_none(), "should drop the EID-level ext"); } #[test] - fn oversized_ext_is_dropped_and_the_id_kept() { - let id = ResolvedId::new(&source("a.example.com"), "u").with_ext(json!({ "pad": "x".repeat(300) })); - assert_eq!(id.uid(), "u", "should keep the id"); - assert!(id.ext().is_none(), "should drop the ext"); - let small = ResolvedId::new(&source("a.example.com"), "u").with_ext(json!({ "linkType": 2 })); - assert_eq!(small.ext(), Some(&json!({ "linkType": 2 })), "control: a small ext is kept"); + fn debug_redacts_every_uid() { + let eid = ResolvedEid::new(&source("a.example.com"), uid("secret-one")).with_uid(uid("secret-two")); + let debug = format!("{eid:?} {:?}", eid.uids()[0]); + assert!(!debug.contains("secret"), "should not print a UID"); + assert!(debug.contains("a.example.com"), "should print the source"); } #[test] fn opted_out_is_reported() { - assert!(Resolution::opted_out().is_opted_out(), "opted out"); - assert!(!Resolution::no_match().is_opted_out(), "control"); + assert!(Resolution::opted_out().is_opted_out(), "should report the opt-out"); + assert!(!Resolution::no_match().is_opted_out(), "should not report an opt-out for no_match"); } } ``` +In `input.rs`, add `Captures` to the test module's `descriptor` import and `use crate::ec::identity::exchange::ProviderResponse;` beside it, then add the test that moved from Task 9: + +```rust + #[test] + fn capture_input_exposes_path_and_never_the_request_body() { + let descriptor = ProviderDescriptor::builder("c".parse().expect("should parse the provider id")) + .with_source("example.com".parse().expect("should parse the source")) + .with_captures(Captures::new().with_path(http::Method::POST, "/integrations/c/api")) + .build() + .expect("should build"); + let response = ProviderResponse::for_test(http::StatusCode::OK, b"{}"); + let consent = ProviderConsent::default(); + let input = CaptureInput::new( + &descriptor, + "/integrations/c/api", + &response, + &BTreeMap::new(), + &consent, + "www.example.com", + UnixTime::from_secs(1), + ); + assert_eq!(input.path(), "/integrations/c/api", "should expose the proxied path"); + assert_eq!(input.primary_source().as_str(), "example.com", "should expose the primary source"); + } +``` + - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- ec::identity::exchange` and `cargo test-fastly -- ec::identity::resolution` +Run: `cargo test-fastly -- ec::identity::exchange`, `cargo test-fastly -- ec::identity::resolution` and `cargo test-fastly -- ec::identity::input` Expected: compile errors. - [ ] **Step 3: Implement `exchange.rs`** @@ -3443,7 +4335,6 @@ Expected: compile errors. use core::fmt; -use base64::Engine as _; use error_stack::Report; use http::header::{AUTHORIZATION, CONTENT_TYPE, COOKIE}; use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode}; @@ -3720,20 +4611,30 @@ impl ProviderResponse { } ``` -Check where `Redacted` lives (`grep -rn "pub struct Redacted" crates/trusted-server-core/src`) and fix the import. Drop the unused `base64` import if clippy flags it. +`Redacted` is `crate::redacted::Redacted` (`redacted.rs:28`). -- [ ] **Step 4: Implement the rest of `resolution.rs`** +- [ ] **Step 4: Implement the rest of `resolution.rs` and `CaptureInput`** -Append to the file from Task 5: +Merge these imports into the top import block of `resolution.rs` from Task 5, so the file keeps one block (`AGENTS.md` allows no imports after items): ```rust +use std::collections::BTreeMap; +use std::iter; use std::time::Duration; use serde::Serialize; use super::ids::{SourceDomain, UnixTime}; -use crate::ec::kv_types::{MAX_PARTNER_EXT_BYTES, MAX_PROVIDER_STATE_BYTES}; +use crate::ec::kv_types::{ + KvEid, KvUid, MAX_EID_FIELD_BYTES, MAX_ENTRIES_PER_SOURCE, MAX_ENTRY_SET_BYTES, + MAX_PARTNER_EXT_BYTES, MAX_PROVIDER_STATE_BYTES, MAX_UID_LENGTH, MAX_UIDS_PER_SOURCE, + entry_set_bytes, +}; +``` + +Then append the items: +```rust /// A time as a provider gave it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum When { @@ -3754,37 +4655,110 @@ impl When { } } -/// One ID a provider returned. +fn ext_fits(ext: &serde_json::Value) -> bool { + serde_json::to_vec(ext).is_ok_and(|bytes| bytes.len() <= MAX_PARTNER_EXT_BYTES) +} + +/// One UID within a [`ResolvedEid`]. #[derive(Clone, PartialEq)] -pub struct ResolvedId { - source: SourceDomain, - uid: String, +pub struct ResolvedUid { + id: String, atype: Option, ext: Option, +} + +impl core::fmt::Debug for ResolvedUid { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("ResolvedUid") + .field("id", &"[redacted]") + .field("atype", &self.atype) + .finish_non_exhaustive() + } +} + +impl ResolvedUid { + /// A UID with no `atype` or ext. + #[must_use] + pub fn new(id: impl Into) -> Self { + Self { id: id.into(), atype: None, ext: None } + } + + /// Sets `atype`. + #[must_use] + pub fn with_atype(mut self, atype: i32) -> Self { + self.atype = Some(atype); + self + } + + /// UID-level ext. Over 2,048 bytes as compact JSON it is dropped and the + /// UID kept. + #[must_use] + pub fn with_ext(mut self, ext: serde_json::Value) -> Self { + let fits = ext_fits(&ext); + if !fits { + log::debug!("Dropping an oversized UID-level ext"); + } + self.ext = fits.then_some(ext); + self + } + + /// The UID. + #[must_use] + pub fn id(&self) -> &str { + &self.id + } + + /// `atype`. + #[must_use] + pub fn atype(&self) -> Option { + self.atype + } + + /// UID-level ext. + #[must_use] + pub fn ext(&self) -> Option<&serde_json::Value> { + self.ext.as_ref() + } +} + +/// One `OpenRTB` EID entry a provider returns: a source, its UIDs and its +/// EID-level fields. +#[derive(Clone, PartialEq)] +pub struct ResolvedEid { + source: SourceDomain, + uids: Vec, + inserter: Option, + matcher: Option, + mm: Option, + ext: Option, expiry: Option, refresh: Option, refresh_until: Option, } -impl core::fmt::Debug for ResolvedId { +impl core::fmt::Debug for ResolvedEid { fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter - .debug_struct("ResolvedId") + .debug_struct("ResolvedEid") .field("source", &self.source) - .field("uid", &"[redacted]") + .field("uids", &self.uids) + .field("mm", &self.mm) .field("expiry", &self.expiry) .finish_non_exhaustive() } } -impl ResolvedId { - /// An ID for `source`. +impl ResolvedEid { + /// An entry with one UID, the common case. #[must_use] - pub fn new(source: &SourceDomain, uid: impl Into) -> Self { + pub fn new(source: &SourceDomain, uid: ResolvedUid) -> Self { Self { source: source.clone(), - uid: uid.into(), - atype: None, + uids: vec![uid], + inserter: None, + matcher: None, + mm: None, ext: None, expiry: None, refresh: None, @@ -3792,25 +4766,52 @@ impl ResolvedId { } } - /// Sets `atype`. + /// Adds another UID to this entry. Repeatable. #[must_use] - pub fn with_atype(mut self, atype: i32) -> Self { - self.atype = Some(atype); + pub fn with_uid(mut self, uid: ResolvedUid) -> Self { + self.uids.push(uid); + self + } + + /// Sets `inserter`. An entry that sets any of `inserter`, `matcher` and + /// `mm` states its EID-level fields completely, so the descriptor's + /// `with_provenance` default no longer applies to it (spec 4.3). + #[must_use] + pub fn with_inserter(mut self, inserter: &str) -> Self { + self.inserter = Some(inserter.to_owned()); + self + } + + /// Sets `matcher`. As [`Self::with_inserter`], it turns off the + /// descriptor's default for this entry. + #[must_use] + pub fn with_matcher(mut self, matcher: &str) -> Self { + self.matcher = Some(matcher.to_owned()); + self + } + + /// Sets `mm`. As [`Self::with_inserter`], it turns off the descriptor's + /// default for this entry. + #[must_use] + pub fn with_mm(mut self, mm: i32) -> Self { + self.mm = Some(mm); self } - /// UID-level ext. Over 256 serialized bytes it is dropped and the ID kept. + /// EID-level ext. Over 2,048 bytes as compact JSON it is dropped and the + /// entry kept. #[must_use] pub fn with_ext(mut self, ext: serde_json::Value) -> Self { - let fits = serde_json::to_vec(&ext).is_ok_and(|bytes| bytes.len() <= MAX_PARTNER_EXT_BYTES); - self.ext = fits.then_some(ext); + let fits = ext_fits(&ext); if !fits { - log::debug!("Dropping oversized ext for source {}", self.source); + log::debug!("Dropping an oversized EID-level ext for source {}", self.source); } + self.ext = fits.then_some(ext); self } - /// Expires after `delay`. + /// Expires after `delay`. Timing applies to the source's whole entry + /// set, and the earliest time across its entries wins. #[must_use] pub fn with_expires_in(mut self, delay: Duration) -> Self { self.expiry = Some(When::In(delay)); @@ -3852,19 +4853,31 @@ impl ResolvedId { &self.source } - /// The UID. + /// The UIDs, in order. #[must_use] - pub fn uid(&self) -> &str { - &self.uid + pub fn uids(&self) -> &[ResolvedUid] { + &self.uids } - /// `atype`. + /// `inserter`, if set. #[must_use] - pub fn atype(&self) -> Option { - self.atype + pub fn inserter(&self) -> Option<&str> { + self.inserter.as_deref() } - /// UID-level ext. + /// `matcher`, if set. + #[must_use] + pub fn matcher(&self) -> Option<&str> { + self.matcher.as_deref() + } + + /// `mm`, if set. + #[must_use] + pub fn mm(&self) -> Option { + self.mm + } + + /// EID-level ext. #[must_use] pub fn ext(&self) -> Option<&serde_json::Value> { self.ext.as_ref() @@ -3900,7 +4913,7 @@ pub enum StateChange<'a> { Clear, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Clone, PartialEq)] enum StateOp { Keep, Replace(String), @@ -3908,10 +4921,22 @@ enum StateOp { Rejected, } -/// What one provider call produced. +impl core::fmt::Debug for StateOp { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Keep => formatter.write_str("Keep"), + Self::Replace(_) => formatter.write_str("Replace([redacted])"), + Self::Clear => formatter.write_str("Clear"), + Self::Rejected => formatter.write_str("Rejected"), + } + } +} + +/// What one provider call produced. Its `Debug` prints entries through +/// [`ResolvedEid`]'s redacting `Debug` and never prints provider state. #[derive(Debug, Clone, PartialEq)] pub struct Resolution { - ids: Vec, + eids: Vec, invalidated: Vec, opted_out: bool, retry_in: Option, @@ -3920,13 +4945,18 @@ pub struct Resolution { impl Resolution { fn empty() -> Self { - Self { ids: Vec::new(), invalidated: Vec::new(), opted_out: false, retry_in: None, state: StateOp::Keep } + Self { eids: Vec::new(), invalidated: Vec::new(), opted_out: false, retry_in: None, state: StateOp::Keep } } - /// One or more IDs, at most one per source. An empty set is `no_match`. + /// One or more EID entries. A source may appear in several entries, in + /// the order bidders should see them. The entries of one source form its + /// new entry set, which replaces the stored set whole for an owned + /// source. For an unowned source the set only fills an absent or expired + /// record (spec 7.2). A source the result does not name, or whose set + /// the limits discard, is left as it is. An empty set is `no_match`. #[must_use] - pub fn found(ids: impl IntoIterator) -> Self { - Self { ids: ids.into_iter().collect(), ..Self::empty() } + pub fn found(eids: impl IntoIterator) -> Self { + Self { eids: eids.into_iter().collect(), ..Self::empty() } } /// Nothing recognized, or a capture held nothing. @@ -3981,10 +5011,10 @@ impl Resolution { self } - /// The returned IDs, as given. + /// The returned entries, as given. #[must_use] - pub fn ids(&self) -> &[ResolvedId] { - &self.ids + pub fn eids(&self) -> &[ResolvedEid] { + &self.eids } /// Invalidated sources. @@ -3996,7 +5026,7 @@ impl Resolution { /// Whether nothing was found, invalidated or opted out. #[must_use] pub fn is_no_match(&self) -> bool { - self.ids.is_empty() && self.invalidated.is_empty() && !self.opted_out + self.eids.is_empty() && self.invalidated.is_empty() && !self.opted_out } /// Whether the provider reported an opt-out. @@ -4033,35 +5063,343 @@ impl Resolution { pub(crate) fn changes_state(&self) -> bool { self.state != StateOp::Keep } +} - /// The first ID per source, in order, and how many duplicates were dropped. - #[must_use] - pub(crate) fn found_ids_by_source(&self) -> (Vec<&ResolvedId>, usize) { - let mut seen = std::collections::BTreeSet::new(); - let mut kept = Vec::new(); - let mut dropped = 0; - for id in &self.ids { - if seen.insert(id.source()) { - kept.push(id); - } else { - dropped += 1; - } - } - (kept, dropped) - } +/// The part of a result that a spec 4.5 limit dropped. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub(crate) enum DroppedPart { + /// A UID that is empty, blank or over `MAX_UID_LENGTH`. + #[display("uid")] + Uid, + /// A negative `atype`. + #[display("atype")] + Atype, + /// A UID-level ext that is not an object or is oversized. + #[display("uid ext")] + UidExt, + /// A blank or oversized `inserter`. + #[display("inserter")] + Inserter, + /// A blank or oversized `matcher`. + #[display("matcher")] + Matcher, + /// A negative `mm`. + #[display("mm")] + Mm, + /// An EID-level ext that is not an object or is oversized. + #[display("eid ext")] + EidExt, + /// An entry left with no UID. + #[display("entry without uid")] + EmptyEntry, + /// An entry equal to an earlier kept entry of its source. + #[display("duplicate entry")] + DuplicateEntry, + /// An entry past the source's first 8, or with no room left for a UID. + #[display("entry over the limit")] + ExtraEntry, + /// A UID past the source's first 32. + #[display("uid over the limit")] + ExtraUid, } -``` -Add `exchange.rs` to `mod.rs` (`pub mod exchange; pub use exchange::{ProviderRequest, ProviderResponse};`) and `pub use resolution::{Resolution, ResolvedId, StateChange, When};`. Add `exchange.rs` to `migration_guards.rs`. Uncomment the capture test from Task 9. +/// One dropped part, for the coordinator's log line (provider, source and +/// part only, never a value). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct EntryDrop { + pub(crate) source: SourceDomain, + pub(crate) part: DroppedPart, +} -- [ ] **Step 5: Run and see them pass** +/// One source's new entry set, as it is stored, and its timing resolved +/// against `now`. +#[derive(Clone, PartialEq)] +pub(crate) struct EntrySet { + pub(crate) entries: Vec, + pub(crate) expires_at: Option, + pub(crate) refresh_at: Option, + pub(crate) refresh_until: Option, +} -Run: `cargo test-fastly -- ec::identity` -Expected: PASS. +impl core::fmt::Debug for EntrySet { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("EntrySet") + .field("entries", &self.entries.len()) + .field("expires_at", &self.expires_at) + .field("refresh_at", &self.refresh_at) + .field("refresh_until", &self.refresh_until) + .finish() + } +} -- [ ] **Step 6: Commit** +/// A result's entries, grouped by source and checked against spec 4.5. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct EntrySets { + /// Each source with at least one kept entry, under every limit. + pub(crate) sets: BTreeMap, + /// What the limits dropped, in order. + pub(crate) drops: Vec, + /// Sources whose set was over `MAX_ENTRY_SET_BYTES` and was discarded + /// whole. A stateful provider's positive part is then rejected. + pub(crate) oversized: Vec, +} -```bash +impl EntrySets { + /// Whether the result named at least one source and every set was + /// discarded. Every entry that is not kept records a drop, so a result + /// with entries and no kept set always has a drop or an oversized set. + /// The coordinator handles such a result as `Malformed` (spec 4.5). + #[must_use] + pub(crate) fn all_discarded(&self) -> bool { + self.sets.is_empty() && (!self.drops.is_empty() || !self.oversized.is_empty()) + } +} + +fn earliest(current: Option, candidate: Option) -> Option { + match (current, candidate) { + (Some(current), Some(candidate)) => Some(current.min(candidate)), + (current, candidate) => current.or(candidate), + } +} + +fn clean_text(value: Option<&str>, part: DroppedPart, dropped: &mut Vec) -> Option { + let value = value?; + if value.trim().is_empty() || value.len() > MAX_EID_FIELD_BYTES { + dropped.push(part); + return None; + } + Some(value.to_owned()) +} + +fn clean_number(value: Option, part: DroppedPart, dropped: &mut Vec) -> Option { + let value = value?; + if value < 0 { + dropped.push(part); + return None; + } + Some(value) +} + +fn clean_ext( + value: Option<&serde_json::Value>, + part: DroppedPart, + dropped: &mut Vec, +) -> Option { + let value = value?; + if !value.is_object() || !ext_fits(value) { + dropped.push(part); + return None; + } + Some(value.clone()) +} + +fn clean_uid(uid: &ResolvedUid, dropped: &mut Vec) -> Option { + if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + dropped.push(DroppedPart::Uid); + return None; + } + Some(KvUid { + id: uid.id.clone(), + atype: clean_number(uid.atype, DroppedPart::Atype, dropped), + ext: clean_ext(uid.ext.as_ref(), DroppedPart::UidExt, dropped), + }) +} + +fn clean_entry(eid: &ResolvedEid, dropped: &mut Vec) -> Option { + let uids: Vec = eid.uids.iter().filter_map(|uid| clean_uid(uid, dropped)).collect(); + if uids.is_empty() { + return None; + } + Some(KvEid { + uids, + inserter: clean_text(eid.inserter.as_deref(), DroppedPart::Inserter, dropped), + matcher: clean_text(eid.matcher.as_deref(), DroppedPart::Matcher, dropped), + mm: clean_number(eid.mm, DroppedPart::Mm, dropped), + ext: clean_ext(eid.ext.as_ref(), DroppedPart::EidExt, dropped), + }) +} + +/// Groups `eids` by source, in order, and applies every limit of spec 4.5: +/// the browser parser's field and UID rules, then per-source caps the +/// browser parser does not have: duplicates, the first 8 entries and 32 UIDs +/// of a source, and `MAX_ENTRY_SET_BYTES` as [`entry_set_bytes`] measures +/// the stored set. A set over that is discarded whole, never trimmed: field +/// caps do not guarantee the set cap. Each set's `exp`, `ra` and `ru` are +/// the earliest its kept entries set; a dropped or discarded entry's times +/// do not count. +/// +/// Source claims and operator narrowing are the coordinator's, which drops +/// a set for an unclaimed source before using the result. +/// +/// # Performance +/// +/// Serializes each set once to measure it, and compares each entry with +/// the at most 8 entries already kept. +#[must_use] +pub(crate) fn entry_sets(eids: &[ResolvedEid], now: UnixTime) -> EntrySets { + let mut result = EntrySets::default(); + for eid in eids { + let mut dropped = Vec::new(); + let cleaned = clean_entry(eid, &mut dropped); + let set = result.sets.entry(eid.source.clone()).or_insert_with(|| EntrySet { + entries: Vec::new(), + expires_at: None, + refresh_at: None, + refresh_until: None, + }); + match cleaned { + None => dropped.push(DroppedPart::EmptyEntry), + Some(entry) if set.entries.contains(&entry) => dropped.push(DroppedPart::DuplicateEntry), + Some(_) if set.entries.len() >= MAX_ENTRIES_PER_SOURCE => dropped.push(DroppedPart::ExtraEntry), + Some(mut entry) => { + let used: usize = set.entries.iter().map(|kept| kept.uids.len()).sum(); + let room = MAX_UIDS_PER_SOURCE.saturating_sub(used); + if entry.uids.len() > room { + dropped.extend(iter::repeat_n(DroppedPart::ExtraUid, entry.uids.len() - room)); + entry.uids.truncate(room); + } + if entry.uids.is_empty() { + dropped.push(DroppedPart::ExtraEntry); + } else { + set.expires_at = earliest(set.expires_at, eid.expiry.map(|when| when.resolve(now))); + set.refresh_at = earliest(set.refresh_at, eid.refresh.map(|when| when.resolve(now))); + set.refresh_until = earliest(set.refresh_until, eid.refresh_until); + set.entries.push(entry); + } + } + } + result + .drops + .extend(dropped.into_iter().map(|part| EntryDrop { source: eid.source.clone(), part })); + } + result.sets.retain(|source, set| { + if set.entries.is_empty() { + return false; + } + let fits = entry_set_bytes(&set.entries) <= MAX_ENTRY_SET_BYTES; + if !fits { + result.oversized.push(source.clone()); + } + fits + }); + result +} +``` + +`retain` borrows `result.sets` mutably while the closure pushes to `result.oversized`. The two are disjoint fields, and closures capture disjoint fields since edition 2021, so the borrows do not conflict. + +`entry_sets` does not apply the expiring-source fallback (an expiring set without `exp` keeps the stored `exp` when its entries are unchanged, and is discarded otherwise). That needs the descriptor and the stored record, so it belongs to the coordinator (Task 20). The coordinator also: + +- ignores `refresh_until` from a provider not declared `with_private_state()` +- rejects a stateful result's positive part when `oversized` is not empty, keeping its removals +- handles a result with `all_discarded()` as `Malformed`, keeping its removals, and otherwise leaves the stored set of a source not in `sets` as it is (spec 4.5) +- logs each `EntryDrop` and oversized source with the provider ID, never a value + +An `EntrySet`'s `entries` are what Task 15's mutation stores as the record's `eids` (Task 6). + +In `input.rs`, add `use super::exchange::ProviderResponse;` to the import block and append `CaptureInput` after `ResolveInput`: + +```rust +/// What a capture may read from one proxied response. +pub struct CaptureInput<'a> { + descriptor: &'a ProviderDescriptor, + path: &'a str, + response: &'a ProviderResponse, + stored: &'a BTreeMap, + consent: &'a ProviderConsent, + publisher_domain: &'a str, + now: UnixTime, +} + +impl fmt::Debug for CaptureInput<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CaptureInput") + .field("provider", self.descriptor.id()) + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl<'a> CaptureInput<'a> { + /// The input for one capture. + #[must_use] + pub(crate) fn new( + descriptor: &'a ProviderDescriptor, + path: &'a str, + response: &'a ProviderResponse, + stored: &'a BTreeMap, + consent: &'a ProviderConsent, + publisher_domain: &'a str, + now: UnixTime, + ) -> Self { + Self { descriptor, path, response, stored, consent, publisher_domain, now } + } + + /// The proxied request path, without query string. + #[must_use] + pub fn path(&self) -> &str { + self.path + } + + /// The proxied response, decoded. + #[must_use] + pub fn response(&self) -> &ProviderResponse { + self.response + } + + /// The descriptor's primary source. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + self.descriptor.primary_source() + } + + /// Finds a declared source by a vendor's spelling. + #[must_use] + pub fn source_named(&self, name: &str) -> Option<&SourceDomain> { + self.descriptor.source_named(name) + } + + /// The stored record for a source. + #[must_use] + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId> { + self.stored.get(source) + } + + /// The request's consent, from its cookies and headers. The request + /// body is never read (spec 5.5). + #[must_use] + pub fn consent(&self) -> &ProviderConsent { + self.consent + } + + /// The publisher's domain. + #[must_use] + pub fn publisher_domain(&self) -> &str { + self.publisher_domain + } + + /// The request's time. + #[must_use] + pub fn now(&self) -> UnixTime { + self.now + } +} +``` + +`CaptureInput::new` takes 7 arguments, which is the maximum `AGENTS.md` allows. Do not add an eighth. Use a struct if you need one. + +Add `exchange.rs` to `mod.rs` (`pub mod exchange; pub use exchange::{ProviderRequest, ProviderResponse};`), `pub use resolution::{Resolution, ResolvedEid, ResolvedUid, StateChange, When};`, and add `CaptureInput` to the `pub use input::{...}` line. Add `exchange.rs` to `migration_guards.rs`. + +- [ ] **Step 5: Run and see them pass** + +Run: `cargo test-fastly -- ec::identity` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs git commit --signoff -S -m "Add provider requests, responses and resolutions" ``` @@ -4105,8 +5443,8 @@ mod tests { #[test] fn default_capability_methods_return_the_not_implemented_marker() { let provider = DescriptorOnly( - ProviderDescriptor::builder("p".parse().expect("id")) - .with_source("example.com".parse().expect("source")) + ProviderDescriptor::builder("p".parse().expect("should parse the provider id")) + .with_source("example.com".parse().expect("should parse the source")) .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) .build() .expect("should build"), @@ -4125,11 +5463,11 @@ mod tests { UnixTime::from_secs(1), ); - let error = boxed.capture(&input).expect_err("should be the marker"); + let error = boxed.capture(&input).expect_err("should return the not-implemented marker"); assert!(error.current_context().is_not_implemented(), "should carry the marker"); assert!( !IdentityProviderError::config("other").current_context().is_not_implemented(), - "control: an ordinary Config error is not the marker" + "should not treat an ordinary Config error as the marker" ); } } @@ -4161,7 +5499,7 @@ use crate::settings::IntegrationConfig; const _: Option<&dyn IdentityProvider> = None; ``` -Check where `IntegrationConfig` lives (`grep -rn "pub trait IntegrationConfig" crates/trusted-server-core/src`) and fix the path. Add `pub mod provider; pub use provider::{IdentityProvider, IdentityProviderConfig};` to `mod.rs`, and the file to `migration_guards.rs`. +`IntegrationConfig` is `crate::settings::IntegrationConfig` (`settings.rs:235`). Write `OpenRTB`, `LiveRamp` and other mixed-case names in backticks in the copied doc comments (`clippy::doc_markdown`). Add `pub mod provider; pub use provider::{IdentityProvider, IdentityProviderConfig};` to `mod.rs`, and the file to `migration_guards.rs`. - [ ] **Step 4: Run and see it pass** @@ -4180,21 +5518,25 @@ git commit --signoff -S -m "Add the IdentityProvider trait" **Files:** - Create: `crates/trusted-server-core/src/ec/identity/policy.rs` -- Modify: `crates/trusted-server-core/src/consent/tcf.rs` (add `has_vendor_consent`) +- Modify: `crates/trusted-server-core/src/consent/mod.rs` (make `effective_tcf` `pub(crate)`, visibility only) - Modify: `mod.rs`, `migration_guards.rs` -- Test: `policy.rs`, `consent/tcf.rs` +- Test: `policy.rs` **Interfaces:** -- Consumes: `ConsentPolicy` (Task 8), `ConsentContext`, `allows_eid_persistence`, `ec_consent_granted`, and the TCF and GPP decoded forms in `consent/` +- Consumes: `ConsentPolicy` (Task 8), `ConsentContext`, the decoded forms `TcfConsent` (with `has_purpose_consent(usize)` and `has_vendor_consent(u16)`, `consent/types.rs:230-252`), `GppConsent` (with `eu_tcf` and `us_sale_opt_out: Option`) and `UsPrivacy` (with `opt_out_sale: PrivacyFlag`) - Produces: - `pub(crate) struct PolicySignals { dnt: bool }`. DNT is not in `ConsentContext`, so the caller passes it. - `pub(crate) fn policy_allows(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> bool` - `pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool`, the same test `consent_allows_server_side_auction` uses (`consent/mod.rs:298-308`) - - `pub(crate) fn has_vendor_consent(tcf: &DecodedTcf, vendor_id: u16) -> bool` in `consent/tcf.rs` + - a private `fn us_sale_opt_out(consent: &ConsentContext) -> bool` in `policy.rs`: the GPP US sale opt-out or the US Privacy sale flag, the same two checks `allows_ec_creation` makes (`consent/mod.rs:539-547`). No consent function changes behaviour + +Vendor consent needs no new helper: `TcfConsent::has_vendor_consent` already exists (`consent/types.rs:250-252`). Part 2 (Task 34) extends `policy_allows` with the sharing and targeted-advertising opt-outs, GPP section 6, and the GPC and known-child fields. Until then, a strict policy blocks on the signals already decoded: TCF purposes and vendor, the US sale opt-out (GPP and US Privacy), `Sec-GPC`, DNT, and any raw string present without a decoded form. +A GPP string that decodes but has no applicable section, such as the header-only `DBAA` a CMP sends when no section applies, has a decoded form and carries no opt-out, so it does not block `strict()` by itself (spec 9.1). Where TCF applies, the policy still needs TCF consent, from a TC string or from a GPP TCF section. Task 34 keeps both rules. + - [ ] **Step 1: Write the failing tests** Build `ConsentContext` values the way existing consent tests do (`grep -n "fn .*consent_context\|ConsentContext {" crates/trusted-server-core/src/consent/mod.rs | head`). Reuse their helper. Then: @@ -4209,7 +5551,7 @@ mod tests { fn strict_blocks_each_signal_it_decodes() { let strict = ConsentPolicy::strict(); let allowed = consent_with(Signals::default()); - assert!(policy_allows(&strict, &allowed, PolicySignals { dnt: false }), "control: nothing set"); + assert!(policy_allows(&strict, &allowed, PolicySignals { dnt: false }), "should allow a request with no signal set"); for (name, signals, dnt) in [ ("gdpr without purpose 4", Signals { gdpr: true, purposes: vec![1], ..Signals::default() }, false), @@ -4220,7 +5562,7 @@ mod tests { ] { assert!( !policy_allows(&strict, &consent_with(signals), PolicySignals { dnt }), - "strict should block: {name}" + "should block under strict: {name}" ); } } @@ -4228,15 +5570,24 @@ mod tests { #[test] fn relaxations_and_vendor_consent() { let gpc = consent_with(Signals { gpc: true, ..Signals::default() }); - assert!(policy_allows(&ConsentPolicy::strict().allowing_gpc(), &gpc, PolicySignals { dnt: false }), "gpc relaxed"); + assert!( + policy_allows(&ConsentPolicy::strict().allowing_gpc(), &gpc, PolicySignals { dnt: false }), + "should allow GPC when the policy relaxes it" + ); let sale = consent_with(Signals { us_sale_opt_out: true, ..Signals::default() }); assert!( policy_allows(&ConsentPolicy::strict().allowing_us_opt_out(UsOptOut::Sale), &sale, PolicySignals { dnt: false }), - "sale relaxed" + "should allow a sale opt-out when the policy relaxes it" ); let gdpr = consent_with(Signals { gdpr: true, purposes: vec![1, 4], vendors: vec![9999], ..Signals::default() }); - assert!(policy_allows(&ConsentPolicy::strict().with_tcf_vendor(9999), &gdpr, PolicySignals { dnt: false }), "vendor granted"); - assert!(!policy_allows(&ConsentPolicy::strict().with_tcf_vendor(1234), &gdpr, PolicySignals { dnt: false }), "vendor missing"); + assert!( + policy_allows(&ConsentPolicy::strict().with_tcf_vendor(9999), &gdpr, PolicySignals { dnt: false }), + "should allow a granted vendor" + ); + assert!( + !policy_allows(&ConsentPolicy::strict().with_tcf_vendor(1234), &gdpr, PolicySignals { dnt: false }), + "should block a vendor without consent" + ); } #[test] @@ -4244,13 +5595,34 @@ mod tests { let consent = consent_with(Signals { gpc: true, us_sale_opt_out: true, ..Signals::default() }); assert!( policy_allows(&ConsentPolicy::eid_persistence_only(), &consent, PolicySignals { dnt: true }), - "generic pull's parity policy checks only the global gates, which the coordinator applies separately" + "should check only the global gates for generic pull's parity policy, which the coordinator applies separately" + ); + } + + #[test] + fn a_header_only_gpp_string_does_not_block_strict() { + let header_only = consent_with(Signals { gpp_header_only: true, ..Signals::default() }); + assert!( + policy_allows(&ConsentPolicy::strict(), &header_only, PolicySignals { dnt: false }), + "should not block on a decoded GPP string with no applicable section" + ); + + let under_gdpr = consent_with(Signals { gdpr: true, gpp_header_only: true, ..Signals::default() }); + assert!( + !policy_allows(&ConsentPolicy::strict(), &under_gdpr, PolicySignals { dnt: false }), + "should still need TCF consent where TCF applies" + ); + + let gpp_tcf = consent_with(Signals { gdpr: true, purposes: vec![1, 4], tcf_in_gpp: true, ..Signals::default() }); + assert!( + policy_allows(&ConsentPolicy::strict(), &gpp_tcf, PolicySignals { dnt: false }), + "should accept TCF consent from a GPP TCF section" ); } } ``` -`Signals` and `consent_with` are a small test helper you write in this module. It builds a `ConsentContext` with GDPR applying, the given TCF purposes and vendor consents decoded, the US sale opt-out decoded from a US Privacy string `1YYN`, `gpc` set, and, for `raw_tc_without_decode`, `raw_tc_string = Some(..)` with no decoded TCF. Build it on whatever constructor the consent tests use. Read `consent/types.rs` first. +`Signals` and `consent_with` are a small test helper you write in this module. It builds a `ConsentContext` with the given TCF purposes and vendor consents decoded, the US sale opt-out decoded from a US Privacy string `1YYN`, `gpc` set, and, for `raw_tc_without_decode`, `raw_tc_string = Some(..)` with no decoded TCF. `gdpr: true` sets `gdpr_applies` and `Jurisdiction::Gdpr`; otherwise the jurisdiction is `Jurisdiction::UsState("CA".to_owned())`, because `tcf_applies` treats `Unknown` as TCF applying. `gpp_header_only` sets `raw_gpp_string = Some("DBAA")` and `gpp` to `decode_gpp_string("DBAA")`, with `gpp_section_ids = Some(vec![])`. `tcf_in_gpp` puts the decoded TCF in `gpp.eu_tcf` instead of `tcf`, with `raw_tc_string` unset. Build it on whatever constructor the consent tests use. Read `consent/types.rs` first. If `decode_gpp_string("DBAA")` fails with the pinned `iab_gpp`, stop: the fail-closed rule would then block it, and the spec 9.1 rule needs a decoder change that this task does not make. Report it instead of working around it. - [ ] **Step 2: Run and see them fail** @@ -4259,26 +5631,15 @@ Expected: compile errors. - [ ] **Step 3: Implement** -Add to `consent/tcf.rs`, next to the purpose-consent helpers at about line 145: - -```rust -/// Whether the TC string grants consent to an IAB Global Vendor List vendor. -#[must_use] -pub(crate) fn has_vendor_consent(tcf: &TcfConsent, vendor_id: u16) -> bool { - tcf.vendor_consents.contains(&vendor_id) -} -``` - -Use the real decoded type name and its vendor-consent field. Read the struct at the top of `tcf.rs`. If vendor consents are stored as a bitfield, index into it instead. Add a unit test next to the existing TCF tests that decodes a fixture TC string already used in that file and asserts one vendor present and one absent. - `policy.rs`: ```rust //! Evaluates a provider's [`ConsentPolicy`] against a request's consent. use super::descriptor::{ConsentPolicy, UsOptOut}; -use crate::consent::ConsentContext; use crate::consent::jurisdiction::Jurisdiction; +use crate::consent::types::PrivacyFlag; +use crate::consent::{ConsentContext, effective_tcf}; /// Signals the consent context does not carry. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -4293,6 +5654,16 @@ pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool { consent.gdpr_applies || matches!(consent.jurisdiction, Jurisdiction::Gdpr | Jurisdiction::Unknown) } +/// Whether the request carries a decoded US sale opt-out, from GPP or from +/// the US Privacy string. +fn us_sale_opt_out(consent: &ConsentContext) -> bool { + consent.gpp.as_ref().and_then(|gpp| gpp.us_sale_opt_out) == Some(true) + || consent + .us_privacy + .as_ref() + .is_some_and(|usp| usp.opt_out_sale == PrivacyFlag::Yes) +} + /// Whether `policy` allows a provider under this request's consent. The /// global EC and EID-persistence gates are checked separately (eligibility /// steps 1 and 2) and are not repeated here. @@ -4309,19 +5680,25 @@ pub(crate) fn policy_allows(policy: &ConsentPolicy, consent: &ConsentContext, si return false; } if tcf_applies(consent) { - let Some(tcf) = consent.tcf.as_ref() else { + // A TC string or a GPP TCF section. A header-only GPP string has + // neither, so TCF consent is still missing. + let Some(tcf) = effective_tcf(consent) else { return false; }; - if !policy.tcf_purposes().iter().all(|purpose| tcf.has_purpose_consent(*purpose)) { + if !policy + .tcf_purposes() + .iter() + .all(|purpose| tcf.has_purpose_consent(usize::from(*purpose))) + { return false; } if let Some(vendor) = policy.tcf_vendor() - && !crate::consent::tcf::has_vendor_consent(tcf, vendor) + && !tcf.has_vendor_consent(vendor) { return false; } } - if !policy.allows_us_opt_out(UsOptOut::Sale) && crate::consent::us_sale_opted_out(consent) { + if !policy.allows_us_opt_out(UsOptOut::Sale) && us_sale_opt_out(consent) { return false; } if !policy.allows_gpc() && consent.gpc { @@ -4334,7 +5711,7 @@ pub(crate) fn policy_allows(policy: &ConsentPolicy, consent: &ConsentContext, si } ``` -Every `consent.*` field name and helper above (`tcf`, `gpp`, `us_privacy`, `gpc`, `jurisdiction`, `has_purpose_consent`, `us_sale_opted_out`) must be matched to the real API in `consent/`. Run `grep -n "pub fn\|pub(crate) fn\|pub [a-z_]*:" crates/trusted-server-core/src/consent/*.rs` and use what exists. If there is no single US-sale helper, combine `decode_us_sale_opt_out` (`gpp.rs:125`) and the US Privacy check (`us_privacy.rs:50-63`) in a private `fn us_sale_opt_out(consent: &ConsentContext) -> bool` in this file. Do not change any existing consent function's behaviour. +`effective_tcf` (`consent/mod.rs:283-288`) prefers `tcf` and falls back to `gpp.eu_tcf`, as the auction's consent checks do. Change only its visibility to `pub(crate)`. The fail-closed check needs no special case for `DBAA`: it decodes, so `consent.gpp` is `Some` and nothing blocks. The field names above (`raw_tc_string`, `tcf`, `raw_gpp_string`, `gpp`, `raw_us_privacy`, `us_privacy`, `gpc`, `gdpr_applies`, `jurisdiction`) are `ConsentContext`'s (`consent/types.rs:109-148`). `has_purpose_consent` takes a `usize`, so the `u8` purposes convert with `usize::from`. `us_sale_opt_out` reads the decoded `GppConsent::us_sale_opt_out` and `UsPrivacy::opt_out_sale`; it does not call the private `decode_us_sale_opt_out` (`gpp.rs:125`), which takes the parsed GPP string. Do not change any existing consent function's behaviour. - [ ] **Step 4: Run and see them pass** @@ -4344,11 +5721,11 @@ Expected: PASS. - [ ] **Step 5: Commit** ```bash -git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/consent/tcf.rs crates/trusted-server-core/src/migration_guards.rs +git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/consent/mod.rs crates/trusted-server-core/src/migration_guards.rs git commit --signoff -S -m "Evaluate provider consent policies against decoded consent" ``` -### Task 13: Revisions and keyed hashes +### Task 13: Revisions and hashes **Files:** @@ -4358,27 +5735,36 @@ git commit --signoff -S -m "Evaluate provider consent policies against decoded c **Interfaces:** +- Consumes: `KvPartnerId`, `KvEid`, `KvUid`, `RECORD_REVISION_HEX_LEN` (Task 6), `RandomSource` (Task 9) - Produces: - - `pub(crate) struct IdentityKeys`, derived once per settings load, with `pub(crate) fn from_settings(settings: &Settings) -> Self`. It holds an HMAC key for each of: `revision` (legacy `pv` fallback), `invalidated` (`ps.iv`), `visitor` (rate-limit key), `fingerprint` (consent), and the AES keys `cookie_s` and `cookie_r` (Task 16 uses them). Labels: `trusted-server/identity/revision/v1`, `trusted-server/identity/iv/v1`, `trusted-server/identity/visitor/v1`, `trusted-server/identity/fingerprint/v1`, `trusted-server/ts-eids-s/key/v1`, `trusted-server/ts-eids-r/key/v1`. - - `pub(crate) struct Revision(String)`: 16 lowercase hex digits, with `as_str` - - `pub(crate) fn new_revision(random: &dyn RandomSource) -> Revision`: 8 random bytes as hex - - `pub(crate) fn record_revision(keys: &IdentityKeys, record: &KvPartnerId) -> Revision`: `rv` when present, else the first 16 hex digits of HMAC(`revision` key, `uid \0 at`) - - `pub(crate) fn invalidated_uid_hash(keys: &IdentityKeys, provider: &IdentityProviderId, uid: &str) -> String`: 16 hex + - `pub(crate) struct IdentityKeys`, derived once per settings load from `ec.passphrase`, with `pub(crate) fn from_settings(settings: &Settings) -> Self`. It holds one HMAC key, `visitor` (the rate-limit key), with the label `trusted-server/identity/visitor/v1`. Nothing stored in the row is keyed (spec 9.3), so this is the only key. + - `pub(crate) struct RecordRevision(String)`: 16 lowercase hex digits, with `as_str` and `from_stored(&str)` + - `pub(crate) enum Revision { Unset, Set(RecordRevision) }` (spec 5.4): a record's revision, `Unset` for a record without `rv`, which only a version 1 row holds + - `pub(crate) fn new_revision(random: &dyn RandomSource) -> RecordRevision`: 8 random bytes as hex + - `pub(crate) fn record_revision(record: &KvPartnerId) -> Revision`: `Set` of `rv` when present, else `Unset`. One `rv` covers a source's whole entry set (spec 5.4). + - `pub(crate) struct StateRevision(String)` with `pub(crate) fn of(st: Option<&serde_json::Value>) -> Self`: the first 16 hex digits of the unkeyed SHA-256 of `st` as compact JSON, or of the empty string when there is none. A `ProviderResult`'s `expected_state` (Task 15) holds one when its call read `ps.st`. + - `pub(crate) fn uid_hash(uid: &str) -> String`: the first 16 hex digits of the unkeyed SHA-256 of the UID (spec 6.1) + - `pub(crate) fn set_hashes(set: &[KvEid]) -> Vec`: one `uid_hash` per UID of the set, in set order, without duplicates. A removal hashes every UID of the record it removes (spec 5.4), so a one-UID record costs one hash and a 32-UID set 32. The hashes come from the live record when the removal applies (Task 15). + - `pub(crate) fn push_hashes(list: &mut Vec, hashes: Vec)`: appends each hash to a `ps.iv` or `ps.ov` list, moving a hash already present to the end (spec 6.1). It does not truncate; Task 15's step 4 does. - `pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String`: 16 hex - - `pub(crate) fn consent_fingerprint(keys: &IdentityKeys, bits: &str) -> String`: 16 hex. Task 34 builds `bits`. + +The UID hash is unkeyed because the row stores every live UID in plain text, so a key would protect nothing, and an unkeyed hash survives an `ec.passphrase` rotation (spec 6.1, 9.3). It is not namespaced by provider: `ps.iv` and `ps.ov` already sit under a provider. There is no keyed legacy revision either. A version 1 record has the revision `Unset`, and an `OwnerRemove` naming it matches `Unset` until the record's next write (spec 14). The consent fingerprint is a plain bitmask that Task 20 builds, not a hash. - [ ] **Step 1: Write the failing tests** ```rust #[cfg(test)] mod tests { + use serde_json::json; + use sha2::{Digest as _, Sha256}; + use super::*; use crate::ec::identity::input::SeededRandom; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::test_support::tests::create_test_settings; - fn keys() -> IdentityKeys { - IdentityKeys::from_settings(&create_test_settings()) + fn uid(id: &str) -> KvUid { + KvUid { id: id.to_owned(), ..KvUid::default() } } #[test] @@ -4386,36 +5772,75 @@ mod tests { let random = SeededRandom::new(7); let first = new_revision(&random); let second = new_revision(&random); - assert_eq!(first.as_str().len(), 16, "16 hex digits"); - assert!(first.as_str().bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), "lowercase hex"); - assert_ne!(first, second, "two writes of the same UID get different revisions"); + assert_eq!(first.as_str().len(), 16, "should be 16 hex digits"); + assert!( + first.as_str().bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), + "should be lowercase hex" + ); + assert_ne!(first, second, "should give two writes of the same UID different revisions"); } #[test] - fn record_revision_prefers_rv_and_hashes_legacy_records() { - let keys = keys(); + fn record_revision_is_rv_or_unset() { let with_rv = KvPartnerId { rv: Some("0123456789abcdef".to_owned()), ..KvPartnerId::new("u") }; - assert_eq!(record_revision(&keys, &with_rv).as_str(), "0123456789abcdef", "should use rv"); - - let legacy_a = KvPartnerId::browser("u", Some(10)); - let legacy_b = KvPartnerId::browser("u", Some(11)); - assert_eq!(record_revision(&keys, &legacy_a), record_revision(&keys, &legacy_a.clone()), "stable"); - assert_ne!(record_revision(&keys, &legacy_a), record_revision(&keys, &legacy_b), "at changes it"); + assert_eq!( + record_revision(&with_rv), + Revision::Set(RecordRevision::from_stored("0123456789abcdef")), + "should use rv" + ); + assert_eq!( + record_revision(&KvPartnerId::browser("u", Some(10))), + Revision::Unset, + "should be Unset without rv, whatever the UID and at" + ); } #[test] - fn hashes_are_keyed_and_namespaced() { - let keys = keys(); - let provider: IdentityProviderId = "p".parse().expect("id"); - let iv = invalidated_uid_hash(&keys, &provider, "u"); - let visitor = visitor_key(&keys, "u"); - assert_eq!(iv.len(), 16, "iv length"); - assert_ne!(iv, visitor, "different labels give different hashes for the same input"); + fn uid_hashes_are_unkeyed_sha256_prefixes() { + let expected = hex::encode(&Sha256::digest(b"example-uid")[..8]); + assert_eq!(uid_hash("example-uid"), expected, "should be the first 16 hex digits of SHA-256"); let mut other_settings = create_test_settings(); other_settings.ec.passphrase = crate::redacted::Redacted::new("another-example-passphrase-value".to_owned()); - let other = IdentityKeys::from_settings(&other_settings); - assert_ne!(invalidated_uid_hash(&other, &provider, "u"), iv, "keyed by the passphrase"); + assert_ne!( + visitor_key(&IdentityKeys::from_settings(&other_settings), "ec"), + visitor_key(&IdentityKeys::from_settings(&create_test_settings()), "ec"), + "should key only the visitor key by the passphrase" + ); + } + + #[test] + fn a_removal_hashes_every_uid_of_the_set_and_push_moves_repeats_to_the_end() { + let set = vec![ + KvEid { uids: vec![uid("a"), uid("b")], ..KvEid::default() }, + KvEid { uids: vec![uid("c"), uid("a")], ..KvEid::default() }, + ]; + + let hashes = set_hashes(&set); + + let expected: Vec = ["a", "b", "c"].iter().map(|id| uid_hash(id)).collect(); + assert_eq!(hashes, expected, "should hash each UID once, in set order"); + assert_eq!( + set_hashes(&KvPartnerId::new("u").eids), + vec![uid_hash("u")], + "should cost one hash for a one-UID record" + ); + + let mut list = vec![uid_hash("a"), uid_hash("z")]; + push_hashes(&mut list, vec![uid_hash("a"), uid_hash("y")]); + assert_eq!(list, vec![uid_hash("z"), uid_hash("a"), uid_hash("y")], "should move a repeat to the end"); + } + + #[test] + fn state_revisions_tell_absent_and_changed_state_apart() { + let state = json!({ "token": "A0" }); + assert_eq!(StateRevision::of(Some(&state)), StateRevision::of(Some(&state.clone())), "should be stable"); + assert_ne!( + StateRevision::of(Some(&state)), + StateRevision::of(Some(&json!({ "token": "A1" }))), + "should change with the state" + ); + assert_ne!(StateRevision::of(None), StateRevision::of(Some(&json!(""))), "should tell absent from empty"); } } ``` @@ -4430,27 +5855,24 @@ Expected: compile errors. - [ ] **Step 3: Implement** ```rust -//! Record revisions and keyed hashes of identifiers. +//! Record revisions, state revisions, UID hashes and the visitor key. use hmac::{Hmac, Mac as _}; -use sha2::Sha256; +use sha2::{Digest as _, Sha256}; -use super::ids::IdentityProviderId; use super::input::RandomSource; -use crate::ec::kv_types::KvPartnerId; +use crate::ec::kv_types::{KvEid, KvPartnerId}; use crate::settings::Settings; type HmacSha256 = Hmac; -/// Keys derived from `ec.passphrase`, once per settings load. +/// The visitor key, derived from `ec.passphrase` once per settings load. +/// +/// It is the only keyed value: nothing stored in the row is keyed +/// (spec 9.3). #[derive(Clone)] pub(crate) struct IdentityKeys { - revision: [u8; 32], - invalidated: [u8; 32], visitor: [u8; 32], - fingerprint: [u8; 32], - pub(crate) cookie_s: [u8; 32], - pub(crate) cookie_r: [u8; 32], } impl core::fmt::Debug for IdentityKeys { @@ -4459,45 +5881,27 @@ impl core::fmt::Debug for IdentityKeys { } } -fn derive(passphrase: &[u8], label: &[u8]) -> [u8; 32] { - let mut mac = HmacSha256::new_from_slice(passphrase).expect("should accept any HMAC key length"); - mac.update(label); - mac.finalize().into_bytes().into() -} - -fn keyed_hex16(key: &[u8; 32], parts: &[&[u8]]) -> String { - let mut mac = HmacSha256::new_from_slice(key).expect("should accept a 32-byte key"); - for (index, part) in parts.iter().enumerate() { - if index > 0 { - mac.update(b"\0"); - } - mac.update(part); - } - hex::encode(&mac.finalize().into_bytes()[..8]) -} - impl IdentityKeys { - /// Derives every identity key from `ec.passphrase`. + /// Derives the visitor key from `ec.passphrase`. #[must_use] pub(crate) fn from_settings(settings: &Settings) -> Self { let passphrase = settings.ec.passphrase.expose().as_bytes(); - Self { - revision: derive(passphrase, b"trusted-server/identity/revision/v1"), - invalidated: derive(passphrase, b"trusted-server/identity/iv/v1"), - visitor: derive(passphrase, b"trusted-server/identity/visitor/v1"), - fingerprint: derive(passphrase, b"trusted-server/identity/fingerprint/v1"), - cookie_s: derive(passphrase, b"trusted-server/ts-eids-s/key/v1"), - cookie_r: derive(passphrase, b"trusted-server/ts-eids-r/key/v1"), - } + let mut mac = HmacSha256::new_from_slice(passphrase).expect("should accept any HMAC key length"); + mac.update(b"trusted-server/identity/visitor/v1"); + Self { visitor: mac.finalize().into_bytes().into() } } } -/// A record's opaque revision: 16 lowercase hex digits. +fn hex16(digest: &[u8]) -> String { + hex::encode(&digest[..8]) +} + +/// A record's stored revision: 16 lowercase hex digits. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) struct Revision(String); +pub(crate) struct RecordRevision(String); -impl Revision { - /// Wraps a stored `rv` or `pv`. Callers have validated the format. +impl RecordRevision { + /// Wraps a stored `rv`. `KvEntry::validate` has checked the format. #[must_use] pub(crate) fn from_stored(value: &str) -> Self { Self(value.to_owned()) @@ -4510,45 +5914,84 @@ impl Revision { } } +/// A record's revision (spec 5.4). `Unset` for a record without `rv`, which +/// only a schema version 1 row holds, until its next write gives it one. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) enum Revision { + Unset, + Set(RecordRevision), +} + /// A fresh random revision, drawn on every write of a record. #[must_use] -pub(crate) fn new_revision(random: &dyn RandomSource) -> Revision { +pub(crate) fn new_revision(random: &dyn RandomSource) -> RecordRevision { let mut bytes = [0u8; 8]; random.fill(&mut bytes); - Revision(hex::encode(bytes)) + RecordRevision(hex::encode(bytes)) } -/// The revision of a stored record: its `rv`, or for a legacy record without -/// one, the keyed hash of its `(uid, at)` until its next write. +/// The revision of a stored record. One revision covers the source's whole +/// entry set. #[must_use] -pub(crate) fn record_revision(keys: &IdentityKeys, record: &KvPartnerId) -> Revision { - if let Some(rv) = &record.rv { - return Revision(rv.clone()); +pub(crate) fn record_revision(record: &KvPartnerId) -> Revision { + record.rv.as_deref().map_or(Revision::Unset, |rv| Revision::Set(RecordRevision::from_stored(rv))) +} + +/// The hash of a provider's `ps.st` a call read, so a later apply can tell +/// whether another writer changed it since. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct StateRevision(String); + +impl StateRevision { + /// The first 16 hex digits of the SHA-256 of `st` as compact JSON, or of + /// the empty string when there is none. Compact JSON is never empty, so + /// absent and present state never share a revision. + #[must_use] + pub(crate) fn of(st: Option<&serde_json::Value>) -> Self { + let bytes = st.and_then(|value| serde_json::to_vec(value).ok()).unwrap_or_default(); + Self(hex16(&Sha256::digest(bytes))) } - let at = record.at.unwrap_or_default().to_string(); - Revision(keyed_hex16(&keys.revision, &[record.uid.as_bytes(), at.as_bytes()])) } -/// The keyed hash stored in `ps.iv` for a UID the owner invalidated. +/// The hash stored in `ps.iv` or `ps.ov` for a removed UID: the first 16 hex +/// digits of its unkeyed SHA-256 (spec 6.1). #[must_use] -pub(crate) fn invalidated_uid_hash(keys: &IdentityKeys, provider: &IdentityProviderId, uid: &str) -> String { - keyed_hex16(&keys.invalidated, &[provider.as_str().as_bytes(), uid.as_bytes()]) +pub(crate) fn uid_hash(uid: &str) -> String { + hex16(&Sha256::digest(uid.as_bytes())) } -/// The per-visitor rate-limit key component, from the full EC ID. +/// One hash per UID of a removed set, in set order, without duplicates. #[must_use] -pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String { - keyed_hex16(&keys.visitor, &[ec_id.as_bytes()]) +pub(crate) fn set_hashes(set: &[KvEid]) -> Vec { + let mut hashes = Vec::new(); + for uid in set.iter().flat_map(|entry| &entry.uids) { + let hash = uid_hash(&uid.id); + if !hashes.contains(&hash) { + hashes.push(hash); + } + } + hashes } -/// The consent fingerprint over the signals a provider's policy reads. +/// Appends hashes to a `ps.iv` or `ps.ov` list, oldest first. A hash +/// already present moves to the end. The caller truncates. +pub(crate) fn push_hashes(list: &mut Vec, hashes: Vec) { + for hash in hashes { + list.retain(|existing| *existing != hash); + list.push(hash); + } +} + +/// The per-visitor rate-limit key component, from the full EC ID. #[must_use] -pub(crate) fn consent_fingerprint(keys: &IdentityKeys, bits: &str) -> String { - keyed_hex16(&keys.fingerprint, &[bits.as_bytes()]) +pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String { + let mut mac = HmacSha256::new_from_slice(&keys.visitor).expect("should accept a 32-byte key"); + mac.update(ec_id.as_bytes()); + hex16(&mac.finalize().into_bytes()) } ``` -Check that `Settings.ec.passphrase` is `Redacted` with `.expose()`. `pull_sync_marker.rs:293` uses exactly that. Register the module in `mod.rs` (`pub(crate) mod revision;`) and in `migration_guards.rs`. +Check that `Settings.ec.passphrase` is `Redacted` with `.expose()`. `pull_sync_marker.rs:293` uses exactly that. `sha2`, `hmac` and `hex` are already dependencies of core (check with `grep -n '^sha2\|^hmac\|^hex' crates/trusted-server-core/Cargo.toml`). Register the module in `mod.rs` (`pub(crate) mod revision;`) and in `migration_guards.rs`. - [ ] **Step 4: Run and see them pass** @@ -4559,7 +6002,7 @@ Expected: PASS. ```bash git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs -git commit --signoff -S -m "Add record revisions and keyed identity hashes" +git commit --signoff -S -m "Add record revisions, state revisions and UID hashes" -m "A removal hashes every UID of the record it removes with an unkeyed SHA-256, since the row already holds the UIDs in plain text. A record without rv has the revision Unset. Only the rate-limit visitor key is derived from ec.passphrase." ``` --- @@ -4570,7 +6013,9 @@ git commit --signoff -S -m "Add record revisions and keyed identity hashes" - Create: `crates/trusted-server-core/src/ec/identity/catalog.rs` - Create: `crates/trusted-server-core/src/ec/identity/servable.rs` -- Modify: `crates/trusted-server-core/src/ec/eids.rs` (`resolve_partner_ids`, and the stale module comment at lines 1-5) +- Create: `crates/trusted-server-core/src/ec/identity/test_support.rs` +- Modify: `crates/trusted-server-core/src/ec/eids.rs` (`ResolvedPartnerId`, `resolve_partner_ids`, `to_eids`, and the stale module comment at lines 1-5) +- Modify: `crates/trusted-server-core/src/openrtb.rs` (the `Eid` doc comment at about line 69 says KV-resolved entries leave the EID-level fields empty) - Modify: callers of `resolve_partner_ids`: `auction/endpoints.rs:465`, `ec/admin.rs:485` - Modify: `crates/trusted-server-core/src/ec/identify.rs` (`entry.ids` read at about lines 99-103) - Modify: `mod.rs`, `migration_guards.rs` @@ -4578,19 +6023,26 @@ git commit --signoff -S -m "Add record revisions and keyed identity hashes" **Interfaces:** +- Consumes: `KvPartnerId`, `KvEid`, `KvUid`, `first_uid()` (Task 6) - Produces: - - `pub(crate) struct ProviderCatalog` with `pub(crate) fn new(providers: Vec>, owners: BTreeMap) -> Self`, `get(&IdentityProviderId) -> Option<&Arc>`, `providers() -> impl Iterator>`, `owner_of(&SourceDomain) -> Option<&IdentityProviderId>`, `is_configured(&IdentityProviderId) -> bool`, `configured_sources() -> &[SourceDomain]` (sorted and de-duplicated, the order that `ts-eids-s` indexes use), `fingerprint() -> &str` (`cf`: the first 8 hex digits of SHA-256 over the sorted `provider|source|owner` lines), and `ProviderCatalog::empty()`. Part 1 builds it from generic pull providers only (Task 22), with no owners. + - `pub(crate) struct ProviderCatalog` with `pub(crate) fn new(providers: Vec>, owners: BTreeMap) -> Self`, `get(&IdentityProviderId) -> Option<&Arc>`, `providers() -> impl Iterator>`, `owner_of(&SourceDomain) -> Option<&IdentityProviderId>`, `is_configured(&IdentityProviderId) -> bool`, and `ProviderCatalog::empty()`. Part 1 builds it from generic pull providers only (Task 22), with no owners. + - Narrowed sources (spec 10.1): `with_narrowing(self, narrowing: BTreeMap>) -> Self` keeps, for each listed provider, only the listed sources its descriptor declares. `sources_of(&IdentityProviderId) -> &[SourceDomain]` returns the narrowed list, else the descriptor's. Every other accessor reads sources through it, so a provider never claims a source it was narrowed away from. + - `claimants(&SourceDomain) -> impl Iterator`: every provider whose narrowed sources include the source. Task 28's `iv` and `ov` denials read the providers that claim a source through it. + - `is_capture_only(&IdentityProviderId) -> bool` (a descriptor with `captures` and no `resolves`) and `capture_only_sources(&IdentityProviderId) -> &[SourceDomain]` (its narrowed sources, empty for any other provider). An opt-out of a capture-only provider also removes the records of these sources that no provider wrote (spec 5.4), and its sticky `ps.o` denies the browser's value for them (spec 7.1). - `pub(crate) struct ServeContext<'a> { now: UnixTime, consent: &'a ConsentContext, signals: PolicySignals, catalog: &'a ProviderCatalog, opted_out: &'a BTreeSet }` - - `pub(crate) fn is_servable(context: &ServeContext<'_>, source: &SourceDomain, record: &KvPartnerId) -> bool` - - `resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry, serve: &ServeContext<'_>) -> Vec`. The new third argument is required. `ResolvedPartnerId` gains `atype` from the record (`record.atype.unwrap_or(partner.openrtb_atype)`), `ext: Option`, and `writer: Option`. + - `pub(crate) fn is_servable(context: &ServeContext<'_>, source: &SourceDomain, record: &KvPartnerId) -> bool`. A record is the source's whole entry set, so the set is servable or not as a whole. No check looks at one entry or one UID. + - `pub(crate) fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry, serve: &ServeContext<'_>) -> Vec`. The new third argument is required. It drops from `pub` to `pub(crate)`: its only callers are in core (`auction/endpoints.rs:465`, `ec/admin.rs:485`), and a `pub` function naming the crate-private `ServeContext` trips `private_interfaces` under `-D warnings`. `ResolvedPartnerId` keeps `source_domain`, `uid` (the first UID of the first stored entry, as before for a one-UID record) and `openrtb_atype` (the partner's), and gains `entries: Vec` (the record's `eids`, one entry for a one-UID record) and `writer: Option`. + - `to_eids` emits one `Eid` per stored entry, in stored order. Each UID uses its stored `atype`, else `openrtb_atype`, and its stored UID-level `ext`. Each entry keeps its stored `inserter`, `matcher`, `mm` and EID-level `ext`. A one-UID record without `atype` or `ext`, which is every record on `main`, gives exactly the `Eid` it gives today. Task 28 fills the EID-level fields of a provider-written entry that sets none of them from `with_provenance`. + - `x-ts-eids` truncation stays per entry, as today: `encode_eids_header` (`ec/eids.rs:106`) and `build_eids_header` are unchanged. The header also carries the browser's entries, whose truncation must not change, so it can hold part of a stored set; `user.eids` always holds the set whole (spec 7.1). + - `ec/identity/test_support.rs`, declared as `#[cfg(any(test, feature = "test-utils"))] pub mod test_support;`: `test_consent_allowing_everything`, `test_consent_gdpr_without_vendor`, `example_entry_set() -> Vec` (spec 6.1's two entries and three UIDs: `example-a` and `example-b` with `inserter` and `mm`, then `example-c` with `matcher`) and `full_record(writer: Option<&str>, set: Vec) -> KvPartnerId`, all `pub`. Later tasks add to it. The rule (spec 6.1 and 7.1), applied in this order: -1. An empty UID is not servable. -2. `exp` at or before `now` is not servable. +1. A record with an empty `eids` is not servable. `KvEntry::validate` never lets one through, so this only guards a caller that builds a record by hand. +2. `exp` at or before `now` is not servable. The set's `exp` is the earliest any of its entries set (spec 4.5). 3. A record with a writer `w`: - The writer must be in the catalog. A disabled or unconfigured writer means not servable. - - The writer must not be in `opted_out`, which is the union of KV `ps.o` and the `ts-eids-r` opt-outs. The caller computes it. + - The writer must not be in `opted_out`: the providers with a `ps.o` in the row the request read. The caller computes it from KV only (spec 7.1). - The writer's consent policy must pass under this request's consent (`policy_allows`). 4. An owned source (the catalog has an owner): - The owner's policy must pass. @@ -4608,8 +6060,11 @@ mod tests { use std::sync::Arc; use super::*; - use crate::ec::identity::descriptor::{ConsentPolicy, ProviderDescriptor, Resolves}; + use crate::ec::identity::descriptor::{Captures, ConsentPolicy, ProviderDescriptor, Resolves}; use crate::ec::identity::provider::IdentityProvider; + use crate::ec::identity::test_support::{ + example_entry_set, full_record, test_consent_allowing_everything, test_consent_gdpr_without_vendor, + }; use crate::ec::kv_types::KvPartnerId; struct Fixed(ProviderDescriptor); @@ -4620,8 +6075,8 @@ mod tests { } fn provider(id: &str, source: &str, policy: ConsentPolicy, expiring: bool) -> Arc { - let mut builder = ProviderDescriptor::builder(id.parse().expect("id")) - .with_source(source.parse().expect("source")) + let mut builder = ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_source(source.parse().expect("should parse the source")) .with_resolves(Resolves::new().with_endpoint("https://api.example.com")) .with_consent_policy(policy); if expiring { @@ -4643,7 +6098,7 @@ mod tests { catalog, opted_out, }; - is_servable(&context, &source.parse().expect("source"), record) + is_servable(&context, &source.parse().expect("should parse the source"), record) } #[test] @@ -4653,25 +6108,65 @@ mod tests { BTreeMap::new(), ); let none = BTreeSet::new(); - assert!(check(&catalog, &none, "a.example.com", &record(Some("p"), Some(101))), "control: valid"); - assert!(!check(&catalog, &none, "a.example.com", &record(Some("p"), Some(100))), "expired at exp"); - assert!(!check(&catalog, &none, "a.example.com", &record(Some("gone"), None)), "unknown writer"); - let opted: BTreeSet<_> = ["p".parse().expect("id")].into_iter().collect(); - assert!(!check(&catalog, &opted, "a.example.com", &record(Some("p"), None)), "opted-out writer"); - assert!(check(&catalog, &none, "b.example.com", &record(None, None)), "browser record, unowned"); + assert!(check(&catalog, &none, "a.example.com", &record(Some("p"), Some(101))), "should serve a valid record"); + assert!(!check(&catalog, &none, "a.example.com", &record(Some("p"), Some(100))), "should treat now == exp as expired"); + assert!(!check(&catalog, &none, "a.example.com", &record(Some("gone"), None)), "should not serve an unknown writer"); + let opted: BTreeSet<_> = ["p".parse().expect("should parse the provider ID")].into_iter().collect(); + assert!(!check(&catalog, &opted, "a.example.com", &record(Some("p"), None)), "should not serve an opted-out writer"); + assert!(check(&catalog, &none, "b.example.com", &record(None, None)), "should serve an unowned browser record"); + } + + #[test] + fn a_full_record_is_servable_or_not_as_a_whole() { + let catalog = ProviderCatalog::new( + vec![provider("p", "a.example.com", ConsentPolicy::eid_persistence_only(), false)], + BTreeMap::new(), + ); + let none = BTreeSet::new(); + let valid = KvPartnerId { exp: Some(101), ..full_record(Some("p"), example_entry_set()) }; + let expired = KvPartnerId { exp: Some(100), ..full_record(Some("p"), example_entry_set()) }; + assert!(check(&catalog, &none, "a.example.com", &valid), "should serve the whole valid set"); + assert!(!check(&catalog, &none, "a.example.com", &expired), "should never serve an expired set"); } #[test] fn rollback_rule_for_expiring_owned_sources() { - let owner: IdentityProviderId = "owner".parse().expect("id"); + let owner: IdentityProviderId = "owner".parse().expect("should parse the provider ID"); let catalog = ProviderCatalog::new( vec![provider("owner", "a.example.com", ConsentPolicy::eid_persistence_only(), true)], - [("a.example.com".parse().expect("source"), owner)].into_iter().collect(), + [("a.example.com".parse().expect("should parse the source"), owner)].into_iter().collect(), ); let none = BTreeSet::new(); - assert!(check(&catalog, &none, "a.example.com", &record(Some("owner"), Some(200))), "control: owner value"); - assert!(!check(&catalog, &none, "a.example.com", &record(None, None)), "browser-seeded, never from server"); - assert!(!check(&catalog, &none, "a.example.com", &record(Some("owner"), None)), "stripped exp"); + assert!(check(&catalog, &none, "a.example.com", &record(Some("owner"), Some(200))), "should serve the owner's value"); + assert!(!check(&catalog, &none, "a.example.com", &record(None, None)), "should never serve a browser-seeded value from the server"); + assert!(!check(&catalog, &none, "a.example.com", &record(Some("owner"), None)), "should not serve a value with a stripped exp"); + } + + #[test] + fn narrowing_limits_claims_and_capture_only_sources() { + let capture_only: Arc = Arc::new(Fixed( + ProviderDescriptor::builder("c".parse().expect("should parse the provider ID")) + .with_source("a.example.com".parse().expect("should parse the source")) + .with_source("b.example.com".parse().expect("should parse the source")) + .with_captures(Captures::new().with_path_prefix("/integrations/c/")) + .build() + .expect("should build"), + )); + let resolver = provider("p", "a.example.com", ConsentPolicy::eid_persistence_only(), false); + let a: SourceDomain = "a.example.com".parse().expect("should parse the source"); + let b: SourceDomain = "b.example.com".parse().expect("should parse the source"); + let c: IdentityProviderId = "c".parse().expect("should parse the provider ID"); + let p: IdentityProviderId = "p".parse().expect("should parse the provider ID"); + + let catalog = ProviderCatalog::new(vec![capture_only, resolver], BTreeMap::new()) + .with_narrowing([(c.clone(), vec![b.clone()])].into_iter().collect()); + + assert_eq!(catalog.sources_of(&c), &[b.clone()], "should narrow the capture-only provider"); + assert!(catalog.is_capture_only(&c) && !catalog.is_capture_only(&p), "should tell capture-only apart"); + assert_eq!(catalog.capture_only_sources(&c), &[b.clone()], "should list its narrowed sources"); + assert!(catalog.capture_only_sources(&p).is_empty(), "should list none for a resolver"); + assert_eq!(catalog.claimants(&a).collect::>(), vec![&p], "should drop a narrowed-away claim"); + assert_eq!(catalog.claimants(&b).collect::>(), vec![&c], "should keep the narrowed claim"); } #[test] @@ -4687,39 +6182,126 @@ mod tests { catalog: &catalog, opted_out: &none, }; - let source = "a.example.com".parse().expect("source"); - assert!(!is_servable(&context, &source, &record(Some("p"), None)), "writer's vendor denied"); - assert!(is_servable(&context, &source, &record(None, None)), "control: browser value unaffected"); + let source = "a.example.com".parse().expect("should parse the source"); + assert!(!is_servable(&context, &source, &record(Some("p"), None)), "should not serve when the writer's vendor is denied"); + assert!(is_servable(&context, &source, &record(None, None)), "should leave a browser value unaffected"); } } ``` -`test_consent_allowing_everything` and `test_consent_gdpr_without_vendor` reuse the helper you wrote in Task 12. Move it to `ec/identity/test_support.rs` now (create the file with `#![cfg(any(test, feature = "test-utils"))]` items and `pub(crate) mod test_support;` in `mod.rs`) so every identity test can share it. - -In `ec/eids.rs` tests, extend the existing `resolve_partner_ids` test (line 195) to pass a `ServeContext` built from `ProviderCatalog::empty()`, and add: +`test_consent_allowing_everything` and `test_consent_gdpr_without_vendor` reuse the helper you wrote in Task 12. Move it to `ec/identity/test_support.rs` now so every identity test can share it. Declare the module in `mod.rs` as `#[cfg(any(test, feature = "test-utils"))] pub mod test_support;`, and keep it public from the start, as the crate's other `test-utils` items are (`IntegrationRegistry::from_request_filters`): adapter tests reach it in Part 1 (Task 30). A helper whose signature names only public types (`ConsentContext`, `KvEid`, `KvPartnerId`, `IdentityProvider`) is `pub` with a doc comment. A helper that takes or returns a crate-private type (`ProviderCatalog`, `IdentityKeys`) stays `pub(crate)`, because a `pub` function naming a `pub(crate)` type trips `private_interfaces` under `-D warnings`. Start the file with `#![cfg_attr(not(test), allow(dead_code, reason = "the test-utils library build compiles the crate-private helpers without their unit-test callers"))]`, so `cargo clippy-fastly`, which enables `test-utils` for the adapter's dev-dependency, does not flag them. Add the entry-set helpers next to it: ```rust - #[test] - fn resolve_partner_ids_skips_expired_records_and_uses_stored_atype() { - let registry = registry_with(&["a.example.com", "b.example.com"]); - let mut entry = KvEntry::minimal("a.example.com", "ua", 1); - entry.ids.insert( +/// Spec 6.1's example set: two entries, three UIDs, with UID-level and +/// EID-level fields. +#[must_use] +pub fn example_entry_set() -> Vec { + vec![ + KvEid { + uids: vec![ + KvUid { id: "example-a".to_owned(), atype: Some(1), ext: None }, + KvUid { id: "example-b".to_owned(), atype: Some(3), ext: Some(json!({ "stype": "ppuid" })) }, + ], + inserter: Some("example.com".to_owned()), + mm: Some(3), + ..KvEid::default() + }, + KvEid { + uids: vec![KvUid { id: "example-c".to_owned(), atype: Some(1), ext: None }], + matcher: Some("partner.example.com".to_owned()), + ..KvEid::default() + }, + ] +} + +/// A record holding `set`, written by `writer`. +#[must_use] +pub fn full_record(writer: Option<&str>, set: Vec) -> KvPartnerId { + KvPartnerId { w: writer.map(str::to_owned), eids: set, ..KvPartnerId::default() } +} +``` + +In `ec/eids.rs` tests, extend the existing `resolve_partner_ids` test (line 195) to pass a `ServeContext` built from `ProviderCatalog::empty()`, and add: + +```rust + #[test] + fn resolve_partner_ids_skips_expired_records_and_uses_stored_atype() { + let registry = registry_with(&["a.example.com", "b.example.com"]); + let mut entry = KvEntry::minimal("a.example.com", "ua", 1); + entry.ids.insert( "b.example.com".to_owned(), KvPartnerId { exp: Some(50), ..KvPartnerId::new("ub") }, ); - entry.ids.get_mut("a.example.com").expect("a").atype = Some(3); + entry.ids.get_mut("a.example.com").expect("should hold source a").eids[0].uids[0].atype = Some(3); let resolved = resolve_partner_ids(®istry, &entry, &serve_context_at(100)); assert_eq!(resolved.len(), 1, "should drop the expired b"); assert_eq!(resolved[0].source_domain, "a.example.com", "should keep a"); - assert_eq!(resolved[0].openrtb_atype, 3, "should prefer the stored atype"); + assert_eq!(to_eids(&resolved)[0].uids[0].atype, Some(3), "should prefer the stored atype"); + } + + #[test] + fn a_one_uid_record_goes_out_as_on_main() { + let registry = registry_with(&["a.example.com"]); + let entry = KvEntry::minimal("a.example.com", "ua", 1); + + let eids = to_eids(&resolve_partner_ids(®istry, &entry, &serve_context_at(100))); + + assert_eq!( + eids, + vec![Eid::new( + "a.example.com".to_owned(), + vec![Uid { id: "ua".to_owned(), atype: Some(99), ext: None }], + )], + "should be the EID main sends" + ); + } + + #[test] + fn a_full_record_goes_out_as_one_eid_per_stored_entry() { + let registry = registry_with(&["a.example.com"]); + let mut set = example_entry_set(); + set[1].uids[0].atype = None; + let mut entry = KvEntry::minimal("b.example.com", "unregistered", 1); + entry.ids.insert("a.example.com".to_owned(), full_record(None, set)); + + let eids = to_eids(&resolve_partner_ids(®istry, &entry, &serve_context_at(100))); + + assert_eq!( + eids, + vec![ + Eid { + source: "a.example.com".to_owned(), + uids: vec![ + Uid { id: "example-a".to_owned(), atype: Some(1), ext: None }, + Uid { id: "example-b".to_owned(), atype: Some(3), ext: Some(json!({ "stype": "ppuid" })) }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + Eid { + source: "a.example.com".to_owned(), + uids: vec![Uid { id: "example-c".to_owned(), atype: Some(99), ext: None }], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ], + "should send every entry in stored order, a UID without atype taking the partner's" + ); } ``` -Write `registry_with` and `serve_context_at` as local helpers. `registry_with` follows the existing test's registry setup, and `serve_context_at` uses `ProviderCatalog::empty()` with permissive consent. +Write `registry_with` and `serve_context_at` as local helpers. `registry_with` follows the existing test's registry setup and gives every partner `openrtb_atype = 99`, so a fallback is visible. `serve_context_at` uses `ProviderCatalog::empty()` with permissive consent. The full record above has no writer, because an empty catalog holds no writer for it to pass. -In `ec/identify.rs` tests, add one that stores a row with an expired record and a valid one and checks that the identify response lists only the valid one. Copy the setup from the nearest existing identify test (`grep -n "fn .*identify" crates/trusted-server-core/src/ec/identify.rs`). +In `ec/identify.rs` tests, add two. Copy the setup from the nearest existing identify test (`grep -n "fn .*identify" crates/trusted-server-core/src/ec/identify.rs`): + +- A row with an expired record and a valid one: the identify response lists only the valid one. +- A row whose partner source holds `full_record(None, example_entry_set())`: the response's `uid` is `example-a`, and its `eid` is `Eid::new(source, vec![Uid { id: "example-a", atype: Some(partner.openrtb_atype), ext: None }])`, the shape a one-UID record gives today. `/identify` never returns a full set (spec 7.1). - [ ] **Step 2: Run and see them fail** @@ -4734,19 +6316,16 @@ Expected: compile errors, then the identify test fails because it returns the ex use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; -use sha2::{Digest as _, Sha256}; - use super::ids::{IdentityProviderId, SourceDomain}; use super::provider::IdentityProvider; -/// Enabled providers, source owners, and the derived source index and `cf`. -/// Built once per settings load. +/// Enabled providers, their narrowed sources and source owners. Built once +/// per settings load. #[derive(Clone)] pub(crate) struct ProviderCatalog { providers: BTreeMap>, + sources: BTreeMap>, owners: BTreeMap, - sources: Vec, - fingerprint: String, } impl ProviderCatalog { @@ -4767,22 +6346,24 @@ impl ProviderCatalog { .into_iter() .map(|provider| (provider.descriptor().id().clone(), provider)) .collect(); - let sources: Vec = providers - .values() - .flat_map(|provider| provider.descriptor().sources().iter().cloned()) - .chain(owners.keys().cloned()) - .collect::>() - .into_iter() + let sources = providers + .iter() + .map(|(id, provider)| (id.clone(), provider.descriptor().sources().to_vec())) .collect(); - let mut hasher = Sha256::new(); - for (id, provider) in &providers { - for source in provider.descriptor().sources() { - let owner = owners.get(source).map(IdentityProviderId::as_str).unwrap_or_default(); - hasher.update(format!("{id}|{source}|{owner}\n").as_bytes()); + Self { providers, sources, owners } + } + + /// Narrows providers to the listed sources their descriptors declare + /// (spec 10.1). Config validation has rejected a source a descriptor + /// does not declare and a narrowing to nothing. + #[must_use] + pub(crate) fn with_narrowing(mut self, narrowing: BTreeMap>) -> Self { + for (id, listed) in narrowing { + if let Some(sources) = self.sources.get_mut(&id) { + sources.retain(|source| listed.contains(source)); } } - let fingerprint = hex::encode(&hasher.finalize()[..4]); - Self { providers, owners, sources, fingerprint } + self } /// A provider by ID. @@ -4808,17 +6389,30 @@ impl ProviderCatalog { self.owners.get(source) } - /// Every configured source, sorted. Indexes into this list are the - /// `s` values of `ts-eids-s`. + /// A provider's sources after narrowing. Empty for an unknown provider. #[must_use] - pub(crate) fn configured_sources(&self) -> &[SourceDomain] { - &self.sources + pub(crate) fn sources_of(&self, id: &IdentityProviderId) -> &[SourceDomain] { + self.sources.get(id).map_or(&[], Vec::as_slice) + } + + /// Every provider that claims `source` after narrowing, ordered by ID. + pub(crate) fn claimants<'a>(&'a self, source: &'a SourceDomain) -> impl Iterator { + self.sources.iter().filter(move |(_, sources)| sources.contains(source)).map(|(id, _)| id) + } + + /// Whether `id` captures and never resolves. + #[must_use] + pub(crate) fn is_capture_only(&self, id: &IdentityProviderId) -> bool { + self.get(id).is_some_and(|provider| { + let descriptor = provider.descriptor(); + descriptor.captures().is_some() && descriptor.resolves().is_none() + }) } - /// `cf`: 8 hex digits over providers, sources and owners. + /// The narrowed sources of a capture-only provider. Empty for any other. #[must_use] - pub(crate) fn fingerprint(&self) -> &str { - &self.fingerprint + pub(crate) fn capture_only_sources(&self, id: &IdentityProviderId) -> &[SourceDomain] { + if self.is_capture_only(id) { self.sources_of(id) } else { &[] } } } ``` @@ -4843,14 +6437,15 @@ pub(crate) struct ServeContext<'a> { pub(crate) consent: &'a ConsentContext, pub(crate) signals: PolicySignals, pub(crate) catalog: &'a ProviderCatalog, - /// KV `ps.o` plus `ts-eids-r` opt-outs. + /// The providers with a `ps.o` in the row the request read. pub(crate) opted_out: &'a BTreeSet, } -/// Whether a stored record may be sent from the server. +/// Whether a stored record, the source's whole entry set, may be sent from +/// the server. #[must_use] pub(crate) fn is_servable(context: &ServeContext<'_>, source: &SourceDomain, record: &KvPartnerId) -> bool { - if record.uid.is_empty() { + if record.eids.is_empty() { return false; } if record.exp.is_some_and(|exp| context.now.as_secs() >= exp) { @@ -4888,22 +6483,78 @@ pub(crate) fn is_servable(context: &ServeContext<'_>, source: &SourceDomain, rec - [ ] **Step 5: Switch the readers** -In `ec/eids.rs`, replace the module comment at lines 1-5 with one that says KV IDs are filtered by the shared servability check in `ec::identity::servable`. Add `serve: &ServeContext<'_>` to `resolve_partner_ids`, and inside the loop add `if !is_servable(serve, &source, partner_uid) { continue; }` after parsing `source_domain` into a `SourceDomain`. Skip any key that does not parse. Take `atype` from the record and keep `ext` and the writer on `ResolvedPartnerId` (Task 28 adds provenance in `to_eids`). Update `to_eids` to emit the UID-level `ext` when present. +In `ec/eids.rs`, replace the module comment at lines 1-5 with one that says KV IDs are filtered by the shared servability check in `ec::identity::servable`, and that a stored record goes out as one EID per stored entry. Add `serve: &ServeContext<'_>` to `resolve_partner_ids`, and inside the loop add `if !is_servable(serve, &source, partner_uid) { continue; }` after parsing `source_domain` into a `SourceDomain`. Skip any key that does not parse. Then change `ResolvedPartnerId` and `to_eids`: -At each caller, build a `ServeContext`. Until Task 23 puts the catalog and `ts-eids-r` opt-outs on `EcContext`, pass `ProviderCatalog::empty()` and the KV `ps.o` keys as `opted_out`. Task 23 replaces both. `now` comes from `checked_current_timestamp()`. If that fails, resolve no KV IDs, because a zero time would make every expiry check pass (spec 4.4). `consent` is the request's `ConsentContext`. DNT is read from the request header there (`auction/routing.rs:336` shows how). +```rust +/// A source-domain keyed record resolved from a KV entry against the partner +/// registry and the servability check. +/// +/// Only includes partners with `bidstream_enabled = true`, a non-empty UID and +/// a servable record. +pub struct ResolvedPartnerId { + /// The partner's identity source domain and EC KV `ids` key. + pub source_domain: String, + /// The first UID of the first stored entry. + pub uid: String, + /// The partner's `OpenRTB` agent type, used for a UID without a stored one. + pub openrtb_atype: i32, + /// The source's stored entry set, in order. One entry for a one-UID record. + pub entries: Vec, + /// The provider that wrote the record. `None` for browser ingest and version 1 rows. + pub writer: Option, +} +``` + +In the loop, fill `uid` from `partner_uid.first_uid()`, `entries` with `partner_uid.eids.clone()` and `writer` with the parsed `w`. + +```rust +/// Converts resolved partner IDs to `OpenRTB` `Eid` entries: one per stored +/// entry, in stored order. +#[must_use] +pub fn to_eids(resolved: &[ResolvedPartnerId]) -> Vec { + resolved + .iter() + .flat_map(|item| item.entries.iter().map(move |entry| stored_entry_to_eid(item, entry))) + .collect() +} -In `ec/identify.rs`, replace the direct `entry.ids` read with `resolve_partner_ids`, using the same context built from the request's consent, never the row's stored `consent` (spec 7.1). +fn stored_entry_to_eid(item: &ResolvedPartnerId, entry: &KvEid) -> Eid { + let uids = entry + .uids + .iter() + .map(|uid| Uid { + id: uid.id.clone(), + atype: Some(uid.atype.unwrap_or(item.openrtb_atype)), + ext: uid.ext.clone(), + }) + .collect(); + Eid { + source: item.source_domain.clone(), + uids, + inserter: entry.inserter.clone(), + matcher: entry.matcher.clone(), + mm: entry.mm, + ext: entry.ext.clone(), + } +} +``` + +Leave `encode_eids_header` and `build_eids_header` as they are. They truncate per `Eid` entry, so a full set can be cut part-way in `x-ts-eids` and reach the 4 KB cap sooner; `x-ts-eids-truncated` is set as today (spec 7.1). `user.eids` is built from `to_eids` without that cap, so bidders always get the set whole. Update the `Eid` doc comment in `openrtb.rs`: KV-resolved entries carry the EID-level fields a provider stored, and a record from browser ingest or a version 1 row has none. + +At each caller, build a `ServeContext` with the KV `ps.o` keys of the row as `opted_out`. Until Task 23 puts the request's `IdentityRuntime` on `EcContext`, pass `ProviderCatalog::empty()`. Task 23 replaces it with the runtime's catalog, and adds `generic_pull_record_with_writer_is_served_after_flush`: from Task 29 on, generic pull records carry `w = pull-…`, which an empty catalog would never serve. `now` comes from `checked_current_timestamp()`. If that fails, resolve no KV IDs, because a zero time would make every expiry check pass (spec 4.4). `consent` is the request's `ConsentContext`. DNT is read from the request header there (`auction/routing.rs:336` shows how). In `ec/admin.rs`, `build_auction_view` reports a record the check drops in `skipped` with the reason `not_servable`, so the view still explains every stored ID it leaves out. + +In `ec/identify.rs`, keep the lookup of this partner's own source and gate it with `is_servable`, using the same context built from the request's consent, never the row's stored `consent` (spec 7.1). Do not route it through `resolve_partner_ids`, which also drops partners with `bidstream_enabled = false`; `/identify` returns those today. The response keeps its shape: `uid` is `record.first_uid()`, the first UID of the first entry, and `eid` is still built from it with `partner.openrtb_atype` and no `ext` (`ec/identify.rs:127-135`). - [ ] **Step 6: Run and see them pass** -Run: `cargo test-fastly -- ec::` and `cargo test-fastly -- auction::endpoints` +Run: `cargo test-fastly -- ec::` and `cargo test-fastly -- auction::` Expected: PASS. - [ ] **Step 7: Commit** ```bash git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Filter served KV IDs through one servability check" -m "Expired IDs, IDs from disabled or opted-out writers, and IDs whose writer or owner policy fails are no longer sent to bidders, the admin view or /identify. /identify uses the request's consent, never the row's stored snapshot." +git commit --signoff -S -m "Filter served KV IDs through one servability check" -m "Expired IDs, IDs from disabled or opted-out writers, and IDs whose writer or owner policy fails are no longer sent to bidders, the admin view or /identify. /identify uses the request's consent, never the row's stored snapshot." -m "A stored record goes out as one EID per stored entry. A one-UID record goes out exactly as before, x-ts-eids keeps its per-entry truncation, and /identify keeps returning one UID." ``` ### Task 15: `IdentityMutation` and its merge into a live row @@ -4911,274 +6562,405 @@ git commit --signoff -S -m "Filter served KV IDs through one servability check" **Files:** - Create: `crates/trusted-server-core/src/ec/identity/mutation.rs` -- Modify: `mod.rs`, `migration_guards.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs`, `mod.rs`, `migration_guards.rs` - Test: `mutation.rs` **Interfaces:** -- Consumes: `KvEntry`, `KvPartnerId`, `KvProviderState`, `KvOptOut` (Task 6), `Revision`, `new_revision`, `record_revision`, `invalidated_uid_hash`, `IdentityKeys` (Task 13), `ProviderCatalog` (Task 14), `RandomSource` (Task 9) +- Consumes: `KvEntry`, `KvPartnerId`, `KvProviderState`, `KvOptOut`, `MAX_INVALIDATED_UIDS` (128), `MAX_PROVIDER_STATES` (Task 6), `Revision`, `StateRevision`, `new_revision`, `record_revision`, `set_hashes`, `push_hashes` (Task 13), `ProviderCatalog` with `claimants`, `is_capture_only` and `capture_only_sources` (Task 14), `RandomSource` (Task 9) - Produces: - - `pub(crate) enum PartnerIdChange`: - - `FillMissing { source, record: KvPartnerId, writer: Option, staged_at: Option }` - - `OwnerWrite { source, record, writer, expected: Option, staged_at: Option }` - - `OwnerRemove { source, writer, expected: Revision }` - - `WriterRemove { source, writer, staged_at: Option }` - - `pub(crate) enum ExpectedState { NotRead, Absent, Json(String) }` - - `pub(crate) enum StateUpdate { Keep, Replace(serde_json::Value), Clear }` - - `pub(crate) struct ProviderStateChange { n: Option, f: Option, opt_out: Option, opt_out_clear: Option, iv_add: Vec, st: StateUpdate }`, deriving `Default` - - `pub(crate) struct ProviderResult { provider, ids: Vec, state: ProviderStateChange, expected_state: ExpectedState }` - - `pub(crate) struct IdentityMutation`, deriving `Default`, with `add_change(PartnerIdChange)`, `add_result(ProviderResult)`, `merge_state(IdentityProviderId, ProviderStateChange)`, `set_cluster_size(u32)`, `is_empty()`, `has_removals()` - - `pub(crate) struct ApplyContext<'a> { now: u64, keys: &'a IdentityKeys, random: &'a dyn RandomSource, catalog: &'a ProviderCatalog }` + - `pub(crate) enum PartnerIdChange` (spec 5.4): + - `FillMissing { source, record: KvPartnerId }`: from browser ingest (`record.w` is `None`) or an unowned provider (`record.w` names it) + - `OwnerWrite { source, record: KvPartnerId, expected: Option }`: from the owning provider, whose ID `record.w` holds + - `OwnerRemove { source, provider: IdentityProviderId, expected: Revision }`: from `Resolution::invalidate` or `also_invalidate`. `provider` is the provider that invalidated, whose `ps.iv` receives the hashes. Spec 5.4's enum leaves it out, but step 3 reads the `ps.iv` of every provider that claims a source, owned or unowned, so the change must say whose list it feeds. + - `WriterRemove { source, writer: IdentityProviderId }`: from `Resolution::opted_out`. Removes the source's record if `writer` wrote it, at any revision, and for a capture-only `writer` also a record no provider wrote. Its hashes go to the writer's `ps.ov`. + - There is no `staged_at` and no `invalidate` flag: nothing is staged in a cookie, so every change comes from this request's own KV read (spec 5.4). + - Each `record` is the source's whole record: its `eids`, `w`, the timing fields and `rv`. Every change writes or removes a source's set whole. No change edits one entry or one UID, so one `rv` per source covers every entry in it. + - `pub(crate) enum StateUpdate { Keep, Replace(serde_json::Value), Clear }`, defaulting to `Keep` + - `pub(crate) struct ProviderStateChange { n: Option, f: Option, opt_out: Option, opt_out_clear: Option, st: StateUpdate }`, deriving `Default`. It has no `iv` or `ov` field: those hashes come only from applied removals. + - `pub(crate) struct ProviderResult { provider, ids: Vec, state: ProviderStateChange, expected_state: Option }`. `None` means the call read no state, so the state check always passes. + - `pub(crate) struct IdentityMutation`, deriving `Default`, with `add_change(PartnerIdChange)`, `add_result(ProviderResult)`, `merge_state(IdentityProviderId, ProviderStateChange)`, `set_cluster_size(u32)`, `is_empty()`, `has_removals()` (any `OwnerRemove`, `WriterRemove`, opt-out or clear), and `removals(&self) -> IdentityMutation` (a copy holding only the removals, opt-outs and clears, which Task 18's removal write and Task 29's removal-only retry apply). Task 18 adds `removal_labels()`. + - `pub(crate) struct ApplyContext<'a> { now: u64, random: &'a dyn RandomSource, catalog: &'a ProviderCatalog }` - `pub(crate) struct Applied { entry: KvEntry, changed: bool, removals_only: bool }` - `pub(crate) fn apply(mutation: &IdentityMutation, live: &KvEntry, context: &ApplyContext<'_>) -> Option`. `None` means drop everything: the live row is a tombstone, or even the removals-only result failed validation. + - `pub(crate) fn apply_removals_only(mutation: &IdentityMutation, live: &KvEntry, context: &ApplyContext<'_>) -> Option`: spec 5.4's steps 1, 2, 4 (only `o`, `oc`, `iv` and `ov`), 5 and 6, for the capture removal write and the removal-only retry. It ignores every positive change, `n`, `f`, `st` and `cluster_size` in `mutation`. Apply order (spec 5.4). Copy it into the doc comment of `apply`: 1. A tombstone (`consent.ok == false`) gives `None`. -2. Removals. Every `OwnerRemove` whose `expected` equals the live record's revision. Every `WriterRemove` whose live record has `w == writer` and, when `staged_at` is set, `at <= staged_at` (equal seconds go to the removal). Each new opt-out applies when `at > live.o.at` (or there is no live `o`) and `at >= live.oc` (or there is no live `oc`). Applying it removes every live record that provider wrote, at any revision, and clears its `st`. Then each staged clear (`opt_out_clear`) that is strictly later than the provider's latest opt-out `at`, live or staged, is stored as the live `oc`. A clear that is not later is dropped. `iv_add` hashes are appended, de-duplicated, and trimmed to the newest 4. -3. Positive changes. Skip every change by a provider whose (post-step-2) live `o` is set and not cleared (`oc` strictly later than `o.at`), and skip a staged change whose `staged_at <= o.at` even after a clear. A `FillMissing` applies when the source is absent, or its record is expired, or its writer is not configured. It is skipped when its UID's hash is in the source owner's `ps.iv`. An `OwnerWrite` applies when the live revision equals `expected`, or when `expected` is `None` and the source has no valid record with `w == writer`. A written record gets `w = writer`, a fresh `rv`, and `at = record.at.or(now)`. Each `ProviderResult` applies only when every one of its ID changes would apply on its own and the live `ps.st` matches `expected_state`. Its state change then applies with its IDs. Otherwise the whole result is dropped. -4. `ps` changes: `n` takes the later value, `f` is replaced, `st` is replaced or cleared. Then `cluster_size`. Then prune the `ps` entries of providers the catalog does not hold. Then drop expired records, except refresh anchors: the source's owner holds a `st` and the record's `ru` has not passed. -5. `validate()`. On failure, rebuild from the live entry with only step 2 applied (removals, opt-outs, clears, `iv`) and validate again. If that also fails, give `None` and log. -6. `changed` is whether the result differs from `live`. +2. Removals, never skipped: + - Every `OwnerRemove` whose `expected` equals `record_revision(live_record)`. A version 1 record matches `Revision::Unset`. It appends `set_hashes(&live_record.eids)` to `ps[provider].iv`. + - Every `WriterRemove` whose live record has `w == writer`, or, when `writer` is capture-only and the source is one of its `capture_only_sources`, has no `w`. It appends the removed record's hashes to `ps[writer].ov`. + - Each new opt-out applies when `at > live.o.at` (or there is no live `o`) and `at >= live.oc` (or there is no live `oc`). Applying it stores `o` and removes every live record that provider wrote, at any revision, and for a capture-only provider every record of its sources that no provider wrote, each adding its hashes to `ps.ov`. + - Then each clear (`opt_out_clear`) that is strictly later than the provider's live `o.at` is stored as `oc` and empties `ps.ov`. A clear that is not later, or that finds no live opt-out, is dropped and writes nothing. `ps.iv` is never emptied. + - Hashes always come from the record KV held. A removal that no longer applies adds none. +3. Positive changes: + - A provider is opted out when its live `o` is set and its `oc` is not strictly later than `o.at`. Every change whose `record.w` is opted out is skipped, `OwnerWrite` and `FillMissing` alike. + - A `FillMissing` applies when the source is absent, or its record is expired, or its record's writer is not configured. It is skipped when the hash of any UID of its set is in the `ps.iv` or `ps.ov` of any provider in `catalog.claimants(source)`. A browser `FillMissing` (`record.w` is `None`) is also skipped when the source belongs to an opted-out capture-only provider's `capture_only_sources`. + - An `OwnerWrite` applies when the live revision equals `expected`, or when `expected` is `None` and the source has no valid record with `w == record.w`. A browser-seeded value never blocks the owner. + - A written record replaces the live record whole, never merges into it, so no entry of the old set survives. It gets a fresh `rv` and `at = record.at.or(now)`, and keeps the change's `w`. + - Each `ProviderResult` applies only when every one of its ID changes would apply on its own and, when `expected_state` is `Some`, `StateRevision::of(live ps.st)` equals it. Its state change, `n` and `f` then apply with its IDs. Otherwise the whole result is dropped. +4. `ps` changes: `n` takes the later value, `f` is replaced, `st` is replaced or cleared. Then `cluster_size`. Then truncate each `ps.iv` and `ps.ov` from the front to `MAX_INVALIDATED_UIDS`. Then prune the `ps` entries of providers the catalog does not hold: each keeps only `o`, `oc` and `ov`, so a retired provider's opt-out survives its removal from config and holds again if it comes back, and an entry with none of them is dropped. While `ps` is over `MAX_PROVIDER_STATES`, drop retained entries first, oldest `o.at` first. Then drop expired records, except refresh anchors: the source's owner is configured and holds a `st`, and the record's `ru` has not passed. +5. `validate()`. On failure, rebuild from the live entry through `apply_removals_only`, and if that also fails, give `None` and log. `validate()` checks the entry-set limits of spec 4.5, so a set that breaks one, such as a set over `MAX_ENTRY_SET_BYTES`, is dropped whole here and never trimmed. +6. `changed` is whether the result differs from `live`. A caller skips the write when it is `false`. - [ ] **Step 1: Write the failing tests** +Add the fixtures to `ec/identity/test_support.rs`: + +- `pub fn fixed_provider(id: &str, sources: &[&str]) -> Arc`: a descriptor-only resolver with a dummy `https://api.example.com` endpoint and the `eid_persistence_only` policy +- `pub fn fixed_capture_provider(id: &str, sources: &[&str]) -> Arc`: a descriptor-only provider with `Captures::new().with_path_prefix("/integrations//")` and no `resolves` + +Then the tests in `mutation.rs`: + ```rust #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use std::sync::Arc; - use serde_json::json; use super::*; use crate::ec::identity::catalog::ProviderCatalog; use crate::ec::identity::input::SeededRandom; - use crate::ec::identity::revision::IdentityKeys; - use crate::ec::identity::test_support::{fixed_provider, test_keys}; - use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId, KvProviderState}; + use crate::ec::identity::revision::{RecordRevision, StateRevision, set_hashes, uid_hash}; + use crate::ec::identity::test_support::{example_entry_set, fixed_capture_provider, fixed_provider, full_record}; + use crate::ec::kv_types::{ + KvEntry, KvOptOut, KvPartnerId, KvProviderState, KvUid, MAX_INVALIDATED_UIDS, MAX_PROVIDER_STATES, + }; const NOW: u64 = 1_000; fn src(name: &str) -> SourceDomain { - name.parse().expect("source") + name.parse().expect("should parse the source") } fn pid(name: &str) -> IdentityProviderId { - name.parse().expect("id") + name.parse().expect("should parse the provider ID") + } + + fn rev(value: &str) -> Revision { + Revision::Set(RecordRevision::from_stored(value)) } + /// `owner` owns a.example.com, `other` claims b.example.com unowned, and + /// the capture-only `cap` is narrowed to c.example.com. fn catalog() -> ProviderCatalog { ProviderCatalog::new( - vec![fixed_provider("owner", &["a.example.com"]), fixed_provider("other", &["b.example.com"])], + vec![ + fixed_provider("owner", &["a.example.com"]), + fixed_provider("other", &["b.example.com"]), + fixed_capture_provider("cap", &["c.example.com"]), + ], [(src("a.example.com"), pid("owner"))].into_iter().collect(), ) } fn run(mutation: &IdentityMutation, live: &KvEntry) -> Option { - let keys = test_keys(); let random = SeededRandom::new(1); let catalog = catalog(); - apply(mutation, live, &ApplyContext { now: NOW, keys: &keys, random: &random, catalog: &catalog }) + apply(mutation, live, &ApplyContext { now: NOW, random: &random, catalog: &catalog }) + } + + fn run_removals(mutation: &IdentityMutation, live: &KvEntry) -> Option { + let random = SeededRandom::new(1); + let catalog = catalog(); + apply_removals_only(mutation, live, &ApplyContext { now: NOW, random: &random, catalog: &catalog }) + } + + fn written(writer: &str, uid: &str, rv: &str) -> KvPartnerId { + KvPartnerId { w: Some(writer.to_owned()), rv: Some(rv.to_owned()), at: Some(900), ..KvPartnerId::new(uid) } } - fn owned(uid: &str, rv: &str) -> KvPartnerId { - KvPartnerId { w: Some("owner".to_owned()), rv: Some(rv.to_owned()), at: Some(900), ..KvPartnerId::new(uid) } + fn opt_out(at: u64) -> ProviderStateChange { + ProviderStateChange { opt_out: Some(KvOptOut { at, fp: "1".to_owned() }), ..ProviderStateChange::default() } + } + + fn fill(source: &str, record: KvPartnerId) -> PartnerIdChange { + PartnerIdChange::FillMissing { source: src(source), record } } #[test] fn tombstone_drops_everything() { let mut mutation = IdentityMutation::default(); - mutation.add_change(PartnerIdChange::FillMissing { - source: src("b.example.com"), - record: KvPartnerId::browser("u", Some(NOW)), - writer: None, - staged_at: None, - }); + mutation.add_change(fill("b.example.com", KvPartnerId::browser("u", Some(NOW)))); assert!(run(&mutation, &KvEntry::tombstone(1)).is_none(), "should drop on a tombstone"); + assert!(run_removals(&mutation, &KvEntry::tombstone(1)).is_none(), "should drop removals on a tombstone"); } #[test] - fn fill_missing_fills_absent_and_expired_only() { + fn fill_missing_fills_absent_expired_and_unconfigured_only() { let mut live = KvEntry::minimal("b.example.com", "existing", 1); - live.ids.insert("c.example.com".to_owned(), KvPartnerId { exp: Some(NOW), ..KvPartnerId::new("old") }); + live.ids.insert("d.example.com".to_owned(), KvPartnerId { exp: Some(NOW), ..KvPartnerId::new("old") }); + live.ids.insert("f.example.com".to_owned(), written("retired", "stale", "1111111111111111")); let mut mutation = IdentityMutation::default(); - for source in ["b.example.com", "c.example.com", "d.example.com"] { - mutation.add_change(PartnerIdChange::FillMissing { - source: src(source), - record: KvPartnerId::browser("new", Some(NOW)), - writer: None, - staged_at: None, - }); + for source in ["b.example.com", "d.example.com", "e.example.com", "f.example.com"] { + mutation.add_change(fill(source, KvPartnerId::browser("new", Some(NOW)))); } let applied = run(&mutation, &live).expect("should apply"); - assert!(applied.changed, "should change"); - assert_eq!(applied.entry.ids["b.example.com"].uid, "existing", "should not replace a live value"); - assert_eq!(applied.entry.ids["c.example.com"].uid, "new", "should replace an expired value"); - assert_eq!(applied.entry.ids["d.example.com"].uid, "new", "should fill an absent source"); + let uid = |source: &str| applied.entry.ids[source].first_uid().map(str::to_owned); + assert_eq!(uid("b.example.com").as_deref(), Some("existing"), "should not replace a live value"); + assert_eq!(uid("d.example.com").as_deref(), Some("new"), "should replace an expired value"); + assert_eq!(uid("e.example.com").as_deref(), Some("new"), "should fill an absent source"); + assert_eq!(uid("f.example.com").as_deref(), Some("new"), "should treat an unconfigured writer as absent"); + let record = &applied.entry.ids["e.example.com"]; + assert_eq!((record.at, record.w.as_deref()), (Some(NOW), None), "should be a browser record"); + assert!(record.rv.is_some(), "should give every record it writes an rv"); } #[test] fn owner_write_checks_revision_and_gets_a_fresh_rv() { let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); - - let mut stale = IdentityMutation::default(); - stale.add_change(PartnerIdChange::OwnerWrite { - source: src("a.example.com"), - record: KvPartnerId::new("v2"), - writer: pid("owner"), - expected: Some(Revision::from_stored("2222222222222222")), - staged_at: None, - }); - let applied = run(&stale, &live).expect("should apply"); - assert!(!applied.changed, "a stale revision loses to the newer write"); + live.ids.insert("a.example.com".to_owned(), written("owner", "v1", "1111111111111111")); + let write = |expected| { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { w: Some("owner".to_owned()), ..KvPartnerId::new("v2") }, + expected, + }); + run(&mutation, &live).expect("should apply") + }; - let mut fresh = IdentityMutation::default(); - fresh.add_change(PartnerIdChange::OwnerWrite { - source: src("a.example.com"), - record: KvPartnerId::new("v2"), - writer: pid("owner"), - expected: Some(Revision::from_stored("1111111111111111")), - staged_at: None, - }); - let applied = run(&fresh, &live).expect("should apply"); + assert!(!write(Some(rev("2222222222222222"))).changed, "should lose a stale revision to the newer write"); + let applied = write(Some(rev("1111111111111111"))); let record = &applied.entry.ids["a.example.com"]; - assert_eq!(record.uid, "v2", "should write"); - assert_eq!(record.w.as_deref(), Some("owner"), "should set the writer"); + assert_eq!(record.first_uid(), Some("v2"), "should write"); + assert_eq!(record.w.as_deref(), Some("owner"), "should keep the writer"); assert_ne!(record.rv.as_deref(), Some("1111111111111111"), "should draw a new rv"); assert_eq!(record.at, Some(NOW), "should stamp at"); } #[test] - fn browser_seeded_value_never_blocks_the_owner() { - let mut live = KvEntry::minimal("a.example.com", "seeded", 1); - live.ids.get_mut("a.example.com").expect("a").at = Some(5); + fn owner_write_replaces_the_whole_set_and_leaves_other_sources() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { rv: Some("1111111111111111".to_owned()), ..full_record(Some("owner"), example_entry_set()) }, + ); let mut mutation = IdentityMutation::default(); mutation.add_change(PartnerIdChange::OwnerWrite { source: src("a.example.com"), - record: KvPartnerId::new("owned"), - writer: pid("owner"), - expected: None, - staged_at: None, + record: KvPartnerId { w: Some("owner".to_owned()), ..KvPartnerId::new("only") }, + expected: Some(rev("1111111111111111")), }); + let applied = run(&mutation, &live).expect("should apply"); - assert_eq!(applied.entry.ids["a.example.com"].uid, "owned", "owner takes over a seeded value"); + + assert_eq!(applied.entry.ids["a.example.com"], KvPartnerId { + rv: applied.entry.ids["a.example.com"].rv.clone(), + at: Some(NOW), + w: Some("owner".to_owned()), + ..KvPartnerId::new("only") + }, "should keep no entry of the old set"); + assert_eq!(applied.entry.ids["b.example.com"].first_uid(), Some("x"), "should leave other sources alone"); + + let mut full = IdentityMutation::default(); + full.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: full_record(Some("owner"), example_entry_set()), + expected: None, + }); + let applied = run(&full, &KvEntry::minimal("b.example.com", "x", 1)).expect("should apply"); + assert_eq!(applied.entry.ids["a.example.com"].eids, example_entry_set(), "should write every entry, in order"); } #[test] - fn removals_win_ties_and_opt_out_removes_everything_the_writer_wrote() { + fn browser_seeded_and_version_1_values_never_block_the_owner() { + let live = KvEntry::minimal("a.example.com", "seeded", 1); + let mut write = IdentityMutation::default(); + write.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { w: Some("owner".to_owned()), ..KvPartnerId::new("owned") }, + expected: None, + }); + assert_eq!( + run(&write, &live).expect("should apply").entry.ids["a.example.com"].first_uid(), + Some("owned"), + "should let the owner take over a seeded value" + ); + + let mut remove = IdentityMutation::default(); + remove.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: Revision::Unset, + }); + let applied = run(&remove, &live).expect("should apply"); + assert!(!applied.entry.ids.contains_key("a.example.com"), "should match a record without rv as Unset"); + assert_eq!(applied.entry.ps["owner"].iv, vec![uid_hash("seeded")], "should hash the removed UID into iv"); + } + + #[test] + fn an_applied_invalidation_hashes_every_uid_of_the_live_set() { let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); live.ids.insert( - "c.example.com".to_owned(), - KvPartnerId { w: Some("owner".to_owned()), at: Some(950), ..KvPartnerId::new("unowned-by-owner") }, + "a.example.com".to_owned(), + KvPartnerId { rv: Some("1111111111111111".to_owned()), ..full_record(Some("owner"), example_entry_set()) }, ); + let invalidate = |expected: &str| { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev(expected), + }); + run(&mutation, &live).expect("should apply") + }; + + let applied = invalidate("1111111111111111"); + let stale = invalidate("2222222222222222"); + assert!(!applied.entry.ids.contains_key("a.example.com"), "should remove the set"); + assert_eq!(applied.entry.ps["owner"].iv, set_hashes(&example_entry_set()), "should hash every UID it removed"); + assert!(stale.entry.ids.contains_key("a.example.com"), "should keep a set at another revision"); + assert!(stale.entry.ps.get("owner").is_none_or(|state| state.iv.is_empty()), "should add no hash"); + } + + #[test] + fn iv_keeps_the_128_most_recent_hashes() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + let old: Vec = (0..MAX_INVALIDATED_UIDS).map(|index| format!("{index:016x}")).collect(); + live.ps.insert("owner".to_owned(), KvProviderState { iv: old.clone(), ..KvProviderState::default() }); + live.ids.insert("a.example.com".to_owned(), written("owner", "u", "1111111111111111")); let mut mutation = IdentityMutation::default(); - mutation.merge_state( - pid("owner"), - ProviderStateChange { opt_out: Some(KvOptOut { at: NOW, fp: "0".repeat(16) }), ..ProviderStateChange::default() }, - ); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("1111111111111111"), + }); + + let applied = run(&mutation, &live).expect("should apply"); + + let iv = &applied.entry.ps["owner"].iv; + assert_eq!(iv.len(), MAX_INVALIDATED_UIDS, "should cap at 128"); + assert_eq!(iv[..127], old[1..], "should drop the oldest hash"); + assert_eq!(iv[127], uid_hash("u"), "should append the new hash"); + } + + #[test] + fn opt_out_removes_what_the_writer_wrote_fills_ov_and_skips_its_changes() { + let mut live = KvEntry::minimal("e.example.com", "browser", 1); + live.ids.insert("a.example.com".to_owned(), written("owner", "v1", "1111111111111111")); + live.ids.insert("b.example.com".to_owned(), written("owner", "unowned-by-owner", "2222222222222222")); + let mut mutation = IdentityMutation::default(); + mutation.merge_state(pid("owner"), opt_out(NOW)); mutation.add_change(PartnerIdChange::OwnerWrite { source: src("a.example.com"), - record: KvPartnerId::new("v2"), - writer: pid("owner"), + record: KvPartnerId { w: Some("owner".to_owned()), ..KvPartnerId::new("v2") }, expected: None, - staged_at: Some(NOW), }); let applied = run(&mutation, &live).expect("should apply"); assert!(!applied.entry.ids.contains_key("a.example.com"), "should remove the owned record"); - assert!(!applied.entry.ids.contains_key("c.example.com"), "should remove the unowned record it wrote"); - assert_eq!(applied.entry.ids["b.example.com"].uid, "x", "control: other writers untouched"); - assert_eq!(applied.entry.ps["owner"].o.as_ref().map(|o| o.at), Some(NOW), "should store the opt-out"); + assert!(!applied.entry.ids.contains_key("b.example.com"), "should remove the unowned record it wrote"); + assert_eq!(applied.entry.ids["e.example.com"].first_uid(), Some("browser"), "should leave other records"); + let state = &applied.entry.ps["owner"]; + assert_eq!(state.o.as_ref().map(|o| o.at), Some(NOW), "should store the opt-out"); + assert_eq!(state.ov, vec![uid_hash("v1"), uid_hash("unowned-by-owner")], "should hash what it removed into ov"); + assert!(state.iv.is_empty(), "should leave iv for invalidations"); } #[test] - fn writer_remove_without_revision_respects_staged_at() { - let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); - let remove = |staged_at| { - let mut mutation = IdentityMutation::default(); - mutation.add_change(PartnerIdChange::WriterRemove { - source: src("a.example.com"), - writer: pid("owner"), - staged_at: Some(staged_at), - }); - run(&mutation, &live).expect("should apply") - }; - assert!(remove(900).entry.ids.get("a.example.com").is_none(), "same second: the removal wins"); - assert!(remove(899).entry.ids.get("a.example.com").is_some(), "re-acquired after sa: kept"); + fn a_capture_only_opt_out_removes_browser_records_of_its_sources_and_blocks_their_fill() { + let mut live = KvEntry::minimal("c.example.com", "browser-c", 1); + live.ids.insert("e.example.com".to_owned(), KvPartnerId::new("browser-e")); + let mut mutation = IdentityMutation::default(); + mutation.merge_state(pid("cap"), opt_out(NOW)); + + let applied = run(&mutation, &live).expect("should apply"); + + assert!(!applied.entry.ids.contains_key("c.example.com"), "should remove a browser record of its source"); + assert!(applied.entry.ids.contains_key("e.example.com"), "should leave a source it is not narrowed to"); + + let mut refill = IdentityMutation::default(); + refill.add_change(fill("c.example.com", KvPartnerId::browser("another", Some(NOW)))); + let applied = run(&refill, &applied.entry).expect("should apply"); + assert!(!applied.entry.ids.contains_key("c.example.com"), "should skip a browser fill while it is opted out"); } #[test] - fn opt_out_and_clear_in_the_same_second_leave_it_opted_out() { + fn opt_out_and_clear_ordering() { let live = KvEntry::minimal("b.example.com", "x", 1); - let mut mutation = IdentityMutation::default(); - mutation.merge_state( - pid("owner"), - ProviderStateChange { - opt_out: Some(KvOptOut { at: NOW, fp: "0".repeat(16) }), - opt_out_clear: Some(NOW), - ..ProviderStateChange::default() - }, - ); - let applied = run(&mutation, &live).expect("should apply"); + let mut same_second = IdentityMutation::default(); + same_second.merge_state(pid("owner"), ProviderStateChange { opt_out_clear: Some(NOW), ..opt_out(NOW) }); + let applied = run(&same_second, &live).expect("should apply"); assert!(applied.entry.ps["owner"].o.is_some(), "should stay opted out"); assert_eq!(applied.entry.ps["owner"].oc, None, "should drop a clear that is not later"); - } - #[test] - fn invalidated_uid_is_not_reseeded() { - let keys = test_keys(); - let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ps.insert( + let mut no_opt_out = IdentityMutation::default(); + no_opt_out.merge_state(pid("owner"), ProviderStateChange { opt_out_clear: Some(NOW), ..ProviderStateChange::default() }); + assert!(!run(&no_opt_out, &live).expect("should apply").changed, "should drop a clear with nothing to clear"); + + let mut opted = live.clone(); + opted.ps.insert( "owner".to_owned(), KvProviderState { - iv: vec![crate::ec::identity::revision::invalidated_uid_hash(&keys, &pid("owner"), "rejected")], + o: Some(KvOptOut { at: 900, fp: "1".to_owned() }), + iv: vec![uid_hash("invalidated")], + ov: vec![uid_hash("opted-out")], ..KvProviderState::default() }, ); + let mut clear = IdentityMutation::default(); + clear.merge_state(pid("owner"), ProviderStateChange { opt_out_clear: Some(NOW), ..ProviderStateChange::default() }); + let state = &run(&clear, &opted).expect("should apply").entry.ps["owner"]; + assert_eq!(state.oc, Some(NOW), "should store a later clear"); + assert!(state.ov.is_empty(), "should empty ov"); + assert_eq!(state.iv, vec![uid_hash("invalidated")], "should never empty iv"); + + let mut older = IdentityMutation::default(); + older.merge_state(pid("owner"), opt_out(899)); + assert!(!run(&older, &opted).expect("should apply").changed, "should drop an opt-out not after the live one"); + } + + #[test] + fn fill_missing_is_skipped_when_any_uid_is_in_a_claimants_iv_or_ov() { + let mut live = KvEntry::minimal("e.example.com", "x", 1); + live.ps.insert("owner".to_owned(), KvProviderState { iv: vec![uid_hash("example-b")], ..KvProviderState::default() }); + live.ps.insert("other".to_owned(), KvProviderState { ov: vec![uid_hash("removed")], ..KvProviderState::default() }); let mut mutation = IdentityMutation::default(); - mutation.add_change(PartnerIdChange::FillMissing { - source: src("a.example.com"), - record: KvPartnerId::browser("rejected", Some(NOW)), - writer: None, - staged_at: None, - }); + mutation.add_change(fill("a.example.com", full_record(None, example_entry_set()))); + mutation.add_change(fill("b.example.com", KvPartnerId::browser("removed", Some(NOW)))); + mutation.add_change(fill("d.example.com", KvPartnerId::browser("removed", Some(NOW)))); + let applied = run(&mutation, &live).expect("should apply"); - assert!(!applied.entry.ids.contains_key("a.example.com"), "should not re-seed the rejected UID"); + + assert!(!applied.entry.ids.contains_key("a.example.com"), "should block an owned set on its second UID"); + assert!(!applied.entry.ids.contains_key("b.example.com"), "should block an unowned source by its claimant's ov"); + assert!(applied.entry.ids.contains_key("d.example.com"), "should not block a source no denying provider claims"); } #[test] fn provider_result_applies_atomically() { let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ids.insert("a.example.com".to_owned(), owned("newer", "3333333333333333")); + live.ids.insert("a.example.com".to_owned(), written("owner", "newer", "3333333333333333")); live.ps.insert("owner".to_owned(), KvProviderState { st: Some(json!({ "token": "B" })), ..KvProviderState::default() }); - - let mut mutation = IdentityMutation::default(); - mutation.add_result(ProviderResult { + let result = |expected, expected_state| ProviderResult { provider: pid("owner"), ids: vec![PartnerIdChange::OwnerWrite { source: src("a.example.com"), - record: KvPartnerId::new("mine"), - writer: pid("owner"), - expected: Some(Revision::from_stored("1111111111111111")), - staged_at: None, + record: KvPartnerId { w: Some("owner".to_owned()), ..KvPartnerId::new("mine") }, + expected: Some(expected), }], state: ProviderStateChange { st: StateUpdate::Replace(json!({ "token": "A" })), f: Some(0), ..ProviderStateChange::default() }, - expected_state: ExpectedState::Json(r#"{"token":"A0"}"#.to_owned()), - }); + expected_state, + }; - let applied = run(&mutation, &live).expect("should apply"); + let mut lost_id = IdentityMutation::default(); + lost_id.add_result(result(rev("1111111111111111"), Some(StateRevision::of(Some(&json!({ "token": "B" })))))); + let mut lost_state = IdentityMutation::default(); + lost_state.add_result(result(rev("3333333333333333"), Some(StateRevision::of(Some(&json!({ "token": "A0" })))))); + let mut wins = IdentityMutation::default(); + wins.add_result(result(rev("3333333333333333"), Some(StateRevision::of(Some(&json!({ "token": "B" })))))); - assert_eq!(applied.entry.ids["a.example.com"].uid, "newer", "should keep the other writer's ID"); - assert_eq!(applied.entry.ps["owner"].st, Some(json!({ "token": "B" })), "with that writer's state"); - assert!(!applied.changed, "nothing else changed"); + for (name, mutation) in [("an ID", lost_id), ("the state", lost_state)] { + assert!(!run(&mutation, &live).expect("should apply").changed, "should drop the whole result when {name} lost"); + } + let applied = run(&wins, &live).expect("should apply"); + assert_eq!(applied.entry.ids["a.example.com"].first_uid(), Some("mine"), "should write the ID"); + assert_eq!(applied.entry.ps["owner"].st, Some(json!({ "token": "A" })), "should write the state with it"); } #[test] @@ -5186,57 +6968,117 @@ mod tests { let mut live = KvEntry::minimal("b.example.com", "x", 1); live.ids.insert( "a.example.com".to_owned(), - KvPartnerId { exp: Some(NOW - 1), ru: Some(NOW + 100), ..owned("expired", "1111111111111111") }, + KvPartnerId { exp: Some(NOW - 1), ru: Some(NOW + 100), ..written("owner", "expired", "1111111111111111") }, ); live.ps.insert("owner".to_owned(), KvProviderState { st: Some(json!({ "r": 1 })), ..KvProviderState::default() }); let mut unrelated = IdentityMutation::default(); unrelated.set_cluster_size(1); - let kept = run(&unrelated, &live).expect("should apply"); - assert!(kept.entry.ids.contains_key("a.example.com"), "anchor kept while state lives"); + assert!(run(&unrelated, &live).expect("should apply").entry.ids.contains_key("a.example.com"), "should keep the anchor"); + live.ps.get_mut("owner").expect("should hold the owner's state").st = None; + assert!(!run(&unrelated, &live).expect("should apply").entry.ids.contains_key("a.example.com"), "should drop it"); + } - live.ps.get_mut("owner").expect("owner").st = None; - let dropped = run(&unrelated, &live).expect("should apply"); - assert!(!dropped.entry.ids.contains_key("a.example.com"), "anchor dropped without state"); + #[test] + fn unconfigured_provider_state_keeps_o_oc_and_ov() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + let kept = KvProviderState { + o: Some(KvOptOut { at: 7, fp: "1".to_owned() }), + oc: Some(6), + ov: vec![uid_hash("removed")], + ..KvProviderState::default() + }; + live.ps.insert( + "retired".to_owned(), + KvProviderState { n: Some(5), iv: vec![uid_hash("invalidated")], st: Some(json!(1)), ..kept.clone() }, + ); + live.ps.insert("gone".to_owned(), KvProviderState { n: Some(5), ..KvProviderState::default() }); + live.ps.insert("other".to_owned(), KvProviderState { n: Some(5), ..KvProviderState::default() }); + let mut mutation = IdentityMutation::default(); + mutation.set_cluster_size(1); + + let applied = run(&mutation, &live).expect("should apply"); + + assert_eq!(applied.entry.ps["retired"], kept, "should keep only o, oc and ov"); + assert!(!applied.entry.ps.contains_key("gone"), "should drop an entry with none of them"); + assert_eq!(applied.entry.ps["other"].n, Some(5), "should leave a configured provider untouched"); } #[test] - fn unconfigured_provider_state_is_pruned() { + fn retained_opt_outs_are_pruned_first_when_over_the_state_cap() { let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ps.insert("retired".to_owned(), KvProviderState { n: Some(5), ..KvProviderState::default() }); live.ps.insert("other".to_owned(), KvProviderState { n: Some(5), ..KvProviderState::default() }); + for (at, index) in (100_u64..).zip(0..MAX_PROVIDER_STATES) { + live.ps.insert( + format!("retired-{index:02}"), + KvProviderState { o: Some(KvOptOut { at, fp: "1".to_owned() }), ..KvProviderState::default() }, + ); + } let mut mutation = IdentityMutation::default(); mutation.set_cluster_size(1); + let applied = run(&mutation, &live).expect("should apply"); - assert!(!applied.entry.ps.contains_key("retired"), "should prune"); - assert!(applied.entry.ps.contains_key("other"), "control: configured kept"); + + assert_eq!(applied.entry.ps.len(), MAX_PROVIDER_STATES, "should fit the cap"); + assert!(applied.entry.ps.contains_key("other"), "should keep the configured provider"); + assert!(!applied.entry.ps.contains_key("retired-00"), "should drop the oldest retained opt-out"); + assert!(applied.entry.ps.contains_key("retired-01"), "should keep the newer ones"); } #[test] - fn invalid_result_falls_back_to_removals_only() { + fn an_invalid_result_falls_back_to_removals_only() { let mut live = KvEntry::minimal("b.example.com", "x", 1); - live.ids.insert("a.example.com".to_owned(), owned("v1", "1111111111111111")); + live.ids.insert("a.example.com".to_owned(), written("owner", "v1", "1111111111111111")); + let mut oversized = example_entry_set(); + oversized[0].ext = Some(json!({ "segments": "x".repeat(2_000) })); + oversized[1].ext = Some(json!({ "segments": "y".repeat(2_000) })); let mut mutation = IdentityMutation::default(); - mutation.add_change(PartnerIdChange::FillMissing { - source: src("d.example.com"), - record: KvPartnerId::browser("x".repeat(600), Some(NOW)), - writer: None, - staged_at: None, - }); + mutation.add_change(fill("d.example.com", full_record(Some("other"), oversized))); mutation.add_change(PartnerIdChange::OwnerRemove { source: src("a.example.com"), - writer: pid("owner"), - expected: Revision::from_stored("1111111111111111"), + provider: pid("owner"), + expected: rev("1111111111111111"), }); + let applied = run(&mutation, &live).expect("should apply removals"); - assert!(applied.removals_only, "should fall back"); - assert!(!applied.entry.ids.contains_key("a.example.com"), "removal applied"); - assert!(!applied.entry.ids.contains_key("d.example.com"), "invalid fill dropped"); + + assert!(applied.removals_only, "should fail validation for a set over MAX_ENTRY_SET_BYTES"); + assert!(!applied.entry.ids.contains_key("d.example.com"), "should drop the set whole, never trim it"); + assert!(!applied.entry.ids.contains_key("a.example.com"), "should still apply the removal"); + } + + #[test] + fn apply_removals_only_ignores_everything_but_removals() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert("a.example.com".to_owned(), written("owner", "v1", "1111111111111111")); + live.ids.insert("d.example.com".to_owned(), KvPartnerId { exp: Some(NOW - 1), ..KvPartnerId::new("expired") }); + let mut mutation = IdentityMutation::default(); + mutation.add_change(fill("e.example.com", KvPartnerId::browser("new", Some(NOW)))); + mutation.merge_state(pid("other"), ProviderStateChange { n: Some(5), f: Some(2), ..opt_out(NOW) }); + mutation.set_cluster_size(3); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("1111111111111111"), + }); + + let applied = run_removals(&mutation, &live).expect("should apply"); + + assert!(!applied.entry.ids.contains_key("a.example.com"), "should apply the removal"); + assert!(!applied.entry.ids.contains_key("e.example.com"), "should ignore a positive change"); + assert!(applied.entry.ids.contains_key("d.example.com"), "should not prune expired records"); + let state = &applied.entry.ps["other"]; + assert_eq!((state.n, state.f), (None, 0), "should never touch n or f"); + assert!(state.o.is_some(), "should store the opt-out"); + assert_eq!(applied.entry.network, live.network, "should not touch cluster_size"); + + let nothing = IdentityMutation::default(); + assert!(!run_removals(¬hing, &live).expect("should apply").changed, "should report no change"); } } ``` -Add `fixed_provider(id: &str, sources: &[&str]) -> Arc` (a descriptor-only provider with a dummy `https://api.example.com` endpoint and the `eid_persistence_only` policy) and `test_keys() -> IdentityKeys` to `ec/identity/test_support.rs`. +The ordering tests build `ProviderStateChange` with `..opt_out(..)` and `..ProviderStateChange::default()` because they test one field at a time. They are not exhaustiveness tests. Check the field name of `KvEntry`'s network block (`network` above) against `kv_types.rs` and use the real one. - [ ] **Step 2: Run and see them fail** @@ -5245,7 +7087,7 @@ Expected: compile errors. - [ ] **Step 3: Implement** -Write `mutation.rs` with the types listed under Interfaces, then implement `apply` as five private steps over a cloned `KvEntry`. Use one function per step so each rule can be read on its own: +Write `mutation.rs` with the types listed under Interfaces, then implement `apply` and `apply_removals_only` from private steps over a cloned `KvEntry`. Use one function per step so each rule can be read on its own: ```rust pub(crate) fn apply(mutation: &IdentityMutation, live: &KvEntry, context: &ApplyContext<'_>) -> Option { @@ -5255,36 +7097,48 @@ pub(crate) fn apply(mutation: &IdentityMutation, live: &KvEntry, context: &Apply let mut entry = live.clone(); apply_removals(&mut entry, mutation, context); - let removals_only = entry.clone(); - apply_positive(&mut entry, mutation, context); - apply_state_and_cleanup(&mut entry, mutation, context); - - let (entry, removals_only_used) = match entry.validate() { - Ok(()) => (entry, false), - Err(message) => { - log::warn!("Identity mutation produced an invalid row, keeping removals only: {message}"); - let mut fallback = removals_only; - prune_and_drop_expired(&mut fallback, context); - if let Err(message) = fallback.validate() { - log::warn!("Identity mutation removals-only row is invalid, dropping mutation: {message}"); - return None; - } - (fallback, true) - } - }; + apply_state(&mut entry, mutation); + finish_removal_fields(&mut entry, context); + drop_expired_records(&mut entry, context); + if let Err(message) = entry.validate() { + log::warn!("Identity mutation produced an invalid row, keeping removals only: {message}"); + return apply_removals_only(mutation, live, context).map(|applied| Applied { removals_only: true, ..applied }); + } let changed = &entry != live; - Some(Applied { entry, changed, removals_only: removals_only_used }) + Some(Applied { entry, changed, removals_only: false }) +} + +pub(crate) fn apply_removals_only( + mutation: &IdentityMutation, + live: &KvEntry, + context: &ApplyContext<'_>, +) -> Option { + if !live.consent.ok { + return None; + } + + let mut entry = live.clone(); + apply_removals(&mut entry, mutation, context); + finish_removal_fields(&mut entry, context); + + if let Err(message) = entry.validate() { + log::warn!("Identity removals produced an invalid row, dropping them: {message}"); + return None; + } + let changed = &entry != live; + Some(Applied { entry, changed, removals_only: true }) } ``` -- `apply_removals`: iterate `OwnerRemove` and `WriterRemove` from `partner_ids` and from every `ProviderResult`'s state-free removals. When building a `ProviderResult`, the coordinator always puts removals in `partner_ids`, never in `ProviderResult.ids` (spec 5.4 comment on `ProviderResult`). Then opt-outs, clears and `iv` from `provider_state` and from each result's `state`. A revision is compared with `record_revision(context.keys, live_record)`. -- `apply_positive`: compute `opted_out(provider)` from the post-removal `ps`. For each plain change, call `fn change_applies(entry, change, context) -> bool` and, when true, `fn write_change(entry, change, context)`. For each `ProviderResult`, first check that every ID change applies and that `expected_state` matches `entry.ps[provider].st` serialized with `serde_json::to_string`. `ExpectedState::NotRead` always matches. Then apply all of them and the result's `state` together. -- `apply_state_and_cleanup`: merge `provider_state` (`n` = max, `f` replaced when `Some`, `st` per `StateUpdate`), set `entry.network`'s `cluster_size` when given (match the existing `KvNetwork` field), then call `prune_and_drop_expired`. -- `prune_and_drop_expired`: `entry.ps.retain(|id, _| parsed id is configured)`. Drop records with `exp <= now` unless they are anchors: the catalog's owner of that source has `entry.ps[owner].st.is_some()` and `record.ru.is_some_and(|ru| now < ru)`. +- `apply_removals`: collect every `OwnerRemove` and `WriterRemove` from `partner_ids`. The coordinator always puts removals there, never in `ProviderResult.ids` (spec 5.4's comment on `ProviderResult`). Then the opt-outs and clears from `provider_state` and from each result's `state`, in that order: opt-outs first, then clears, so an opt-out and a clear in the same second leave the provider opted out. Compare revisions with `record_revision(live_record)`. Hash a record with `set_hashes(&live_record.eids)` before removing it, and add the hashes with `push_hashes` to the provider's `iv` (an `OwnerRemove`) or `ov` (a `WriterRemove` or an applied opt-out). +- `apply_positive`: for each plain change, call `fn change_applies(entry, change, context) -> bool` and, when true, `fn write_change(entry, change, context)`. `change_applies` checks the opted-out rule, the claimant `iv`/`ov` rule and the capture-only rule of step 3 before the change's own rule. `write_change` inserts the change's record in place of the live one, whole, with a fresh `rv` from `new_revision(context.random)` and `at = record.at.or(Some(now))`, and never copies a field from the live record. For each `ProviderResult`, first check that every ID change applies and that `expected_state` is `None` or equals `StateRevision::of(entry.ps[provider].st.as_ref())`. Then apply all of them and the result's `state` together. +- `apply_state`: merge `provider_state` (`n` = max, `f` replaced when `Some`, `st` per `StateUpdate`) and set the network block's `cluster_size` when given (match the existing `KvNetwork` field). +- `finish_removal_fields`: truncate each `iv` and `ov` from the front to `MAX_INVALIDATED_UIDS`. For each `ps` entry whose provider the catalog does not hold, keep only `o`, `oc` and `ov`, and drop the entry when all three are empty. While `entry.ps.len() > MAX_PROVIDER_STATES`, remove the retained entry with the oldest `o.at` (an entry without `o` sorts first). Configured providers' entries are never pruned here; if they alone exceed the cap, `validate()` reports it. +- `drop_expired_records`: drop records with `exp <= now` unless they are anchors: the catalog's owner of that source is configured, `entry.ps[owner].st.is_some()`, and `record.ru.is_some_and(|ru| now < ru)`. -`IdentityMutation::merge_state(provider, change)` merges into any existing change for the same provider: later `n`, last `f`, union of `iv_add`, the later `opt_out`, the later `opt_out_clear`, and the last non-`Keep` `st`. +`IdentityMutation::merge_state(provider, change)` merges into any existing change for the same provider: later `n`, last `f`, the later `opt_out`, the later `opt_out_clear`, and the last non-`Keep` `st`. `IdentityMutation::removals()` copies the `OwnerRemove` and `WriterRemove` changes and, from every `provider_state` entry and every result's `state`, only `opt_out` and `opt_out_clear`. - [ ] **Step 4: Run and see them pass** @@ -5295,600 +7149,671 @@ Expected: PASS. ```bash git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs -git commit --signoff -S -m "Add IdentityMutation and its merge into a live identity row" -m "Removals apply first and win equal-second ties, a provider opt-out removes every record that provider wrote, stateful results apply atomically, refresh anchors survive while their owner holds state, and an invalid row falls back to removals only." +git commit --signoff -S -m "Add IdentityMutation and its merge into a live identity row" -m "Removals apply first and are never skipped. An opt-out removes every record its provider wrote, and for a capture-only provider the browser records of its sources, hashing each removed UID into ps.ov; a later clear empties ps.ov. An applied invalidation hashes every UID of the live set into ps.iv, and a fill is skipped when any of its UIDs is in the iv or ov of a provider that claims the source." -m "Stateful results apply atomically, refresh anchors survive while their owner holds state, an unconfigured provider keeps its opt-out and ov, and an invalid row falls back to removals only. apply_removals_only applies a removal-only mutation for the capture removal write and the removal-only retry." ``` -### Task 16: Sealing and opening the identity cookies +### Task 17: The `ts-eids-s` mirror payload and its size cap **Files:** -- Create: `crates/trusted-server-core/src/ec/identity/cookie.rs` (seal, open, read and format helpers) +- Create: `crates/trusted-server-core/src/ec/identity/mirror.rs` - Modify: `mod.rs`, `migration_guards.rs` -- Test: `cookie.rs` +- Test: `mirror.rs` **Interfaces:** -- Consumes: `IdentityKeys.cookie_s`, `.cookie_r` (Task 13), `RandomSource` (Task 9), `COOKIE_TS_EIDS_S`, `COOKIE_TS_EIDS_R_PREFIX` (Task 1) +- Consumes: `crate::openrtb::Eid` - Produces: - - `pub(crate) fn seal(key: &[u8; 32], ec_id: &str, payload: &[u8], random: &dyn RandomSource) -> String`, which returns `v1..` - - `pub(crate) fn open(key: &[u8; 32], ec_id: &str, value: &str) -> Option>` - - `pub(crate) enum CookieRead { Absent, Duplicate, Value(String) }` and `pub(crate) fn read_single_cookie(headers: &HeaderMap, name: &str) -> CookieRead`, which treats two cookies of the same name as none - - `pub(crate) fn read_prefixed_cookies(headers: &HeaderMap, prefix: &str) -> Vec<(String, String)>` - - `pub(crate) fn set_cookie_header(name: &str, value: &str) -> String` and `pub(crate) fn delete_cookie_header(name: &str) -> String` - - `pub(crate) const COOKIE_MAX_AGE_SECS: u64 = 31_536_000;` + - Constants `MIRROR_MAX_VALUE_BYTES = 2048` (the cookie value, after encoding) and `MIRROR_MAX_AGE_SEC = 86_400` (the lifetime of `ts-eids`) + - `pub(crate) struct MirrorSource { source: String, provider: Option, priority: i32, expires_at: Option, eids: Vec }`, with `pub(crate)` fields: one source's servable server entries exactly as bidders receive them (spec 6.2), the provider whose `cookie_priority` ranks it, and the source's earliest `exp` + - `pub(crate) enum MirrorRead<'a> { NotRead, Cleared, Servable(&'a [MirrorSource]) }`: what the request's KV read gave. `Cleared` covers consent that denies EID persistence, a tombstone, `consent.ok = false` and a missing row. + - `pub(crate) enum MirrorPayload { Value { value: String, max_age: u64 }, Clear }` with `pub(crate) fn to_json(&self) -> serde_json::Value`: `{"value": ..., "max_age": ...}` or `{"clear": true}` + - `pub(crate) fn mirror_payload(read: MirrorRead<'_>, now: u64) -> Option`: `None` for `NotRead`, so the response carries no member and the shim leaves the cookie as it is + +The rules (spec 6.2): + +1. `NotRead` gives `None`. `Cleared` gives `Clear`. +2. `Servable` keeps every source, in the order given. While the encoded value is over `MIRROR_MAX_VALUE_BYTES`, it leaves out one whole source: the lowest `priority` first, then the largest (the most bytes of compact JSON), then the later in input order. It logs each omission at info with the provider ID and source, never a value. A source is mirrored whole or not at all. +3. When no source is left, the payload is `Clear`, so the cookie never holds an empty or stale set. +4. `value` is the unpadded base64url of the compact JSON `{"v":1,"eids":[...]}`, the kept sources' entries in order. +5. `max_age` is the smaller of `MIRROR_MAX_AGE_SEC` and the time from `now` to the earliest `expires_at` among the kept sources. The servability check has already left out every record with `exp <= now`, so it is at least 1. + +The mirror never holds `ps`, a revision, a writer or a timing field. `MirrorSource.eids` is built from the merge's server set after `gate_eids_by_consent` (Task 26), so this module does no consent or servability check of its own. - [ ] **Step 1: Write the failing tests** ```rust #[cfg(test)] mod tests { - use http::{HeaderMap, HeaderValue, header}; + use base64::Engine as _; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use serde_json::json; use super::*; - use crate::ec::identity::input::SeededRandom; + use crate::openrtb::{Eid, Uid}; + + const NOW: u64 = 1_760_000_000; + + fn source(name: &str, priority: i32, expires_at: Option, ids: &[&str]) -> MirrorSource { + MirrorSource { + source: name.to_owned(), + provider: Some("p".parse().expect("should parse the provider ID")), + priority, + expires_at, + eids: vec![Eid::new( + name.to_owned(), + ids.iter().map(|id| Uid { id: (*id).to_owned(), atype: Some(1), ext: None }).collect(), + )], + } + } - const EC: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; - const OTHER_EC: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ABC123"; + fn decode(payload: &MirrorPayload) -> (serde_json::Value, u64) { + let MirrorPayload::Value { value, max_age } = payload else { + panic!("should hold a value"); + }; + let bytes = URL_SAFE_NO_PAD.decode(value).expect("should be unpadded base64url"); + (serde_json::from_slice(&bytes).expect("should be JSON"), *max_age) + } + + #[test] + fn no_read_is_absent_and_a_denial_or_empty_set_is_clear() { + assert_eq!(mirror_payload(MirrorRead::NotRead, NOW), None, "should leave the cookie as it is"); + assert_eq!(mirror_payload(MirrorRead::Cleared, NOW), Some(MirrorPayload::Clear), "should clear on a denial"); + assert_eq!(mirror_payload(MirrorRead::Servable(&[]), NOW), Some(MirrorPayload::Clear), "should clear an empty set"); + assert_eq!(MirrorPayload::Clear.to_json(), json!({ "clear": true }), "should serialize the clear shape"); + } #[test] - fn seal_and_open_round_trip_bound_to_the_ec_id() { - let key = [7u8; 32]; - let random = SeededRandom::new(3); - let sealed = seal(&key, EC, br#"{"v":1}"#, &random); + fn value_is_the_v1_entries_in_unpadded_base64url() { + let sources = [source("example.com", 0, None, &["example-a"])]; + + let payload = mirror_payload(MirrorRead::Servable(&sources), NOW).expect("should return a payload"); - assert!(sealed.starts_with("v1."), "should carry the version prefix"); - assert_eq!(sealed.split('.').count(), 3, "should have three parts"); - assert!(!sealed.contains('='), "should be unpadded base64url"); - assert_eq!(open(&key, EC, &sealed).as_deref(), Some(&br#"{"v":1}"#[..]), "should open for its EC"); - assert!(open(&key, OTHER_EC, &sealed).is_none(), "should not open for another EC"); - assert!(open(&[8u8; 32], EC, &sealed).is_none(), "should not open with another key"); - let mut tampered = sealed.clone(); - tampered.pop(); - tampered.push(if sealed.ends_with('A') { 'B' } else { 'A' }); - assert!(open(&key, EC, &tampered).is_none(), "should reject tampering"); - assert!(open(&key, EC, "v2.x.y").is_none(), "should reject another version"); + let (decoded, max_age) = decode(&payload); + assert_eq!( + decoded, + json!({ "v": 1, "eids": [{ "source": "example.com", "uids": [{ "id": "example-a", "atype": 1 }] }] }), + "should hold the entries exactly as bidders receive them" + ); + assert_eq!(max_age, MIRROR_MAX_AGE_SEC, "should live a day without an exp"); + let MirrorPayload::Value { value, .. } = &payload else { unreachable!() }; + assert_eq!(payload.to_json(), json!({ "value": value, "max_age": MIRROR_MAX_AGE_SEC }), "should serialize"); } #[test] - fn duplicate_cookie_names_count_as_none() { - let mut headers = HeaderMap::new(); - headers.append(header::COOKIE, HeaderValue::from_static("__Host-ts-eids-s=one")); - headers.append(header::COOKIE, HeaderValue::from_static("other=1; __Host-ts-eids-s=two")); - assert_eq!(read_single_cookie(&headers, "__Host-ts-eids-s"), CookieRead::Duplicate, "duplicate"); + fn max_age_never_outlives_the_earliest_exp() { + let sources = [ + source("a.example.com", 0, Some(NOW + 3_600), &["a"]), + source("b.example.com", 0, Some(NOW + 600), &["b"]), + source("c.example.com", 0, Some(NOW + 999_999), &["c"]), + ]; - let mut single = HeaderMap::new(); - single.insert(header::COOKIE, HeaderValue::from_static("__Host-ts-eids-s=one; x=2")); - assert_eq!(read_single_cookie(&single, "__Host-ts-eids-s"), CookieRead::Value("one".to_owned()), "single"); + let (_, max_age) = decode(&mirror_payload(MirrorRead::Servable(&sources), NOW).expect("should return a payload")); + + assert_eq!(max_age, 600, "should take the earliest exp"); } #[test] - fn prefixed_cookies_and_attributes() { - let mut headers = HeaderMap::new(); - headers.insert( - header::COOKIE, - HeaderValue::from_static("__Host-ts-eids-r-0a1b2c3d=a; __Host-ts-eids-r-11111111=b; x=1"), - ); - let found = read_prefixed_cookies(&headers, "__Host-ts-eids-r-"); - assert_eq!(found.len(), 2, "should find both removal cookies"); - assert_eq!( - set_cookie_header("__Host-ts-eids-s", "v1.a.b"), - "__Host-ts-eids-s=v1.a.b; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=31536000", - "should use the spec attributes and no Domain" - ); + fn an_oversized_set_drops_whole_sources_lowest_priority_then_largest() { + let big = |name: &str, priority, count| { + let ids: Vec = (0..count).map(|index| format!("{name}-{index:040}")).collect(); + source(name, priority, None, &ids.iter().map(String::as_str).collect::>()) + }; + let sources = [ + big("keep-high", 10, 6), + big("drop-low-large", 0, 12), + big("keep-low-small", 0, 6), + big("keep-mid", 5, 6), + ]; + + let payload = mirror_payload(MirrorRead::Servable(&sources), NOW).expect("should return a payload"); + + let MirrorPayload::Value { value, .. } = &payload else { panic!("should hold a value") }; + assert!(value.len() <= MIRROR_MAX_VALUE_BYTES, "should fit the cap"); + let (decoded, _) = decode(&payload); + let kept: Vec<&str> = + decoded["eids"].as_array().expect("should be an array").iter().map(|eid| eid["source"].as_str().expect("should name the source")).collect(); + assert_eq!(kept, vec!["keep-high", "keep-low-small", "keep-mid"], "should drop the largest lowest-priority source whole"); + } + + #[test] + fn a_set_where_nothing_fits_is_clear() { + let ids: Vec = (0..80).map(|index| format!("only-{index:040}")).collect(); + let sources = [source("only.example.com", 0, None, &ids.iter().map(String::as_str).collect::>())]; + assert_eq!( - delete_cookie_header("__Host-ts-eids-s"), - "__Host-ts-eids-s=; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=0", - "should expire with matching attributes" + mirror_payload(MirrorRead::Servable(&sources), NOW), + Some(MirrorPayload::Clear), + "should clear rather than keep an empty or partial set" ); } } ``` +Check the sizes once before Step 3: in `an_oversized_set_drops_whole_sources_lowest_priority_then_largest`, the four sources together must encode to more than 2,048 bytes and the three kept ones to at most 2,048. Adjust the UID counts if the encoding differs, keeping the order of the rule the test pins. + - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- ec::identity::cookie` +Run: `cargo test-fastly -- ec::identity::mirror` Expected: compile errors. - [ ] **Step 3: Implement** ```rust -//! The server-owned identity cookies: sealing, reading and formatting. +//! The `ts-eids-s` mirror payload: the servable server set for page scripts, +//! capped to one cookie (spec 6.2). The server never reads the cookie back. -use aes_gcm::aead::{Aead as _, KeyInit as _, Payload}; -use aes_gcm::{Aes256Gcm, Nonce}; use base64::Engine as _; use base64::engine::general_purpose::URL_SAFE_NO_PAD; -use http::{HeaderMap, header}; +use serde_json::json; -use super::input::RandomSource; - -/// Lifetime of both identity cookie kinds. -pub(crate) const COOKIE_MAX_AGE_SECS: u64 = 31_536_000; - -const SEAL_VERSION: &str = "v1"; - -/// Seals `payload` with AES-256-GCM, the EC ID as associated data. -#[must_use] -pub(crate) fn seal(key: &[u8; 32], ec_id: &str, payload: &[u8], random: &dyn RandomSource) -> String { - let cipher = Aes256Gcm::new(key.into()); - let mut nonce = [0u8; 12]; - random.fill(&mut nonce); - let ciphertext = cipher - .encrypt(Nonce::from_slice(&nonce), Payload { msg: payload, aad: ec_id.as_bytes() }) - .expect("should encrypt a bounded in-memory payload"); - format!( - "{SEAL_VERSION}.{}.{}", - URL_SAFE_NO_PAD.encode(nonce), - URL_SAFE_NO_PAD.encode(ciphertext) - ) -} - -/// Opens a sealed value. `None` for any failure; never log the value. -#[must_use] -pub(crate) fn open(key: &[u8; 32], ec_id: &str, value: &str) -> Option> { - let mut parts = value.split('.'); - if parts.next()? != SEAL_VERSION { - return None; - } - let nonce = URL_SAFE_NO_PAD.decode(parts.next()?).ok()?; - let ciphertext = URL_SAFE_NO_PAD.decode(parts.next()?).ok()?; - if parts.next().is_some() || nonce.len() != 12 { - return None; - } - Aes256Gcm::new(key.into()) - .decrypt(Nonce::from_slice(&nonce), Payload { msg: &ciphertext, aad: ec_id.as_bytes() }) - .ok() +use super::ids::IdentityProviderId; +use crate::openrtb::Eid; + +/// Maximum bytes of the encoded cookie value. +pub(crate) const MIRROR_MAX_VALUE_BYTES: usize = 2048; +/// Maximum cookie lifetime: that of `ts-eids`. +pub(crate) const MIRROR_MAX_AGE_SEC: u64 = 86_400; + +/// One source's servable server entries, as bidders receive them. +pub(crate) struct MirrorSource { + pub(crate) source: String, + /// The provider whose `cookie_priority` ranks the source, for the log. + pub(crate) provider: Option, + pub(crate) priority: i32, + /// The source's earliest `exp`, in Unix seconds. + pub(crate) expires_at: Option, + pub(crate) eids: Vec, } -/// One cookie read by exact name. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum CookieRead { - Absent, - /// Two or more cookies share the name; treated as none (spec 6.2). - Duplicate, - Value(String), +/// What the request's KV read gave the mirror. +pub(crate) enum MirrorRead<'a> { + /// No KV read: the response carries no mirror member. + NotRead, + /// Consent denies EID persistence, or the row is a tombstone, withdrawn + /// or missing. + Cleared, + /// The servable server set, in output order. + Servable(&'a [MirrorSource]), } -fn cookie_pairs(headers: &HeaderMap) -> impl Iterator { - headers - .get_all(header::COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(';')) - .filter_map(|pair| pair.trim().split_once('=')) - .map(|(name, value)| (name.trim(), value.trim())) +/// The mirror member of an `/auction` or `/_ts/page-bids` response. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum MirrorPayload { + Value { value: String, max_age: u64 }, + Clear, } -/// Reads a cookie that must occur at most once. -#[must_use] -pub(crate) fn read_single_cookie(headers: &HeaderMap, name: &str) -> CookieRead { - let mut found = cookie_pairs(headers).filter(|(cookie, _)| *cookie == name).map(|(_, value)| value); - match (found.next(), found.next()) { - (None, _) => CookieRead::Absent, - (Some(_), Some(_)) => CookieRead::Duplicate, - (Some(value), None) => CookieRead::Value(value.to_owned()), +impl MirrorPayload { + /// The JSON the shim reads. + #[must_use] + pub(crate) fn to_json(&self) -> serde_json::Value { + match self { + Self::Value { value, max_age } => json!({ "value": value, "max_age": max_age }), + Self::Clear => json!({ "clear": true }), + } } } -/// Every cookie whose name starts with `prefix`, as `(name, value)`. -#[must_use] -pub(crate) fn read_prefixed_cookies(headers: &HeaderMap, prefix: &str) -> Vec<(String, String)> { - cookie_pairs(headers) - .filter(|(name, _)| name.starts_with(prefix)) - .map(|(name, value)| (name.to_owned(), value.to_owned())) - .collect() +fn encode(sources: &[&MirrorSource]) -> String { + let eids: Vec<&Eid> = sources.iter().flat_map(|source| &source.eids).collect(); + let bytes = serde_json::to_vec(&json!({ "v": 1, "eids": eids })).expect("should serialize EIDs"); + URL_SAFE_NO_PAD.encode(bytes) } -/// A `Set-Cookie` value with the identity cookie attributes. -#[must_use] -pub(crate) fn set_cookie_header(name: &str, value: &str) -> String { - format!("{name}={value}; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age={COOKIE_MAX_AGE_SECS}") +fn source_bytes(source: &MirrorSource) -> usize { + serde_json::to_vec(&source.eids).map_or(usize::MAX, |bytes| bytes.len()) } -/// A `Set-Cookie` value that deletes an identity cookie. +/// The mirror payload for a request (spec 6.2). +/// +/// # Performance +/// +/// Encodes the set once per omission. A set holds at most one entry set per +/// configured source, so this stays small. #[must_use] -pub(crate) fn delete_cookie_header(name: &str) -> String { - format!("{name}=; Path=/; Secure; SameSite=Lax; HttpOnly; Max-Age=0") +pub(crate) fn mirror_payload(read: MirrorRead<'_>, now: u64) -> Option { + let sources = match read { + MirrorRead::NotRead => return None, + MirrorRead::Cleared => return Some(MirrorPayload::Clear), + MirrorRead::Servable(sources) => sources, + }; + let mut kept: Vec<&MirrorSource> = sources.iter().collect(); + let mut value = encode(&kept); + while value.len() > MIRROR_MAX_VALUE_BYTES && !kept.is_empty() { + let index = (0..kept.len()) + .min_by_key(|&index| (kept[index].priority, core::cmp::Reverse(source_bytes(kept[index])), core::cmp::Reverse(index))) + .expect("should hold a source"); + let left_out = kept.remove(index); + log::info!( + "Leaving source '{}' of provider '{}' out of the ts-eids-s mirror, which exceeds {MIRROR_MAX_VALUE_BYTES} bytes", + left_out.source, + left_out.provider.as_ref().map_or("none", IdentityProviderId::as_str), + ); + value = encode(&kept); + } + if kept.is_empty() { + return Some(MirrorPayload::Clear); + } + let max_age = kept + .iter() + .filter_map(|source| source.expires_at) + .map(|exp| exp.saturating_sub(now).max(1)) + .fold(MIRROR_MAX_AGE_SEC, u64::min); + Some(MirrorPayload::Value { value, max_age }) } ``` -If the `aes-gcm` 0.10 key conversion `key.into()` does not infer, use `Aes256Gcm::new_from_slice(key).expect("should accept a 32-byte key")`. +The `Reverse(index)` term makes "the later in input order" the tie-break, so the result never depends on sort stability. Register the module in `mod.rs` (`pub(crate) mod mirror;`) and in `migration_guards.rs`. `base64` is already a dependency of core: Task 9's `HashedEmail::to_base64` uses it. - [ ] **Step 4: Run and see them pass** -Run: `cargo test-fastly -- ec::identity::cookie` and `cargo check-cloudflare` -Expected: PASS and a clean build. The Cloudflare check proves that `aes-gcm` compiles for `wasm32-unknown-unknown`, because core is shared. +Run: `cargo test-fastly -- ec::identity::mirror` +Expected: PASS. - [ ] **Step 5: Commit** ```bash git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs -git commit --signoff -S -m "Seal and read the server-owned identity cookies" +git commit --signoff -S -m "Add the ts-eids-s mirror payload and its size cap" -m "The payload holds the servable server entries as bidders receive them, base64url-encoded, or tells the shim to clear the cookie. A set over 2,048 bytes leaves out whole sources, lowest cookie_priority first, then largest, and max_age never outlives the earliest exp." ``` -### Task 17: The `ts-eids-s` payload, its size cap and its merge with KV +--- + +### Task 18: The shared removal write **Files:** -- Modify: `crates/trusted-server-core/src/ec/identity/cookie.rs` -- Test: `cookie.rs` +- Create: `crates/trusted-server-core/src/ec/identity/removal.rs` +- Modify: `crates/trusted-server-core/src/ec/kv.rs` (a crate-visible generation-checked write) +- Modify: `mod.rs`, `migration_guards.rs` +- Test: `removal.rs` **Interfaces:** -- Consumes: Task 16, `ProviderCatalog` (Task 14), `KvEntry` (Task 6), `record_revision` (Task 13), `EcKvSnapshot` (`ec/mod.rs:118`) +- Consumes: `IdentityMutation`, `removals()`, `removal_labels()`, `apply_removals_only`, `ApplyContext` (Task 15), `KvIdentityGraph::load_snapshot`, `EcKvSnapshot`, `MAX_CAS_RETRIES` (`ec/kv.rs:28`) - Produces: - - Payload types with the spec 6.2 short field names, and serde `skip_serializing_if` on every optional field: - - `SCookie { v: u8, kc: u64, cf: String, m: bool, ids: Vec, ps: BTreeMap }` - - `SEntry { s: usize, u, w, t, e, r, a, x, rv, pv, sa, b, k, d }`. `b`, `k` and `d` serialize as `1`/`0` through small `bool_as_int` helpers. - - `SProviderState { n, f, o: Option, iv: Vec, oc, p, sg, kn }`. `kn` is the KV `n` that a cookie-only `n` was derived from (spec 6.3). - - `pub(crate) const MAX_S_COOKIE_BYTES: usize = 2048;` - - `pub(crate) fn open_s_cookie(keys, ec_id, catalog, value) -> SOpen`, where `SOpen` is `Missing | Failed | ConfigChanged(SCookie) | Valid(SCookie)`. `ConfigChanged` keeps only markers. - - `pub(crate) fn encode_s_cookie(keys, ec_id, cookie: &mut SCookie, random) -> String`, which applies the drop order until the sealed value fits in 2,048 bytes, else falls back to a minimal cookie with `kc = 0` and the markers only (spec 5.6) - - `pub(crate) fn merge_with_kv(cookie: &SCookie, snapshot: &EcKvSnapshot, context: &CookieMergeContext<'_>) -> MergeOutcome`, where `MergeOutcome { cookie: SCookie, changed: bool, expire: bool }` - - `pub(crate) fn cookie_view(cookie: &SCookie, catalog) -> BTreeMap`: the cookie's entries resolved to sources. `Staging` is `Confirmed | Staged { pv, sa, browser }`. - - `pub(crate) fn minimal_missing_cookie(now) -> SCookie` (`m: true`, no entries) - -Merge rules (spec 6.3, step 3). Each is one test: - -- A tombstone or `consent.ok = false` gives `expire: true`. -- `Missing` gives `minimal_missing_cookie`. Markers are dropped too. -- For an owned source, KV wins, except that a staged provider entry `OwnerWrite` would still apply (its `pv` matches the live revision, or it has no `pv` and KV holds only a value the provider did not write) stays `k: 0`. -- A `k: 1` entry whose source is gone from KV, or present at another revision, is replaced by the KV record (when owned) or dropped. -- A staged entry KV already holds (same UID, same writer) becomes `k: 1` with KV's `rv`. -- A `b: 1` entry is dropped once KV holds any value for its source. -- A stub becomes the KV record. -- Markers (`p`) are cleared when answered: KV `n >= p`, KV `o` set, a claimed source with `at >= p` or a source that lost the cookie's value, or the provider ineligible on the row. The last one takes a callback, `answered_by_eligibility`, which Task 20 supplies. -- `m` is cleared when the row is present. -- `kc = now`. `changed` is true when anything but `kc` changed, or the inbound `kc` is older than `kv_revalidate_sec / 2`. - -The cookie caches values only for owned sources once they are in KV (spec 5.3). Unowned confirmed values are not copied into the cookie by the merge. In Part 1 no source is owned, so the cookie holds only staged entries, markers and mirrored `ps`. - -Size-cap drop order (spec 6.2): - -1. `k: 1` entries become stubs `{s, d: 1}`. -2. `ps` entries with no `p` and no `sg` are dropped. -3. Provider `k: 0` entries, lowest `cookie_priority` first. Task 19 adds the priority. Pass it in as `&dyn Fn(&str) -> i32`. -4. `b: 1` entries, last. - -Markers are never dropped. Log each drop with the provider and source only. + - `KvIdentityGraph::write_if_generation(&self, ec_id: &str, entry: &KvEntry, generation: u64) -> Result>`, `pub(crate)`: serializes with `serialize_entry` (so `validate` runs) and writes with `ENTRY_TTL` and `EcKvWriteMode::IfGenerationMatch(generation)`. Phase 4's flush (Task 29) uses it too, so every identity write goes through one path. + - `pub(crate) const REMOVAL_WRITE_ATTEMPTS: u32 = MAX_CAS_RETRIES;` (5) + - `pub(crate) enum RemovalWriteOutcome { Written, NothingToWrite, Failed }`. `NothingToWrite` covers a tombstone, `consent.ok = false`, a missing row and removals that are already applied. `Failed` covers a failed read, a non-CAS write error such as Fastly's per-key limit, and five generation mismatches in a row. + - `pub(crate) fn write_removals(graph: &KvIdentityGraph, ec_id: &str, removals: &IdentityMutation, context: &ApplyContext<'_>) -> RemovalWriteOutcome` + - `pub(crate) fn log_removal_lost(ec_id: &str, removals: &IdentityMutation)`: logs at error with `log_id(ec_id)` and the provider IDs and sources from `removal_labels()`, never a UID. The caller increments `removal_lost` (Task 23 adds the counter to `RequestIdentity`). -- [ ] **Step 1: Write the failing tests** - -```rust - mod s_cookie { - use super::super::*; - use crate::ec::identity::test_support::{catalog_with_owner, test_keys}; - use crate::ec::identity::input::SeededRandom; - use crate::ec::kv_types::{KvEntry, KvPartnerId, KvProviderState}; - use crate::ec::EcKvSnapshot; +This is the CAS loop the header's G4 note names: the capture removal write (spec 5.3, Task 38) and the removal-only retry of a failed flush (spec 5.2, phase 4, step 5; Task 29) both call `write_removals`. The withdrawal tombstone keeps its own loop in `ec/kv.rs` (`tombstone_existing_from_snapshot`, `kv.rs:1349`). Task 27 gives it its after-send attempt. - const EC: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; +The loop (spec 5.3): - fn present(entry: KvEntry) -> EcKvSnapshot { - EcKvSnapshot::Present { ec_id: EC.to_owned(), entry: Box::new(entry), generation: Some(1) } - } - - #[test] - fn round_trips_and_detects_config_change() { - let keys = test_keys(); - let catalog = catalog_with_owner("owner", "a.example.com"); - let random = SeededRandom::new(1); - let mut cookie = SCookie::new(&catalog, 100); - cookie.ps.insert("owner".to_owned(), SProviderState { p: Some(90), ..SProviderState::default() }); - let sealed = encode_s_cookie(&keys, EC, &mut cookie, &random, &|_| 0); - - match open_s_cookie(&keys, EC, &catalog, &sealed) { - SOpen::Valid(opened) => assert_eq!(opened, cookie, "should round-trip"), - other => panic!("should open, got {other:?}"), - } - let other_catalog = catalog_with_owner("owner", "b.example.com"); - match open_s_cookie(&keys, EC, &other_catalog, &sealed) { - SOpen::ConfigChanged(kept) => { - assert!(kept.ids.is_empty(), "should drop positive entries"); - assert_eq!(kept.ps["owner"].p, Some(90), "should keep markers"); - } - other => panic!("should flag the config change, got {other:?}"), - } - } +1. `graph.load_snapshot(ec_id)`. `Missing` gives `NothingToWrite`. `Failed` gives `Failed`. `Present` without a generation is treated as `Failed`, as `upsert_partner_ids_from_snapshot` does (`kv.rs:775-781`). +2. `apply_removals_only(removals, &entry, context)`. `None` (a tombstone, `consent.ok = false`, or a removals-only row that fails validation) gives `NothingToWrite`. `changed == false` gives `NothingToWrite`: the removals already applied. +3. `graph.write_if_generation(ec_id, &applied.entry, generation)`. `Written` gives `Written`. `PreconditionFailed` re-reads and goes back to step 1, up to `REMOVAL_WRITE_ATTEMPTS` writes in total. Any `Err` gives `Failed` at once: an immediate retry after a per-key limit would fail too. +4. Five `PreconditionFailed` in a row give `Failed`. - #[test] - fn tombstone_expires_and_missing_row_minimizes() { - let catalog = catalog_with_owner("owner", "a.example.com"); - let cookie = SCookie::new(&catalog, 100); - let context = CookieMergeContext::for_test(&catalog, 200); - assert!(merge_with_kv(&cookie, &present(KvEntry::tombstone(1)), &context).expire, "tombstone"); - let missing = merge_with_kv(&cookie, &EcKvSnapshot::Missing { ec_id: EC.to_owned() }, &context); - assert!(missing.cookie.m, "should set m"); - assert!(missing.cookie.ids.is_empty() && missing.cookie.ps.is_empty(), "no entries, no markers"); - } +`write_removals` runs whatever the request's consent: a revocation can arrive when consent is unusable. It does not log a lost removal itself, because a capture tries once more after send (Task 38) before the removal counts as lost. `removals` must hold only removals, opt-outs and clears: callers pass `mutation.removals()`. `apply_removals_only` ignores anything else anyway. - #[test] - fn staged_entry_flips_to_confirmed_once_in_kv() { - let catalog = catalog_with_owner("owner", "a.example.com"); - let mut cookie = SCookie::new(&catalog, 100); - cookie.ids.push(SEntry::staged_browser(0, "u", 100)); - let mut entry = KvEntry::minimal("a.example.com", "u", 1); - entry.ids.get_mut("a.example.com").expect("a").rv = Some("1111111111111111".to_owned()); +First add to Task 15's `IdentityMutation`: `pub(crate) fn removal_labels(&self) -> Vec<(IdentityProviderId, Option)>`, one pair per `OwnerRemove` (its provider and source), `WriterRemove` (its writer and source), opt-out and clear (the provider, no source), sorted and de-duplicated. Add a unit test for it to `mutation.rs`. - let merged = merge_with_kv(&cookie, &present(entry), &CookieMergeContext::for_test(&catalog, 200)); +- [ ] **Step 1: Write the failing tests** - assert!(merged.changed, "should change"); - assert!(merged.cookie.ids.is_empty(), "a b:1 entry is dropped once KV holds the source"); - assert_eq!(merged.cookie.kc, 200, "should stamp kc"); - } +The tests drive `write_removals` against the in-memory store with generation tracking, `InMemoryEcKv` (`ec/kv_backend.rs:316`), and a local wrapper modelled on `ConflictInjectingEcKv` (`ec/kv.rs:1877`) that fails the first `n` generation-checked writes with `PreconditionFailed`, or every write with an error. Count writes the way `RecordingEcKv` (`ec/kv.rs:3449`) does. - #[test] - fn marker_is_answered_by_a_newer_n() { - let catalog = catalog_with_owner("owner", "a.example.com"); - let mut cookie = SCookie::new(&catalog, 100); - cookie.ps.insert("owner".to_owned(), SProviderState { p: Some(150), ..SProviderState::default() }); - let mut entry = KvEntry::minimal("b.example.com", "x", 1); - entry.ps.insert("owner".to_owned(), KvProviderState { n: Some(160), ..KvProviderState::default() }); +```rust +#[cfg(test)] +mod tests { + use std::sync::Arc; - let merged = merge_with_kv(&cookie, &present(entry), &CookieMergeContext::for_test(&catalog, 200)); - assert_eq!(merged.cookie.ps.get("owner").and_then(|state| state.p), None, "should clear the marker"); - } + use super::*; + use crate::ec::identity::catalog::ProviderCatalog; + use crate::ec::identity::input::SeededRandom; + use crate::ec::identity::mutation::{PartnerIdChange, ProviderStateChange}; + use crate::ec::identity::revision::{RecordRevision, Revision}; + use crate::ec::identity::test_support::fixed_provider; + use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId}; - #[test] - fn size_cap_drops_in_order_and_keeps_markers() { - let keys = test_keys(); - let catalog = catalog_with_owner("owner", "a.example.com"); - let random = SeededRandom::new(1); - let mut cookie = SCookie::new(&catalog, 100); - cookie.ps.insert("owner".to_owned(), SProviderState { p: Some(90), ..SProviderState::default() }); - for _ in 0..40 { - cookie.ids.push(SEntry::staged_browser(0, &"u".repeat(60), 100)); - } - let sealed = encode_s_cookie(&keys, EC, &mut cookie, &random, &|_| 0); - assert!(sealed.len() <= MAX_S_COOKIE_BYTES, "should fit the cap"); - assert_eq!(cookie.ps["owner"].p, Some(90), "should keep the marker"); - } - } -``` + const EC_ID: &str = "0000000000000000000000000000000000000000000000000000000000000000.ABC123"; -Add `catalog_with_owner(provider, source)` to `test_support.rs`. It builds a `ProviderCatalog` with one `fixed_provider` that owns `source`. Add `CookieMergeContext::for_test(catalog, now)`, which uses `kv_revalidate_sec = 3600`, test keys, and an `answered_by_eligibility` that always returns `false`. + /// Fails the first `conflicts` generation-checked writes with + /// `PreconditionFailed`, or every write with an error when `error` is set. + struct ScriptedEcKv { /* inner: InMemoryEcKv, conflicts, error, writes */ } -- [ ] **Step 2: Run and see them fail** + /// Seeds `entry` with `create`, then wraps the store. Outline only: + /// write the body after `ConflictInjectingEcKv` (`ec/kv.rs:1877`). + fn graph_with(entry: Option<&KvEntry>, conflicts: u32, error: bool) -> (KvIdentityGraph, Arc); -Run: `cargo test-fastly -- ec::identity::cookie::tests::s_cookie` -Expected: compile errors. + fn removal() -> IdentityMutation { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: "a.example.com".parse().expect("should parse the source"), + provider: "owner".parse().expect("should parse the provider ID"), + expected: Revision::Set(RecordRevision::from_stored("1111111111111111")), + }); + mutation + } -- [ ] **Step 3: Implement** + fn live() -> KvEntry { + let mut entry = KvEntry::minimal("b.example.com", "x", 1); + entry.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { w: Some("owner".to_owned()), rv: Some("1111111111111111".to_owned()), ..KvPartnerId::new("u") }, + ); + entry + } -Write the payload types with serde attributes, then `SCookie::new(catalog, now)` (`v: 1`, `kc: now`, `cf: catalog.fingerprint()`), `SEntry::staged_browser(index, uid, sa)`, `open_s_cookie`, `encode_s_cookie`, `merge_with_kv` and `cookie_view`. Give each merge rule above its own private helper, named after the rule (`fn flip_confirmed`, `fn drop_browser_entries_kv_holds`, `fn answer_markers`, and so on). That way a reviewer can match each helper to its spec bullet. A source index that is out of range for `catalog.configured_sources()` is dropped on open. + fn run(graph: &KvIdentityGraph, removals: &IdentityMutation) -> RemovalWriteOutcome { + let random = SeededRandom::new(1); + let catalog = ProviderCatalog::new( + vec![fixed_provider("owner", &["a.example.com"])], + [("a.example.com".parse().expect("should parse the source"), "owner".parse().expect("should parse the provider ID"))] + .into_iter() + .collect(), + ); + write_removals(graph, EC_ID, removals, &ApplyContext { now: 1_000, random: &random, catalog: &catalog }) + } -- [ ] **Step 4: Run and see them pass** + #[test] + fn writes_the_removal_once_with_a_generation_check() { + let (graph, store) = graph_with(Some(&live()), 0, false); -Run: `cargo test-fastly -- ec::identity::cookie` -Expected: PASS. + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::Written, "should write"); -- [ ] **Step 5: Commit** + assert_eq!(store.generation_checked_writes(), 1, "should write once, with IfGenerationMatch"); + let (entry, _) = graph.get(EC_ID).expect("should read").expect("should hold the row"); + assert!(!entry.ids.contains_key("a.example.com"), "should remove the record"); + assert_eq!(entry.ps["owner"].iv.len(), 1, "should hash the removed UID into iv"); + } -```bash -git add crates/trusted-server-core/src/ec/identity -git commit --signoff -S -m "Encode the ts-eids-s cookie and reconcile it with KV" -``` + #[test] + fn retries_only_a_generation_mismatch_up_to_five_writes() { + let (graph, store) = graph_with(Some(&live()), 2, false); + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::Written, "should win on the third write"); + assert_eq!(store.generation_checked_writes(), 3, "should re-read and retry each conflict"); -### Task 18: The `ts-eids-r` removal cookies + let (graph, store) = graph_with(Some(&live()), 5, false); + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::Failed, "should give up after five conflicts"); + assert_eq!(store.generation_checked_writes(), 5, "should stop at five writes"); -**Files:** + let (graph, store) = graph_with(Some(&live()), 0, true); + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::Failed, "should stop at a non-CAS error"); + assert_eq!(store.generation_checked_writes(), 1, "should not retry a per-key limit"); + } -- Modify: `crates/trusted-server-core/src/ec/identity/cookie.rs` -- Test: `cookie.rs` + #[test] + fn writes_nothing_for_a_tombstone_a_missing_row_or_an_applied_removal() { + let (graph, store) = graph_with(Some(&KvEntry::tombstone(1)), 0, false); + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::NothingToWrite, "should skip a tombstone"); -**Interfaces:** + let (graph, _) = graph_with(None, 0, false); + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::NothingToWrite, "should skip a missing row"); -- Produces: - - `RBatch { rm: Vec, o: BTreeMap, iv: BTreeMap> }` and `RRemoval { s: String /* the source domain itself */, w: String, pv: Option, sa: u64 }` - - `pub(crate) const MAX_R_COOKIE_BYTES: usize = 512;`, `pub(crate) const R_COOKIE_SOFT_LIMIT: usize = 4;` - - `pub(crate) struct OpenedRemovals { cookies: Vec<(String, RBatch)>, unopenable: Vec }` and `pub(crate) fn open_r_cookies(keys, ec_id, headers) -> OpenedRemovals` - - `pub(crate) fn r_batch_to_changes(batch: &RBatch) -> (Vec, BTreeMap)`, which drops entries whose source or provider does not parse - - `pub(crate) fn r_batch_applied(batch: &RBatch, entry: &KvEntry, keys) -> bool`, the "applied" definition of spec 6.2 - - `pub(crate) enum RPlan { Cookie { name, value }, Overflow }` and `pub(crate) fn plan_r_batch(keys, ec_id, batch, inbound_openable: usize, random) -> RPlan`. It returns `Overflow` when the batch does not fit one 512-byte cookie or the request already carries four openable cookies. - - `pub(crate) fn split_r_batch(keys, ec_id, batch, random) -> Vec<(String, String)>`: the overflow fallback, as many 512-byte cookies as it takes, never coarsened (each removal, opt-out and `iv` hash kept exactly) - - `pub(crate) fn new_r_cookie_name(random) -> String`: `__Host-ts-eids-r-` plus 8 random hex digits + let (graph, _) = graph_with(Some(&KvEntry::minimal("b.example.com", "x", 1)), 0, false); + assert_eq!(run(&graph, &removal()), RemovalWriteOutcome::NothingToWrite, "should skip an applied removal"); + assert_eq!(store.generation_checked_writes(), 0, "should not write a tombstone"); + } -- [ ] **Step 1: Write the failing tests** + #[test] + fn an_opt_out_and_its_clear_are_written_without_positive_changes() { + let (graph, _) = graph_with(Some(&live()), 0, false); + let mut mutation = IdentityMutation::default(); + mutation.merge_state( + "owner".parse().expect("should parse the provider ID"), + ProviderStateChange { opt_out: Some(KvOptOut { at: 1_000, fp: "1".to_owned() }), ..ProviderStateChange::default() }, + ); -```rust - mod r_cookie { - use super::super::*; - use crate::ec::identity::input::SeededRandom; - use crate::ec::identity::test_support::test_keys; - use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId, KvProviderState}; + assert_eq!(run(&graph, &mutation.removals()), RemovalWriteOutcome::Written, "should write the opt-out"); - const EC: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + let (entry, _) = graph.get(EC_ID).expect("should read").expect("should hold the row"); + assert!(entry.ps["owner"].o.is_some(), "should store o"); + assert!(!entry.ids.contains_key("a.example.com"), "should remove what the provider wrote"); + } +} +``` - fn removal(source: &str, writer: &str, sa: u64) -> RRemoval { - RRemoval { s: source.to_owned(), w: writer.to_owned(), pv: None, sa } - } +The `ScriptedEcKv` and `graph_with` bodies above are outlines: write them in full, after `ConflictInjectingEcKv`. Keep the `ProviderStateChange` struct update syntax: these tests set one field. - #[test] - fn a_typical_batch_fits_one_cookie_and_a_long_domain_fits_alone() { - let keys = test_keys(); - let random = SeededRandom::new(1); - let four = RBatch { rm: (0..4).map(|i| removal(&format!("s{i}.example.com"), "provider", 1)).collect(), ..RBatch::default() }; - assert!(matches!(plan_r_batch(&keys, EC, &four, 0, &random), RPlan::Cookie { .. }), "four typical removals fit"); - - let label = "a".repeat(63); - let long_domain = format!("{label}.{label}.{label}.{}.com", "b".repeat(57)); - assert_eq!(long_domain.len(), 253, "should be a 253-byte domain"); - let one = RBatch { rm: vec![removal(&long_domain, "a-long-provider-id", 1)], ..RBatch::default() }; - assert!(matches!(plan_r_batch(&keys, EC, &one, 0, &random), RPlan::Cookie { .. }), "one long removal fits"); - } +- [ ] **Step 2: Run and see them fail** - #[test] - fn overflow_when_too_big_or_four_cookies_present() { - let keys = test_keys(); - let random = SeededRandom::new(1); - let big = RBatch { rm: (0..12).map(|i| removal(&format!("s{i}.example.com"), "provider", 1)).collect(), ..RBatch::default() }; - assert!(matches!(plan_r_batch(&keys, EC, &big, 0, &random), RPlan::Overflow), "too big"); - let small = RBatch { rm: vec![removal("a.example.com", "p", 1)], ..RBatch::default() }; - assert!(matches!(plan_r_batch(&keys, EC, &small, 4, &random), RPlan::Overflow), "four present"); - } +Run: `cargo test-fastly -- ec::identity::removal` +Expected: compile errors. - #[test] - fn split_keeps_every_entry_exactly() { - let keys = test_keys(); - let random = SeededRandom::new(1); - let big = RBatch { - rm: (0..12).map(|i| removal(&format!("s{i}.example.com"), "provider", 1)).collect(), - o: [("provider".to_owned(), 5)].into_iter().collect(), - iv: [("provider".to_owned(), vec!["0123456789abcdef".to_owned()])].into_iter().collect(), - }; - let cookies = split_r_batch(&keys, EC, &big, &random); - assert!(cookies.iter().all(|(_, value)| value.len() <= MAX_R_COOKIE_BYTES), "each fits"); - let mut headers = http::HeaderMap::new(); - let joined = cookies.iter().map(|(name, value)| format!("{name}={value}")).collect::>().join("; "); - headers.insert(http::header::COOKIE, http::HeaderValue::from_str(&joined).expect("header")); - let opened = open_r_cookies(&keys, EC, &headers); - let rebuilt = opened.cookies.iter().fold(RBatch::default(), |mut all, (_, batch)| { - all.rm.extend(batch.rm.clone()); - all.o.extend(batch.o.clone()); - for (provider, hashes) in &batch.iv { - all.iv.entry(provider.clone()).or_default().extend(hashes.clone()); - } - all - }); - assert_eq!(rebuilt.rm.len(), 12, "every removal survives"); - assert_eq!(rebuilt.o, big.o, "the opt-out survives unchanged"); - assert_eq!(rebuilt.iv, big.iv, "the iv hash survives"); - } +- [ ] **Step 3: Implement** - #[test] - fn applied_rules() { - let keys = test_keys(); - let mut entry = KvEntry::minimal("b.example.com", "x", 1); - entry.ids.insert( - "a.example.com".to_owned(), - KvPartnerId { w: Some("p".to_owned()), at: Some(10), ..KvPartnerId::new("u") }, - ); - assert!(!r_batch_applied(&RBatch { rm: vec![removal("a.example.com", "p", 10)], ..RBatch::default() }, &entry, &keys), "pending"); - assert!(r_batch_applied(&RBatch { rm: vec![removal("a.example.com", "p", 9)], ..RBatch::default() }, &entry, &keys), "re-acquired after sa"); - assert!(r_batch_applied(&RBatch { rm: vec![removal("a.example.com", "q", 10)], ..RBatch::default() }, &entry, &keys), "other writer"); - - entry.ps.insert("p".to_owned(), KvProviderState { o: Some(KvOptOut { at: 5, fp: "0".repeat(16) }), ..KvProviderState::default() }); - let opt_out = |at| RBatch { o: [("p".to_owned(), at)].into_iter().collect(), ..RBatch::default() }; - assert!(r_batch_applied(&opt_out(5), &entry, &keys), "KV o.at at or after"); - assert!(!r_batch_applied(&opt_out(6), &entry, &keys), "newer opt-out pending"); - entry.ps.get_mut("p").expect("p").oc = Some(6); - assert!(!r_batch_applied(&opt_out(6), &entry, &keys), "same-second clear does not count"); - entry.ps.get_mut("p").expect("p").oc = Some(7); - assert!(r_batch_applied(&opt_out(6), &entry, &keys), "later clear makes it stale"); - } +In `ec/kv.rs`, next to `upsert_partner_ids_from_snapshot`: - #[test] - fn unopenable_cookies_are_listed_for_deletion() { - let keys = test_keys(); - let mut headers = http::HeaderMap::new(); - headers.insert(http::header::COOKIE, http::HeaderValue::from_static("__Host-ts-eids-r-00000000=v1.bad.value")); - let opened = open_r_cookies(&keys, EC, &headers); - assert!(opened.cookies.is_empty(), "nothing opens"); - assert_eq!(opened.unopenable, vec!["__Host-ts-eids-r-00000000".to_owned()], "listed for deletion"); - } +```rust + /// Writes `entry` only if the row is still at `generation`. + /// + /// The identity removal write and the phase 4 flush share this path, so + /// every identity write is validated and generation-checked. + /// + /// # Errors + /// + /// Returns an error when `entry` does not serialize or validate, or the + /// store write fails for a reason other than a generation mismatch. + pub(crate) fn write_if_generation( + &self, + ec_id: &str, + entry: &KvEntry, + generation: u64, + ) -> Result> { + let (body, metadata) = Self::serialize_entry(entry, self.store_name())?; + self.write_entry(ec_id, &body, &metadata, ENTRY_TTL, EcKvWriteMode::IfGenerationMatch(generation)) } ``` -- [ ] **Step 2: Run and see them fail** +`removal.rs`: -Run: `cargo test-fastly -- ec::identity::cookie::tests::r_cookie` -Expected: compile errors. +```rust +//! The shared removal write: a removal-only mutation written with the +//! withdrawal path's CAS loop (spec 5.2 phase 4, 5.3). -- [ ] **Step 3: Implement** +use super::ids::SourceDomain; +use super::mutation::{ApplyContext, IdentityMutation, apply_removals_only}; +use crate::ec::kv::{EcKvSnapshot, KvIdentityGraph, MAX_CAS_RETRIES}; +use crate::ec::kv_backend::EcKvWriteOutcome; +use crate::ec::log_id; + +/// Writes the removal write makes at most, as the withdrawal path does. +pub(crate) const REMOVAL_WRITE_ATTEMPTS: u32 = MAX_CAS_RETRIES; + +/// How a removal write ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RemovalWriteOutcome { + /// The removals are in KV. + Written, + /// Nothing to write: a tombstone, a missing row, or already applied. + NothingToWrite, + /// Not written. The caller may try once more after send. + Failed, +} + +/// Reads the row, applies only `removals` and writes them with a CAS, +/// retrying only a generation mismatch. +#[must_use] +pub(crate) fn write_removals( + graph: &KvIdentityGraph, + ec_id: &str, + removals: &IdentityMutation, + context: &ApplyContext<'_>, +) -> RemovalWriteOutcome { + for _attempt in 0..REMOVAL_WRITE_ATTEMPTS { + let (entry, generation) = match graph.load_snapshot(ec_id) { + EcKvSnapshot::Present { entry, generation: Some(generation), .. } => (entry, generation), + EcKvSnapshot::Missing { .. } => return RemovalWriteOutcome::NothingToWrite, + _ => return RemovalWriteOutcome::Failed, + }; + let Some(applied) = apply_removals_only(removals, &entry, context).filter(|applied| applied.changed) else { + return RemovalWriteOutcome::NothingToWrite; + }; + match graph.write_if_generation(ec_id, &applied.entry, generation) { + Ok(EcKvWriteOutcome::Written) => return RemovalWriteOutcome::Written, + Ok(EcKvWriteOutcome::PreconditionFailed) => {} + Err(error) => { + log::warn!("Identity removal write failed for '{}': {error:?}", log_id(ec_id)); + return RemovalWriteOutcome::Failed; + } + } + } + log::warn!( + "Identity removal write for '{}': CAS conflict after {REMOVAL_WRITE_ATTEMPTS} attempts", + log_id(ec_id) + ); + RemovalWriteOutcome::Failed +} + +/// Logs a removal that every attempt failed to write. Names providers and +/// sources only, never a UID. +pub(crate) fn log_removal_lost(ec_id: &str, removals: &IdentityMutation) { + for (provider, source) in removals.removal_labels() { + log::error!( + "Identity removal lost for '{}': provider '{provider}', source '{}'", + log_id(ec_id), + source.as_ref().map_or("*", SourceDomain::as_str), + ); + } +} +``` -Serialize `RBatch` as compact JSON and omit empty maps. `plan_r_batch` seals once and measures. `split_r_batch` packs greedily: removals one by one, then each provider's opt-out, then each `iv` hash, sealing after each addition and starting a new cookie when the sealed value would pass 512. A single item that can never fit (impossible within the measured budget) is logged at error and still emitted in its own cookie. Never drop it. `r_batch_applied` implements exactly the three spec 6.2 bullets. `iv` entries never keep a cookie alive. +Check the real import paths of `EcKvSnapshot`, `EcKvWriteOutcome`, `MAX_CAS_RETRIES` and `log_id` against `ec/kv.rs`, and make `MAX_CAS_RETRIES` `pub(crate)` if it is private. Register the module in `mod.rs` (`pub(crate) mod removal;`) and in `migration_guards.rs`. - [ ] **Step 4: Run and see them pass** -Run: `cargo test-fastly -- ec::identity::cookie` +Run: `cargo test-fastly -- ec::identity::removal` and `cargo test-fastly -- ec::identity::mutation` Expected: PASS. - [ ] **Step 5: Commit** ```bash -git add crates/trusted-server-core/src/ec/identity -git commit --signoff -S -m "Add per-batch ts-eids-r removal cookies" +git add crates/trusted-server-core/src/ec +git commit --signoff -S -m "Add the shared identity removal write" -m "A removal-only mutation is read, applied and written with the withdrawal path's CAS loop: up to five attempts, retrying only a generation mismatch. It writes nothing for a tombstone, a missing row or a removal that already applied. The capture removal write and the removal-only retry of a failed flush both use it." ``` +--- + ### Task 19: `[ec.identity]` settings **Files:** -- Create: `crates/trusted-server-core/src/ec/identity/config.rs` -- Modify: `crates/trusted-server-core/src/settings.rs` (`Ec`, around line 542) -- Modify: `trusted-server.example.toml` (the `[ec]` section) -- Test: `config.rs`, `settings.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` (`IdentitySettings`, `ProviderSettings`, `ProviderMode` and `RevokeScope` added next to `Ec` at about line 542; `Ec.identity`; `Ec.pull_sync_concurrency` removed) +- Modify: `crates/trusted-server-core/src/config.rs` (`validate_identity_settings` and `validate_removed_keys`, called from both validators) +- Modify: `crates/trusted-server-core/src/ec/pull_sync.rs` (reads `[ec.identity].max_concurrency` until Task 29 removes it), `crates/trusted-server-core/src/config_payload.rs` (the round-trip test at about line 294) +- Modify: `trusted-server.example.toml` (line 83), `crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml` (line 15), the Viceroy configs generated from it (`scripts/generate-integration-viceroy-configs.sh`), and `docs/guide/configuration.md` (the `pull_sync_concurrency` row at about line 646) +- Test: `settings.rs`, `config.rs`, and the CLI overlay test `crates/trusted-server-cli/tests/config_env_overlay.rs`, which reads the integration fixture **Interfaces:** -- Produces: - - `IdentitySettings { max_concurrency: Option, hold_budget_ms: u64, hold_retry_sec: u64, max_response_bytes: usize, pending_grace_sec: u64, staged_max_age_sec: u64, kv_revalidate_sec: u64, providers: BTreeMap }`, with the defaults from Global Constraints and `Default` - - `ProviderSettings { mode: ProviderMode, timeout_ms: Option, sources: Option>, refresh_after_sec: u64, max_calls_per_minute: u32, breaker_failures_per_minute: u32, cookie_priority: i32 }` +- Produces, all in `settings.rs` unless noted: + - `IdentitySettings { max_concurrency: usize, hold_budget_ms: u64, hold_retry_sec: u64, max_response_bytes: usize, providers: BTreeMap }`, deriving `Debug, Clone, Deserialize, Serialize` (the settings envelope serializes `Ec`), with spec 10.1's defaults: 4, 100, 300 and 65,536 + - A hand-written `impl Default for IdentitySettings` that calls the same `default_*` functions the serde attributes name. A derived `Default` would give `hold_budget_ms = 0` and the other zeros, which disagree with the serde defaults and fail validation. + - `ProviderSettings { mode: ProviderMode, timeout_ms: Option, after_send_timeout_ms: u64, sources: Option>, refresh_after_sec: u64, max_calls_per_minute: u32, breaker_failures_per_minute: u32, cookie_priority: i32, revoke_scope: Option }`, with the same hand-written `impl Default`. Defaults: `AfterSend`, none, 1,000, none, 86,400, 1, 50, 0, none. - `enum ProviderMode { Hold, AfterSend }`, serde `snake_case`, default `AfterSend` - - `IdentitySettings::effective_max_concurrency(&self, legacy_pull_concurrency: usize) -> usize` - - `ProviderSettings::effective_timeout_ms(&self, settings: &IdentitySettings) -> u64` - - `IdentitySettings::provider(&self, id: &IdentityProviderId) -> ProviderSettings`, which returns the defaults when there is no table - - `pub(crate) fn validate_identity_settings(settings: &IdentitySettings) -> Result<(), Report>`, covering the numeric rules of spec 10.2: `hold_budget_ms <= 1000`, `max_concurrency != Some(0)`, `refresh_after_sec > 0`, `kv_revalidate_sec < 86_400`, `timeout_ms <= hold_budget_ms` for hold mode. Unknown provider IDs are checked later, in Task 22, when the catalog exists. + - `pub enum RevokeScope { Provider, Global }`, serde `snake_case`, no `Default` derive: an unset value is `None`, and `ProviderSettings::revoke_scope(&self) -> RevokeScope` supplies `Provider`. Nothing in Part 1 reads it. Task 38 applies it. + - `ProviderSettings::hold_timeout_ms(&self, identity: &IdentitySettings) -> u64`: `timeout_ms`, else `hold_budget_ms`. The coordinator caps it at the remaining hold budget (Task 37). Any dispatch after send uses `after_send_timeout_ms`, including a hold-mode provider on a navigation that runs no auction (spec 10.1). + - `IdentitySettings::provider(&self, id: &IdentityProviderId) -> ProviderSettings`, which returns the defaults when there is no table. Task 22 adds `provider_with_base` beside it without changing this signature. - `Ec.identity: IdentitySettings` with `#[serde(default)]` + - `Ec.pull_sync_concurrency` is removed. In its place, `#[serde(default, rename = "pull_sync_concurrency", skip_serializing)] pub removed_pull_sync_concurrency: Option` catches the key, so `deny_unknown_fields` does not reject it with a generic message, and validation names its replacement instead. + - In `config.rs`: `pub(crate) fn validate_identity_settings(settings: &IdentitySettings) -> Result<(), Report>`, covering the numeric rules of spec 10.2: `hold_budget_ms <= 1000`, `max_concurrency != 0`, `refresh_after_sec > 0`, `after_send_timeout_ms` in `1..=10_000`, and `timeout_ms <= hold_budget_ms` for a hold-mode provider. A `mode` or `revoke_scope` that is not one of its values fails to parse, so serde enforces those rules. Rules that need descriptors (an unknown provider ID, `revoke_scope` for a provider without `captures`) live in Task 22's `validate_identity_catalog`, which Task 33 extends. + - In `config.rs`: `pub(crate) fn validate_removed_keys(settings: &Settings) -> Result<(), Report>`. In this task it rejects `ec.pull_sync_concurrency` with an error naming `[ec.identity].max_concurrency`. Task 22 adds `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]`. + +Spec 10.2 removes the key outright instead of deprecating it, so there is no fallback and no warning: `effective_max_concurrency` and its legacy argument do not exist. Every config that sets the key fails validation until it moves the value, and the error says where. - [ ] **Step 1: Write the failing tests** +In the `settings.rs` test module (each name starts with `identity_settings_`, so one filter runs them all): + ```rust -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn identity_settings_defaults_without_any_table() { + let settings: IdentitySettings = toml::from_str("").expect("should parse an empty table"); + assert_eq!(settings.max_concurrency, 4, "should default the concurrency to 4"); + assert_eq!(settings.hold_budget_ms, 100, "should default the hold budget to 100 ms"); + assert_eq!(settings.hold_retry_sec, 300, "should default the hold retry to 300 s"); + assert_eq!(settings.max_response_bytes, 65_536, "should default the response cap to 64 KiB"); + let provider = settings.provider(&"anything".parse().expect("should parse the provider id")); + assert_eq!(provider.mode, ProviderMode::AfterSend, "should default to after-send mode"); + assert_eq!(provider.after_send_timeout_ms, 1000, "should default the after-send timeout to 1 s"); + assert_eq!(provider.hold_timeout_ms(&settings), 100, "should default the hold timeout to the budget"); + assert_eq!( + (provider.refresh_after_sec, provider.max_calls_per_minute, provider.breaker_failures_per_minute, provider.cookie_priority), + (86_400, 1, 50, 0), + "should use the spec 10.1 provider defaults" + ); + assert_eq!(provider.revoke_scope(), RevokeScope::Provider, "should scope revokes to the provider by default"); + validate_identity_settings(&settings).expect("should accept the defaults"); + } #[test] - fn defaults_without_any_table() { - let settings: IdentitySettings = toml::from_str("").expect("should parse empty"); - assert_eq!(settings.hold_budget_ms, 100, "hold budget"); - assert_eq!(settings.hold_retry_sec, 300, "hold retry"); - assert_eq!(settings.max_response_bytes, 65_536, "max bytes"); - assert_eq!(settings.pending_grace_sec, 300, "grace"); - assert_eq!(settings.staged_max_age_sec, 604_800, "staged age"); - assert_eq!(settings.kv_revalidate_sec, 3600, "revalidate"); - assert_eq!(settings.effective_max_concurrency(4), 4, "falls back to the legacy key"); - let provider = settings.provider(&"anything".parse().expect("id")); - assert_eq!(provider.mode, ProviderMode::AfterSend, "mode"); - assert_eq!(provider.effective_timeout_ms(&settings), 1000, "after-send timeout"); - assert_eq!((provider.refresh_after_sec, provider.max_calls_per_minute, provider.breaker_failures_per_minute, provider.cookie_priority), (86_400, 1, 50, 0), "provider defaults"); - validate_identity_settings(&settings).expect("defaults are valid"); + fn identity_settings_default_matches_an_empty_table() { + let parsed: IdentitySettings = toml::from_str("").expect("should parse an empty table"); + assert_eq!( + serde_json::to_value(IdentitySettings::default()).expect("should serialize the default"), + serde_json::to_value(parsed).expect("should serialize the parsed value"), + "should give Default the serde defaults, not zeros" + ); + let provider: ProviderSettings = toml::from_str("").expect("should parse an empty provider table"); + assert_eq!( + serde_json::to_value(ProviderSettings::default()).expect("should serialize the default"), + serde_json::to_value(provider).expect("should serialize the parsed value"), + "should give the provider Default the serde defaults" + ); } #[test] - fn hold_mode_timeout_defaults_to_the_budget_and_bounds_are_enforced() { + fn identity_settings_timeouts_and_bounds_are_enforced() { let settings: IdentitySettings = toml::from_str( - "hold_budget_ms = 120\n[providers.id5]\nmode = \"hold\"\n", + "hold_budget_ms = 120\n[providers.id5]\nmode = \"hold\"\nafter_send_timeout_ms = 2500\n", ) - .expect("should parse"); - let provider = settings.provider(&"id5".parse().expect("id")); - assert_eq!(provider.effective_timeout_ms(&settings), 120, "hold timeout = budget"); + .expect("should parse a hold-mode table"); + let provider = settings.provider(&"id5".parse().expect("should parse the provider id")); + assert_eq!(provider.hold_timeout_ms(&settings), 120, "should use the budget as the hold timeout"); + assert_eq!(provider.after_send_timeout_ms, 2500, "should read the after-send timeout"); + validate_identity_settings(&settings).expect("should accept a 2.5 s after-send timeout"); for (name, toml_text) in [ ("budget over 1000", "hold_budget_ms = 1001"), ("zero concurrency", "max_concurrency = 0"), - ("revalidate a day", "kv_revalidate_sec = 86400"), ("zero refresh", "[providers.p]\nrefresh_after_sec = 0"), - ("timeout over budget", "[providers.p]\nmode = \"hold\"\ntimeout_ms = 101"), + ("zero after-send timeout", "[providers.p]\nafter_send_timeout_ms = 0"), + ("after-send timeout over 10 s", "[providers.p]\nafter_send_timeout_ms = 10001"), + ("hold timeout over budget", "[providers.p]\nmode = \"hold\"\ntimeout_ms = 101"), ] { - let settings: IdentitySettings = toml::from_str(toml_text).expect(name); - validate_identity_settings(&settings).expect_err(name); + let settings: IdentitySettings = toml::from_str(toml_text).expect("should parse before validation"); + validate_identity_settings(&settings).expect_err(&format!("should reject {name}")); } - toml::from_str::("[providers.p]\nmode = \"sometimes\"").expect_err("bad mode"); + let edge: IdentitySettings = toml::from_str("[providers.p]\nafter_send_timeout_ms = 10000") + .expect("should parse the upper bound"); + validate_identity_settings(&edge).expect("control: should accept exactly 10 s"); + toml::from_str::("[providers.p]\nmode = \"sometimes\"").expect_err("should reject an unknown mode"); + toml::from_str::("[providers.p]\nrevoke_scope = \"everyone\"").expect_err("should reject an unknown revoke scope"); + } + + #[test] + fn identity_settings_revoke_scope_parses_and_keeps_its_default_when_unset() { + let settings: IdentitySettings = + toml::from_str("[providers.lockr]\nrevoke_scope = \"global\"\n").expect("should parse the revoke scope"); + let lockr = settings.provider(&"lockr".parse().expect("should parse the provider id")); + assert_eq!(lockr.revoke_scope(), RevokeScope::Global, "should read a global revoke scope"); + let other = settings.provider(&"other".parse().expect("should parse the provider id")); + assert_eq!(other.revoke_scope(), RevokeScope::Provider, "control: another provider should keep the provider scope"); + toml::from_str::("[providers.lockr]\ncapture_enabled = true\n") + .expect("should still load a key this build does not know"); } -} ``` -Use whichever TOML crate core already uses in tests (`grep -n "toml" crates/trusted-server-core/Cargo.toml`). If only `config` is available, build the value through the same path `settings.rs` tests use. +Core already has `toml` as a dependency (`crates/trusted-server-core/Cargo.toml`), so the tests parse with `toml::from_str`. The last assertion pins the lenient tables: a provider table holding a key this build does not know still loads, so a rollback from a later Part keeps loading its config. `capture_enabled` was a key of earlier spec revisions and is the example. + +Also in `settings.rs` tests, add `identity_settings_ec_without_identity_table_parses`: a full settings TOML with no `[ec.identity]`, loaded the way the existing `Ec` tests do, gets the defaults. + +In `config.rs` tests, add `removed_pull_sync_concurrency_is_rejected_with_its_replacement`: settings with `ec.pull_sync_concurrency = 3` parse, then fail both validators with an error whose text contains `[ec.identity].max_concurrency`. The control, the same settings with `[ec.identity] max_concurrency = 3` instead, passes. - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- ec::identity::config` +Run: `cargo test-fastly -- identity_settings` and `cargo test-fastly -- config::tests::removed_pull_sync_concurrency` Expected: compile errors. - [ ] **Step 3: Implement** -Write the structs with `#[serde(default = "...")]` per field and `#[serde(deny_unknown_fields)]`, which matches the rest of `settings.rs`. Check the convention there first. Add to `Ec`: +Add `IdentitySettings`, `ProviderSettings`, `ProviderMode` and `RevokeScope` in `settings.rs` next to `Ec`, with each field under `#[serde(default = "...")]` and the hand-written `Default` impls. Do not move these types to another file. + +`IdentitySettings` and `ProviderSettings` stay lenient: no `#[serde(deny_unknown_fields)]`, unlike the rest of `settings.rs`, so a rollback to an earlier Part still loads a config written for a later one (spec 14). Add `Ec.identity`: ```rust /// Server-side identity provider policy (`[ec.identity]`). @@ -5896,62 +7821,97 @@ Write the structs with `#[serde(default = "...")]` per field and `#[serde(deny_u pub identity: IdentitySettings, ``` -Call `validate_identity_settings(&self.ec.identity)` from both `validate_settings_for_deploy` and `validate_settings_for_runtime` in `config.rs`, next to `PartnerRegistry::...`. Mark `pull_sync_concurrency`'s doc comment as deprecated in favour of `[ec.identity].max_concurrency`, and log a `warn!` once at settings load when `identity.max_concurrency` is unset and `pull_sync_concurrency` is not its default. In `trusted-server.example.toml`, add a commented `[ec.identity]` block that copies spec 10.1's comments and values, with `example.com` partner names. +Replace `Ec.pull_sync_concurrency` and `Ec::default_pull_sync_concurrency` with the catch field from Interfaces, documented as removed and rejected by validation. In `ec/pull_sync.rs`, read `settings.ec.identity.max_concurrency.max(1)` where it read `settings.ec.pull_sync_concurrency.max(1)` (`pull_sync.rs:184`), and move its tests' `settings.ec.pull_sync_concurrency = N` to `settings.ec.identity.max_concurrency = N`. Task 29 removes the file. Fix `config_payload.rs:294` to compare `ec.identity.max_concurrency`. + +With the new defaults, the serialized `Ec` holds every `[ec.identity]` default, so whole-config snapshot, envelope-hash and CLI snapshot tests change. Update their expected values in this task and name them in the commit body. + +Call `validate_identity_settings(&settings.ec.identity)` and `validate_removed_keys(settings)` from both `validate_settings_for_deploy` (`config.rs:249`) and `validate_settings_for_runtime` (`config.rs:270`), next to `PartnerRegistry::...`. + +Every config that sets the removed key must move it in this task: + +- `trusted-server.example.toml` (line 83): replace `pull_sync_concurrency = 3` with an `[ec.identity]` table holding `max_concurrency = 3`. Add spec 10.1's other `[ec.identity]` keys and one commented `[ec.identity.providers.]` table (with `after_send_timeout_ms`, `cookie_priority` and `revoke_scope`) as a commented block, with `example.com` partner names. +- `crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml` (line 15), which the parity tests, `src/bin/generate-viceroy-config.rs` and the CLI overlay test read: the same change. Then regenerate the Viceroy configs with `scripts/generate-integration-viceroy-configs.sh`. +- `docs/guide/configuration.md`: replace the `pull_sync_concurrency` row with an `[ec.identity]` section that lists the keys and defaults of spec 10.1. Task 31 adds the rest of the docs. + +The example-config and CLI template tests must keep passing (`grep -rn "trusted-server.example.toml" crates/ --include=*.rs`); run `./scripts/test-cli.sh` for the CLI ones. - [ ] **Step 4: Run and see them pass** -Run: `cargo test-fastly -- ec::identity::config`, `cargo test-fastly -- settings`, `cargo test-fastly -- config::` -Expected: PASS. Existing configs with no `[ec.identity]` still parse. +Run: `cargo test-fastly -- identity_settings`, `cargo test-fastly -- settings`, `cargo test-fastly -- config::`, `cargo test-fastly -- ec::pull_sync`, `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` and `./scripts/test-cli.sh` +Expected: PASS. A config with no `[ec.identity]` still loads with the defaults. A config with `ec.pull_sync_concurrency` is rejected with an error that names its replacement. - [ ] **Step 5: Commit** ```bash -git add -A crates/trusted-server-core/src trusted-server.example.toml -git commit --signoff -S -m "Add [ec.identity] coordinator settings" +git add -A crates/ trusted-server.example.toml docs/guide/configuration.md +git commit --signoff -S -m "Add [ec.identity] coordinator settings" -m "Adds coordinator policy and per-provider tables, including after_send_timeout_ms and revoke_scope. ec.pull_sync_concurrency is removed: a config that sets it fails validation with an error naming [ec.identity].max_concurrency, and the example config and the integration fixture move their value there." ``` +--- + ### Task 20: Coordinator eligibility and outcomes **Files:** - Create: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (pure logic only in this task) -- Modify: `mod.rs`, `migration_guards.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs`, `mod.rs`, `migration_guards.rs` - Test: `coordinator.rs` **Interfaces:** -- Consumes: catalog, servable, mutation, cookie view, config, policy (Tasks 12 to 19) +- Consumes: catalog, servable, mutation, revision, settings, policy (Tasks 12 to 19), and `entry_sets`, `EntrySets` and `EntrySet` (Task 10) - Produces: - - `pub(crate) struct EligibilityView`, the merged view of spec 4.7, built by `EligibilityView::merge(kv: Option<&KvEntry>, cookie: Option<&SCookie>, removals: &[RBatch], catalog, now) -> Self`. It holds per-source records (with KV as the base when read, else the cookie's last confirmed copy, cookie `k: 0` entries on top when their `pv` matches, and `ts-eids-r` removals hiding records), per-provider `n` (the later of cookie and KV), `iv` (union), `o` (the staged opt-out counts unless `ts-eids-s` stages a later `oc`, and the mirrored opt-out yields to KV when KV was read), and markers (cookie only). - - `pub(crate) struct RequestFacts { is_real_browser, ec_persistence_allowed, policy_signals: PolicySignals, writer: bool, new_visitor: bool, runs_auction: bool, now: UnixTime }` - - `pub(crate) enum Phase { Hold, AfterSendDryRun, AfterSend }` - - `pub(crate) enum Eligibility { Eligible { kinds: BTreeMap }, Skip(SkipReason) }`, with a `SkipReason` enum naming each of steps 1 to 8 (step 9, the burst cap, is checked at dispatch) - - `pub(crate) fn eligibility(provider: &dyn IdentityProvider, settings: &ProviderSettings, identity: &IdentitySettings, view: &EligibilityView, facts: &RequestFacts, consent: &ConsentContext, phase: Phase, catalog: &ProviderCatalog) -> Eligibility` + - `pub(crate) struct EligibilityView`, the view of spec 4.7, built by `EligibilityView::from_kv(entry: &KvEntry) -> Self` from the row the request read, and nothing else. It holds the per-source records (whole `KvPartnerId`s) and, per provider, `n`, `f`, `o`, `oc`, `iv`, `ov` and the `st` the call reads. It never reads `ts-eids-s`: every removal is already in KV (spec 5.3). + - `EligibilityView::is_opted_out(&self, provider: &IdentityProviderId) -> bool`: `o` is set and `oc` is not strictly later than `o.at` + - `pub(crate) struct RequestFacts { is_real_browser, ec_persistence_allowed, policy_signals: PolicySignals, writer: bool, new_visitor: bool, runs_auction: bool, now: UnixTime, available_inputs: BTreeSet }` + - `pub(crate) enum Phase { Hold, AfterSend }` + - `pub(crate) enum Eligibility { Eligible { kinds: BTreeMap }, Skip(SkipReason) }`, with a `SkipReason` enum naming each of steps 1 to 7 (step 6, the breaker, is Task 35's; step 8, the burst cap, is checked at dispatch) + - `pub(crate) struct EligibilityInput<'a> { provider: &'a dyn IdentityProvider, settings: &'a ProviderSettings, identity: &'a IdentitySettings, catalog: &'a ProviderCatalog, consent: &'a ConsentContext }`, so `eligibility` stays within 7 arguments + - `pub(crate) fn eligibility(input: &EligibilityInput<'_>, view: &EligibilityView, facts: &RequestFacts, phase: Phase) -> Eligibility` + - `pub(crate) fn fingerprint_bits(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> String`: the consent an opt-out's `fp` records, as a plain lowercase hex bitmask of at most 16 digits (`format!("{bits:x}")` of a `u64`), never a hash (spec 8, 9.3). Part 1 covers the signals TS decodes today, one bit each in this order: each TCF purpose the policy reads (1 to 10), the vendor bit when the policy names one, the US sale opt-out (from `us_privacy` or GPP), GPC and DNT. A signal the policy relaxes stays 0. Task 34 appends the sharing and targeted-advertising opt-outs after these and keeps these bits where they are. - `pub(crate) enum CallOutcome { Resolved(Resolution), Skipped, InputUnavailable, InvalidState(Option), HoldCutOff, InputFailure, TransportFailure, BreakerOpen }` - - `pub(crate) fn outcome_to_changes(provider: &dyn IdentityProvider, outcome: CallOutcome, view: &EligibilityView, context: &OutcomeContext<'_>) -> OutcomeChanges`, where `OutcomeChanges { removals: Vec, result: Option, state: ProviderStateChange }`. This is the section 8 table, row by row. `OutcomeContext` holds `now`, `keys`, `catalog`, provider settings, `identity` settings, `expected_state` and the consent fingerprint bits. + - `pub(crate) fn outcome_to_changes(provider: &dyn IdentityProvider, outcome: CallOutcome, view: &EligibilityView, context: &OutcomeContext<'_>) -> OutcomeChanges`, where `OutcomeChanges { changes: Vec, result: Option, state: ProviderStateChange }`. This is the section 8 table, row by row. `changes` holds every removal, and the positive changes of a provider without private state. `OutcomeContext` holds `now`, `catalog`, provider settings, `identity` settings, `expected_state: Option` and the consent fingerprint bits from `fingerprint_bits`. Eligibility steps, in order (spec 4.7): 1. `facts.is_real_browser` 2. `facts.ec_persistence_allowed` 3. `policy_allows(descriptor.consent_policy(), consent, facts.policy_signals)` -4. Per-source kinds: - - **Owned by this provider.** `Initial` when there is no record. `Refresh` when there is one and it is due: expired, written by someone else, `ra` passed, or no `ra` and `at + refresh_after_sec` passed or `at` missing. A browser-seeded UID whose hash is in `iv` is never due. - - **Unowned.** `Initial` only, and only when there is no servable record. An expired record, or one whose writer is unconfigured, counts as absent. - - A source whose kind the provider does not list, or whose needs (`resolves.needs(kind)`) are not available this request, is left out. No sources left means skip. -5. `n` has passed, and no sticky `o` is set (`oc` not strictly later than `o.at`). -6. No unanswered marker for the provider. -7. (Breaker: checked at dispatch in Part 2, Task 35. Part 1 has no breaker.) -8. Mode: - - No provider runs when `facts.new_visitor`. - - `Hold` needs `settings.mode == Hold` and `facts.runs_auction`. A stateful provider also needs `facts.writer`. - - `AfterSendDryRun` and `AfterSend` need `facts.writer`, and either `settings.mode == AfterSend`, or a stateful hold-mode provider on a writer navigation that does not run an auction. - -The needs check in step 4 needs to know which inputs are available. Add `available_inputs: BTreeSet` to `RequestFacts`. +4. Per-source kinds, over `catalog.sources_of(provider)`: + - **Owned by this provider.** `Initial` when there is no record. `Refresh` when there is one and it is due: expired, written by someone else, `ra` passed, or no `ra` and `at + refresh_after_sec` passed or `at` missing. A browser-seeded record any of whose UIDs hashes into the provider's `iv` is never due (spec 7.2). An expired refresh anchor (spec 5.4) is a stored record, so its kind stays `Refresh`. + - **Unowned.** `Initial` only, and only when KV has no unexpired ID for the source. An expired record, or one whose writer is disabled or no longer configured, counts as absent. This is not the servability check: a record hidden only by this request's consent or by an opt-out still counts as present, because `FillMissing` would not replace it and the call would be wasted. + - A source whose kind the provider does not list, or whose needs (`resolves.needs(kind)`) are not in `facts.available_inputs`, is left out. No sources left means skip, logged at debug. +5. `n` has passed, and `view.is_opted_out(provider)` is false. +6. (Breaker: Task 35. Part 1 has no breaker.) +7. Writer and mode: + - No provider runs when `facts.new_visitor` or when `!facts.writer`: stagers run no resolving provider in either mode (spec 5.1). + - `Phase::Hold` needs `settings.mode == Hold` and `facts.runs_auction`. + - `Phase::AfterSend` needs `settings.mode == AfterSend`, or `settings.mode == Hold` and `!facts.runs_auction`: a hold-mode provider on a writer navigation that runs no auction runs after send instead (spec 4.7 step 7), with `after_send_timeout_ms`. + +How `Resolved(resolution)` becomes changes (spec 4.5, "From entries to an entry set", and spec 8): + +1. Drop entries for sources the provider does not claim after narrowing (`catalog.sources_of`). Log each with the provider ID and source only. +2. Call `entry_sets(&kept, context.now)` (Task 10). It groups the rest by source in order and applies every limit of spec 4.5. Log each `EntryDrop` and each `oversized` source with the provider ID, source and field only, never a value. +3. A source listed in `oversized` has a **discarded set**. A source whose every entry `entry_sets` dropped is absent from `sets`, so the result does not name it and its stored set is left as it is. A set of an expiring source that carries no expiry keeps the stored `exp` when its entries equal the stored `eids`, and is a discarded set otherwise. +4. For a provider with private state, any discarded set rejects the result's positive part whole, exactly as an oversized `replacing_state` does: no `ProviderResult`, `st` unchanged, `f` and `n` as for `Malformed`. The result's removals still apply. For a provider without private state, a discarded set leaves that source's stored record as it is, as if the result did not name the source, and every other set is kept. +5. A result that named at least one source and all of whose sets are discarded is handled as `Malformed` (spec 4.5, section 8): `f += 1` with its backoff, and its removals still apply. Timing comes only from kept entries (Task 10). +6. `EntrySet.refresh_until` from a provider without private state is ignored with a warning. +7. Each kept set becomes `KvPartnerId { w: Some(provider), at: Some(now), exp, ra, ru, eids: set.entries, .. }`, with `exp`, `ra` and `ru` from the `EntrySet`. A one-UID result writes the same one-entry shape browser ingest writes. Descriptor provenance is not stored: Task 28 fills it in at output. +8. Each record becomes `OwnerWrite` with `expected = Some(record_revision(stored))` (or `None` when no record) when this provider owns the source, else `FillMissing`. Either change replaces the source's set whole. A source the result does not name gets no change. +9. `invalidated` becomes `OwnerRemove { source, provider, expected: record_revision(stored) }` for each named source that has a record in the view, plus `n = now`. It carries no hash: `mutation::apply` (Task 15) hashes every UID of the live record it removes into the provider's `ps.iv`. A named source with no record makes no change. +10. `opted_out` becomes `WriterRemove { source, writer: provider }` for every source in the view whose record this provider wrote, and, for a capture-only provider, for every one of its `capture_only_sources` whose record has no writer. Plus `opt_out = KvOptOut { at: now, fp: fingerprint_bits }` and `st = Clear`. The mutation hashes each removed record into `ps.ov`. +11. When the provider has private state, the positive changes plus the state change plus `n` and `f` go into one `ProviderResult` with `expected_state`. Otherwise they go into `changes`, and `n` and `f` into `state`. +12. `state_rejected()` drops the positive part and applies the `Malformed` backoff. + +The other rows of section 8: `Skipped` and `InputUnavailable` set `n = now + refresh_after_sec`. `InvalidState` clears `st`, keeps `f`, and sets `n = now + refresh_after_sec` unless its resolution sets `n`. `HoldCutOff` sets `n = now + hold_retry_sec` and never counts a failure; only a hold dispatch before the auction can be cut off. An after-send timeout is a `TransportFailure`. `InputFailure` is `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)`. `TransportFailure` is `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`. `BreakerOpen` changes nothing. Backoff arithmetic uses `saturating_pow` and `min`, so a large `f` cannot overflow. - [ ] **Step 1: Write the failing tests** -Write one test per eligibility step and one per section 8 row, using `fixed_provider` and `ProviderSettings::default()`. Name them after the rule, for example `skips_when_not_a_real_browser`, `skips_new_visitor_in_every_mode`, `owned_record_written_by_someone_else_is_due_for_refresh`, `browser_seeded_uid_in_iv_is_never_due`, `unowned_source_with_servable_record_is_not_initial`, `expired_unowned_record_counts_as_absent`, `stateful_hold_provider_runs_after_send_without_auction`, `hold_provider_never_runs_on_stager_when_stateful`, `no_match_waits_retry_in_capped_at_seven_days`, `input_failure_backs_off_exponentially_to_a_day`, `transport_failure_backs_off_to_an_hour`, `hold_cut_off_waits_hold_retry_without_counting_a_failure`, `skip_waits_refresh_after`, `invalid_state_clears_state_and_keeps_f`, `oversized_state_rejects_positive_part_but_keeps_removals`, `opted_out_removes_everything_and_clears_state`, `invalidate_records_iv_hash_and_sets_n_now`. Here are three written out in full. Write the rest in the same shape: +Write one test per eligibility step and one per section 8 row, using `fixed_provider` and `ProviderSettings::default()`. Name them after the rule, for example `skips_when_not_a_real_browser`, `skips_new_visitor_in_every_mode`, `stager_runs_no_provider_in_either_phase`, `owned_record_written_by_someone_else_is_due_for_refresh`, `browser_seeded_uid_in_iv_is_never_due`, `expired_refresh_anchor_stays_refresh`, `unowned_source_with_unexpired_record_is_not_initial_even_when_consent_hides_it`, `expired_unowned_record_counts_as_absent`, `record_of_an_unconfigured_writer_counts_as_absent`, `hold_provider_runs_before_the_auction_and_not_after_send`, `hold_provider_runs_after_send_when_the_navigation_runs_no_auction`, `after_send_provider_never_runs_in_hold`, `narrowed_away_source_is_never_a_kind`, `future_n_and_sticky_opt_out_block`, `a_later_clear_lifts_the_opt_out`, `no_match_waits_retry_in_capped_at_seven_days`, `input_failure_backs_off_exponentially_to_a_day`, `transport_failure_backs_off_to_an_hour`, `hold_cut_off_waits_hold_retry_without_counting_a_failure`, `skip_waits_refresh_after`, `invalid_state_clears_state_and_keeps_f`, `oversized_state_rejects_positive_part_but_keeps_removals`, `opted_out_removes_what_it_wrote_and_clears_state`, `capture_only_opt_out_also_removes_browser_records_of_its_sources`, `opted_out_records_the_fingerprint_of_the_current_consent`, `invalidate_removes_the_set_and_sets_n_now` and `invalidating_a_source_without_a_record_makes_no_change`. Add `fingerprint_bits_is_a_plain_hex_bitmask_and_changes_with_each_signal` (one case per signal, each against a control that differs only in it, and an assertion that the value parses with `u64::from_str_radix(.., 16)`). + +Then the entry-set rules (spec 13, "Complete provider EIDs (G13)"): `found_with_several_entries_becomes_one_whole_set_per_source`, `one_uid_result_without_eid_fields_becomes_a_one_entry_record`, `unclaimed_source_entries_are_discarded`, `oversized_set_from_a_stateless_provider_leaves_only_that_source_unchanged`, `oversized_set_from_a_stateful_provider_rejects_the_positive_part_and_keeps_removals`, `result_whose_every_set_is_discarded_is_malformed`, `invalidating_a_set_carries_no_hash_until_it_applies`, `expiring_set_without_expiry_keeps_stored_exp_only_when_unchanged`, `result_without_a_source_leaves_its_set_alone`, and `record_carries_the_earliest_times_of_its_set`. Task 10 tests the limits themselves. These tests check only what the coordinator does with `entry_sets`' output. + +Here are seven written out in full. Write the rest in the same shape: ```rust #[test] @@ -5959,39 +7919,60 @@ Write one test per eligibility step and one per section 8 row, using `fixed_prov let provider = fixed_provider("p", &["a.example.com"]); let view = EligibilityView::empty(); let catalog = ProviderCatalog::new(vec![provider.clone()], BTreeMap::new()); - for phase in [Phase::Hold, Phase::AfterSendDryRun, Phase::AfterSend] { + let settings = ProviderSettings::default(); + let identity = IdentitySettings::default(); + let consent = allowing_consent(); + let input = EligibilityInput { provider: provider.as_ref(), settings: &settings, identity: &identity, catalog: &catalog, consent: &consent }; + for phase in [Phase::Hold, Phase::AfterSend] { let facts = RequestFacts { new_visitor: true, ..facts_for_writer() }; assert!( - matches!( - eligibility(provider.as_ref(), &ProviderSettings::default(), &IdentitySettings::default(), &view, &facts, &allowing_consent(), phase, &catalog), - Eligibility::Skip(SkipReason::NewVisitor) - ), - "no provider runs on a new visitor's first navigation ({phase:?})" + matches!(eligibility(&input, &view, &facts, phase), Eligibility::Skip(SkipReason::NewVisitor)), + "should run no provider on a new visitor's first navigation ({phase:?})" ); } let returning = facts_for_writer(); assert!( - matches!( - eligibility(provider.as_ref(), &ProviderSettings::default(), &IdentitySettings::default(), &view, &returning, &allowing_consent(), Phase::AfterSend, &catalog), - Eligibility::Eligible { .. } - ), - "control: a returning writer is eligible" + matches!(eligibility(&input, &view, &returning, Phase::AfterSend), Eligibility::Eligible { .. }), + "control: a returning writer should be eligible" ); } + #[test] + fn hold_provider_runs_after_send_when_the_navigation_runs_no_auction() { + let provider = fixed_provider("p", &["a.example.com"]); + let view = EligibilityView::empty(); + let catalog = ProviderCatalog::new(vec![provider.clone()], BTreeMap::new()); + let settings = ProviderSettings { mode: ProviderMode::Hold, ..ProviderSettings::default() }; + let identity = IdentitySettings::default(); + let consent = allowing_consent(); + let input = EligibilityInput { provider: provider.as_ref(), settings: &settings, identity: &identity, catalog: &catalog, consent: &consent }; + let eligible = |facts: &RequestFacts, phase| matches!(eligibility(&input, &view, facts, phase), Eligibility::Eligible { .. }); + + let no_auction = RequestFacts { runs_auction: false, ..facts_for_writer() }; + let auction = RequestFacts { runs_auction: true, ..facts_for_writer() }; + let stager = RequestFacts { writer: false, runs_auction: true, ..facts_for_writer() }; + + assert!(eligible(&no_auction, Phase::AfterSend), "should move after send without an auction"); + assert!(!eligible(&no_auction, Phase::Hold), "should not hold without an auction"); + assert!(eligible(&auction, Phase::Hold), "control: should hold before an auction"); + assert!(!eligible(&auction, Phase::AfterSend), "should not also run after send"); + assert!(!eligible(&stager, Phase::Hold), "should never run on a stager"); + } + #[test] fn no_match_waits_retry_in_capped_at_seven_days() { let provider = fixed_provider("p", &["a.example.com"]); - let context = outcome_context_at(1_000); + let fixture = OutcomeFixture::new(); + let context = fixture.context_at(1_000); let default = outcome_to_changes(provider.as_ref(), CallOutcome::Resolved(Resolution::no_match()), &EligibilityView::empty(), &context); - assert_eq!(default.state.n, Some(1_000 + 86_400), "24 h default"); + assert_eq!(default.state.n, Some(1_000 + 86_400), "should wait the 24 h default"); let long = outcome_to_changes( provider.as_ref(), CallOutcome::Resolved(Resolution::no_match().with_retry_in(Duration::from_secs(30 * 86_400))), &EligibilityView::empty(), &context, ); - assert_eq!(long.state.n, Some(1_000 + 7 * 86_400), "capped at 7 days"); + assert_eq!(long.state.n, Some(1_000 + 7 * 86_400), "should cap the wait at 7 days"); } #[test] @@ -5999,13 +7980,129 @@ Write one test per eligibility step and one per section 8 row, using `fixed_prov let provider = fixed_provider("p", &["a.example.com"]); let mut view = EligibilityView::empty(); view.set_failures_for_test("p", 5); - let changes = outcome_to_changes(provider.as_ref(), CallOutcome::TransportFailure, &view, &outcome_context_at(1_000)); + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes(provider.as_ref(), CallOutcome::TransportFailure, &view, &fixture.context_at(1_000)); assert_eq!(changes.state.f, Some(6), "should count the failure"); - assert_eq!(changes.state.n, Some(1_000 + 3_600), "min(5 min x 2^5, 1 h) = 1 h"); + assert_eq!(changes.state.n, Some(1_000 + 3_600), "should back off min(5 min x 2^5, 1 h) = 1 h"); + } + + #[test] + fn found_with_several_entries_becomes_one_whole_set_per_source() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let resolution = Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a").with_atype(1)) + .with_uid(ResolvedUid::new("example-b").with_atype(3).with_ext(json!({ "stype": "ppuid" }))) + .with_inserter("example.com") + .with_mm(3), + ResolvedEid::new(&a, ResolvedUid::new("example-c")).with_matcher("partner.example.com"), + ]); + + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes(provider.as_ref(), CallOutcome::Resolved(resolution), &EligibilityView::empty(), &fixture.context_at(1_000)); + + assert_eq!(changes.changes.len(), 1, "should make one change for the one source"); + let record = positive_record(&changes.changes[0]).expect("should be a positive change"); + assert_eq!(record.w.as_deref(), Some("p"), "should name the writer"); + assert_eq!( + record.eids, + vec![ + KvEid { + uids: vec![ + KvUid { id: "example-a".to_owned(), atype: Some(1), ext: None }, + KvUid { id: "example-b".to_owned(), atype: Some(3), ext: Some(json!({ "stype": "ppuid" })) }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + KvEid { + uids: vec![KvUid { id: "example-c".to_owned(), atype: None, ext: None }], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ], + "should store the whole set, in the order the provider returned it" + ); + } + + #[test] + fn oversized_set_from_a_stateful_provider_rejects_the_positive_part_and_keeps_removals() { + let provider = stateful_fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&b, record_for_test("example-old", Some("p"), 900)); + // Three entries of about 2 KB each: the set is over MAX_ENTRY_SET_BYTES. + let large_ext = json!({ "segments": "x".repeat(2_000) }); + let entries = (0..3).map(|index| ResolvedEid::new(&a, ResolvedUid::new(format!("example-{index}"))).with_ext(large_ext.clone())); + let resolution = Resolution::found(entries) + .also_invalidate(&b) + .replacing_state(&json!({ "token": "example-token" })); + + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes(provider.as_ref(), CallOutcome::Resolved(resolution), &view, &fixture.context_at(1_000)); + + assert!(changes.result.is_none(), "should store no ID and no state change"); + assert!( + changes.changes.iter().any(|change| matches!(change, PartnerIdChange::OwnerRemove { source, .. } if *source == b)), + "should still apply the invalidation" + ); + assert!(matches!(changes.state.st, StateUpdate::Keep), "should keep the old state"); + assert_eq!(changes.state.f, Some(1), "should handle it as Malformed"); + assert_eq!(changes.state.n, Some(1_000 + 3_600), "should back off min(1 h x 2^0, 24 h)"); + } + + #[test] + fn invalidating_a_set_carries_no_hash_until_it_applies() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + let mut record = record_for_test("example-0-0", Some("p"), 900); + record.eids = (0..4) + .map(|chunk| KvEid { + uids: (0..8).map(|index| KvUid { id: format!("example-{chunk}-{index}"), atype: None, ext: None }).collect(), + inserter: None, + matcher: None, + mm: None, + ext: None, + }) + .collect(); + let expected = record_revision(&record); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, record); + + let changes = outcome_to_changes(provider.as_ref(), CallOutcome::Resolved(Resolution::invalidate(&a)), &view, &fixture.context_at(1_000)); + + assert_eq!(changes.changes.len(), 1, "should make one removal for the 32-UID set"); + assert!( + matches!( + &changes.changes[0], + PartnerIdChange::OwnerRemove { source, provider, expected: revision } + if *source == a && provider.as_str() == "p" && *revision == expected + ), + "should remove the whole set at the revision it read" + ); + assert_eq!(changes.state.n, Some(1_000), "should set n = now"); } ``` -Add `facts_for_writer()`, `allowing_consent()` and `outcome_context_at(now)` to `test_support.rs`. `facts_for_writer` returns a real browser, persistence allowed, a writer, a returning visitor, no auction, and every input available. +Add these to `test_support.rs`: + +- `facts_for_writer()`, which returns a real browser, persistence allowed, a writer, a returning visitor, no auction, and every input available +- `allowing_consent()` +- `OutcomeFixture`, an owned fixture holding the catalog, the provider and identity settings and the fingerprint bits, with `OutcomeFixture::new()` and `fn context_at(&self, now: u64) -> OutcomeContext<'_>`. A helper cannot return an `OutcomeContext` that borrows values it creates, so tests keep the fixture in a local and borrow from it. +- `source(domain) -> SourceDomain` and `pid(id) -> IdentityProviderId`, if Task 14 did not already add them +- `stateful_fixed_provider(id, sources)`, a `fixed_provider` declared `with_private_state()` +- `record_for_test(uid, writer: Option<&str>, at) -> KvPartnerId`, a one-UID record with a fresh `rv` +- `EligibilityView::empty()`, `set_record_for_test` and `set_failures_for_test`, `pub` under the gate + +Every item in `test_support.rs` is `pub` under the module's `#![cfg(any(test, feature = "test-utils"))]` (Global Constraints), except one that names a crate-private type, which stays `pub(crate)` (Task 14). + +Add `positive_record(&PartnerIdChange) -> Option<&KvPartnerId>` to the test module. It matches `FillMissing` and `OwnerWrite`, and returns `None` for the two removals. - [ ] **Step 2: Run and see them fail** @@ -6014,18 +8111,7 @@ Expected: compile errors. - [ ] **Step 3: Implement** -Implement `EligibilityView::merge`, `eligibility` and `outcome_to_changes` as plain functions. In `outcome_to_changes`, `Resolved(resolution)` does the following: - -- It drops IDs for sources the provider does not claim (after `sources` narrowing from `ProviderSettings`), IDs with an empty UID or one over `MAX_UID_LENGTH`, and duplicates. It logs each drop with the provider ID and source only. -- An ID of an expiring source without an expiry keeps the stored `exp` when its UID is unchanged and is dropped otherwise. -- `with_refresh_until` from a provider without private state is ignored with a warning. -- Each kept ID becomes `OwnerWrite` with `expected = record_revision(stored)` (or `None` when no record) when this provider owns the source, else `FillMissing { writer: Some(provider) }`. -- `invalidated` becomes `OwnerRemove` with the revision read, plus an `iv` hash of the stored UID, plus `n = now`. -- `opted_out` becomes `WriterRemove` for every source the provider claims (the mutation also removes the unowned records it wrote), plus `opt_out = KvOptOut { at: now, fp }` and `st = Clear`. -- When the provider has private state, the positive IDs plus the state change plus `n` and `f` go into one `ProviderResult`. Otherwise they go into `removals`, a plain change list, and `state`. -- `state_rejected()` drops the positive part and applies the `Malformed` backoff. - -Backoff arithmetic uses `saturating_pow` and `min`, so a large `f` cannot overflow. +Implement `EligibilityView::from_kv`, `eligibility`, `fingerprint_bits` and `outcome_to_changes` as plain functions. Follow the numbered lists above. Give the entry-set steps 1 to 5 their own private helpers (`fn drop_unclaimed`, `fn discarded_sets`, `fn reject_positive_part`, `fn all_sets_discarded`), so a reviewer can match each to its spec bullet. The `iv` check of step 4 hashes each UID of the browser-seeded record with `uid_hash` (Task 13) and looks it up in the provider's `iv`. - [ ] **Step 4: Run and see them pass** @@ -6036,7 +8122,7 @@ Expected: PASS. ```bash git add crates/trusted-server-core/src/ec/identity crates/trusted-server-core/src/migration_guards.rs -git commit --signoff -S -m "Add coordinator eligibility and outcome rules" +git commit --signoff -S -m "Add coordinator eligibility and outcome rules" -m "Eligibility reads the KV row alone. Only a returning writer runs providers; a hold-mode provider moves after send when the navigation runs no auction. Provider results become whole entry sets per source, an invalidation carries no hash until it applies, a discarded set rejects a stateful provider's positive part, and a result whose every set is discarded is Malformed. An opt-out's fingerprint is a plain consent bitmask." ``` ### Task 21: Coordinator dispatch @@ -6051,9 +8137,9 @@ git commit --signoff -S -m "Add coordinator eligibility and outcome rules" - Consumes: `RuntimeServices::http_client().send_async`, `services.backend().ensure(&PlatformBackendSpec { .. })` exactly as `dispatch_pull_sync` uses them (`ec/pull_sync.rs`, the body of `dispatch_pull_sync`), the bounded decoders in `streaming_processor.rs`, and the rate limiter's `exceeded_per_minute` - Produces: - - `pub(crate) struct IdentityDeps<'a> { settings: &'a Settings, catalog: &'a ProviderCatalog, keys: &'a IdentityKeys, services: &'a RuntimeServices, rate_limiter: Option<&'a dyn RateLimiter> }`. This one struct keeps `handle_auction` and `handle_publisher_request` within 7 arguments. + - `pub(crate) struct IdentityDeps<'a> { settings: &'a Settings, catalog: &'a ProviderCatalog, keys: &'a IdentityKeys, random: &'a dyn RandomSource, services: &'a RuntimeServices, rate_limiter: Option<&'a dyn RateLimiter> }`. This one struct keeps `handle_auction` and `handle_publisher_request` within 7 arguments. `keys` holds only the visitor key (Task 13). `random` draws record revisions when a mutation applies. Task 23 builds `IdentityDeps` from `IdentityRuntime`. - `pub(crate) struct Call { provider: IdentityProviderId, kinds: BTreeMap }` - - `pub(crate) fn dispatch(deps: &IdentityDeps<'_>, calls: Vec, signals: &ResolveSignals<'_>, budget: Option) -> Vec<(IdentityProviderId, CallOutcome)>` + - `pub(crate) fn dispatch(deps: &IdentityDeps<'_>, calls: Vec, signals: &ResolveSignals<'_>, budget: Option) -> Vec<(IdentityProviderId, CallOutcome)>`. `budget` is `Some(remaining hold budget)` for a hold dispatch before the auction and `None` after send. - `pub(crate) fn check_request(request: &ProviderRequest, descriptor: &ProviderDescriptor) -> Result<(), Report>`: `https` only, the host in `resolves.hosts()`, no `Cookie` header - `pub(crate) fn decode_body(encoding: Option<&str>, body: Vec, cap: usize) -> Option>`: identity, `gzip` or `br`, the cap applied to the decoded size, `None` when over the cap or for an unknown encoding @@ -6065,14 +8151,15 @@ Dispatch per call: - `Err` with `UndeclaredInput` or any other `Config` is `InputFailure`, logged as `Config` with `current_context()` only. - Any recorded undeclared read is also logged as `Config` and turns the outcome into `InputFailure`. 2. `check_request`. On failure: log `Config`, `InputFailure`. -3. The burst cap: `rate_limiter.exceeded_per_minute(&format!("identity:{provider}:{}", visitor_key(keys, ec_id)), settings.max_calls_per_minute)`. Exceeded means not dispatched (Task 29 adds the 60-second marker backoff). A limiter error is logged and the call is skipped. -4. `ensure` a backend with first-byte timeout `effective_timeout_ms`, or in hold mode `min(timeout, remaining budget)` rounded up to the orchestrator's buckets (`auction/orchestrator.rs`, search for the bucket rounding). Do not follow redirects. Send with `send_async`, keeping at most `effective_max_concurrency` in flight. -5. Collect responses: check `Content-Length` against `max_response_bytes`, buffer, `decode_body`. Over the cap is `TransportFailure`. Build the `ProviderResponse` with the request's call context. Call `response`. +3. Order: calls are dispatched in a stable order. Generic pull providers keep today's fairness rule (spec 10.4, `pull_sync.rs:143-156`): sorted by source domain, then rotated left by `(now / 3600) % len`, so the starting partner changes each hour when `max_concurrency` limits a batch. Other providers follow, by provider ID. +4. The burst cap: `rate_limiter.exceeded_per_minute(&format!("identity:{provider}:{}", visitor_key(keys, ec_id)), settings.max_calls_per_minute)`. Exceeded means not dispatched: the call's outcome is `Skipped` with `n = max(n, now + 60)` (Global Constraints), never a failure. A limiter error is logged and the call is skipped the same way. +5. `ensure` a backend with first-byte timeout `after_send_timeout_ms` after send, or for a hold dispatch `min(hold_timeout_ms, remaining budget)`, rounded up to the orchestrator's buckets (`auction/orchestrator.rs`, search for the bucket rounding). Do not follow redirects. Send with `send_async`, keeping at most `[ec.identity].max_concurrency` in flight. +6. Collect responses: check `Content-Length` against `max_response_bytes`, buffer, `decode_body`. Over the cap is `TransportFailure`. Build the `ProviderResponse` with the request's call context. Call `response`. - `Ok(resolution)` is `Resolved`, or `InvalidState(Some(resolution))` when the input recorded a state decode failure. - `Rejected` with 429 or 5xx is `TransportFailure`. - Other `Rejected`, `Malformed` and `Config` are `InputFailure`. - `InvalidState` is `InvalidState(None)`. -6. A hold-mode call still pending when the budget runs out is dropped and becomes `HoldCutOff`. +7. A hold-mode call still pending when the budget runs out is dropped and becomes `HoldCutOff`. After send there is no budget: a call that misses `after_send_timeout_ms` is an ordinary timeout, `TransportFailure` (spec 4.7 step 7). - [ ] **Step 1: Write the failing tests** @@ -6082,30 +8169,32 @@ Add a `FakeHttpClient` to `test_support.rs`. It implements the platform HTTP cli #[test] fn check_request_enforces_https_host_and_no_cookie() { let descriptor = fixed_provider("p", &["a.example.com"]).descriptor().clone(); - let good = ProviderRequest::get(&"https://api.example.com/x".parse().expect("url")); - check_request(&good, &descriptor).expect("control: allowed"); + let good = ProviderRequest::get(&"https://api.example.com/x".parse().expect("should parse the URL")); + check_request(&good, &descriptor).expect("should allow an https request to a declared host (control)"); for (name, request) in [ - ("http", ProviderRequest::get(&"http://api.example.com/x".parse().expect("url"))), - ("host", ProviderRequest::get(&"https://evil.example.net/x".parse().expect("url"))), - ("cookie", good.clone().try_with_header("cookie", "a=1").expect("header")), + ("http", ProviderRequest::get(&"http://api.example.com/x".parse().expect("should parse the URL"))), + ("host", ProviderRequest::get(&"https://other.example.net/x".parse().expect("should parse the URL"))), + ("cookie", good.clone().try_with_header("cookie", "a=1").expect("should add the header")), ] { - check_request(&request, &descriptor).expect_err(name); + check_request(&request, &descriptor).expect_err(&format!("should reject the {name} case")); } } #[test] fn decode_body_caps_decoded_size() { let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - std::io::Write::write_all(&mut encoder, &vec![b'a'; 10_000]).expect("write"); - let gzipped = encoder.finish().expect("finish"); - assert_eq!(decode_body(Some("gzip"), gzipped.clone(), 10_000).map(|b| b.len()), Some(10_000), "fits"); - assert!(decode_body(Some("gzip"), gzipped, 9_999).is_none(), "over the cap after decoding"); - assert!(decode_body(Some("zstd"), vec![1], 10).is_none(), "unknown encoding"); - assert_eq!(decode_body(None, b"ok".to_vec(), 10).as_deref(), Some(&b"ok"[..]), "identity"); + encoder.write_all(&vec![b'a'; 10_000]).expect("should write to the encoder"); + let gzipped = encoder.finish().expect("should finish the gzip stream"); + assert_eq!(decode_body(Some("gzip"), gzipped.clone(), 10_000).map(|b| b.len()), Some(10_000), "should decode a body that fits"); + assert!(decode_body(Some("gzip"), gzipped, 9_999).is_none(), "should reject a body over the cap after decoding"); + assert!(decode_body(Some("zstd"), vec![1], 10).is_none(), "should reject an unknown encoding"); + assert_eq!(decode_body(None, b"ok".to_vec(), 10).as_deref(), Some(&b"ok"[..]), "should pass an identity body through"); } ``` -Also write `dispatch_maps_status_codes_to_outcomes` (200 is `Resolved`, 503 is `TransportFailure`, 400 is `InputFailure`, a body over the cap is `TransportFailure`) and `undeclared_read_is_an_input_failure_even_when_ignored`, using a provider that reads `user_agent().ok()` without declaring it, against the fake client. +The test module imports `std::io::Write as _` at its top for `write_all`. + +Also write `dispatch_maps_status_codes_to_outcomes` (200 is `Resolved`, 503 is `TransportFailure`, 400 is `InputFailure`, a body over the cap is `TransportFailure`), `after_send_timeout_is_a_transport_failure_not_a_cut_off`, `burst_cap_skips_without_counting_a_failure` and `undeclared_read_is_an_input_failure_even_when_ignored`, using a provider that reads `user_agent().ok()` without declaring it, against the fake client. Port today's rotation test as `generic_pull_dispatch_order_rotates_hourly`: three generic pull providers, dispatch at `now` and at `now + 3600` with `max_concurrency = 1`, and a different first host each hour, in the order `pull_sync.rs` gives today. - [ ] **Step 2: Run and see them fail** @@ -6134,21 +8223,30 @@ git commit --signoff -S -m "Add coordinator dispatch with request checks and byt - Create: `crates/trusted-server-core/src/ec/identity/generic_pull.rs` - Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` (`ProviderHarness` basics, `WriteCountingEcKv`) -- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (`ProviderCatalog::from_settings`) +- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (`ProviderCatalog::from_settings`, `validate_identity_catalog`) +- Modify: `crates/trusted-server-core/src/ec/pull_sync.rs` (`validated_pull_sync_url` takes the URL fields instead of a `PartnerConfig`, and becomes `pub(crate)` with `build_pull_request_url`) +- Modify: `crates/trusted-server-core/src/settings.rs` (`EcPartner.pull_sync_ttl_sec` and `pull_sync_rate_limit` removed, with catch fields), and every `EcPartner` literal that sets them (`ec/finalize.rs:628`, `auction/endpoints.rs:617`, `publisher.rs:7905`, `settings.rs:4047` and `:5859`, and the `PartnerConfig` fixtures in `ec/pull_sync.rs` tests) - Modify: `crates/trusted-server-core/src/ec/kv.rs` (`KvIdentityGraph::write_counting` constructor, test support) -- Test: `generic_pull.rs` +- Modify: `crates/trusted-server-core/src/config.rs` (call `validate_identity_catalog` from both validators; extend `validate_removed_keys`) +- Modify: `crates/trusted-server-core/src/ec/registry.rs` (`PartnerConfig`, if it copies the two removed fields) +- Modify: `trusted-server.example.toml`, the integration fixture and `docs/guide/configuration.md`, wherever they set the two removed partner keys +- Test: `generic_pull.rs`, `catalog.rs` **Interfaces:** -- Consumes: `PartnerConfig` (`ec/registry.rs:22`), `validated_pull_sync_url` and `build_pull_request_url` (`ec/pull_sync.rs`, made `pub(crate)`) +- Consumes: `EcPartner` (`settings.rs:385`, the settings type: `settings.ec.partners` is `Vec`, `settings.rs:572`), `build_pull_request_url` (`ec/pull_sync.rs:368`), `ResolvedEid` and `ResolvedUid` (Task 10) - Produces: - - `pub(crate) struct GenericPullProvider` with `pub(crate) fn from_partner(partner: &PartnerConfig) -> Result>` - - `pub(crate) fn generic_pull_id(source_domain: &str) -> IdentityProviderId`: `pull-` plus the domain with `.` replaced by `-`. Over 63 characters, it is cut to 54 characters, followed by `-` and the first 8 hex digits of SHA-256 of the full domain. - - `ProviderCatalog::from_settings(settings: &Settings, registry: &PartnerRegistry) -> Result>`. In Part 1 it holds generic pull providers only, with no owners, and rejects a duplicate provider ID and more than 32 providers. - - `pub(crate) fn legacy_rate_limit_per_minute(pull_sync_rate_limit: u32) -> u32` = `max(1, ceil(limit / 60))` - - Test support: - - `WriteCountingEcKv` (an in-memory store that counts `Add`, `IfGenerationMatch` and `Overwrite` writes and lookups) with `pub(crate) fn writes(&self) -> WriteCounts` - - `KvIdentityGraph::write_counting(name) -> (KvIdentityGraph, Arc)` + - `pub(crate) fn validated_pull_sync_url(pull_sync_url: Option<&str>, allowed_domains: &[String]) -> Option`. Today's caller passes a `PartnerConfig`'s two fields. The rules do not change. + - `pub(crate) struct GenericPullProvider` with `pub(crate) fn from_partner(partner: &EcPartner) -> Result>`. The descriptor reads only `source_domain`, `pull_sync_url` and `pull_sync_allowed_domains`, which are not secret, so it builds from unresolved deploy-time settings too. The provider keeps `ts_pull_token` and reads it only in `request`. At runtime the settings are resolved, so it holds the token there. + - `pub(crate) fn generic_pull_id(source_domain: &str) -> IdentityProviderId`: `pull-` plus the domain with `.` replaced by `-`. A domain that already contains `-` gets `-` and the first 8 hex digits of SHA-256 of the full domain appended, so `a-b.example.com` and `a.b.example.com` (both valid under `normalize_partner_source_domain`) get different IDs. Over 63 characters, the readable part is cut to 54 characters before the `-` and the hash. + - `pub(crate) fn validate_identity_catalog(descriptors: &[ProviderDescriptor], partners: &[EcPartner], identity: &IdentitySettings) -> Result<(), Report>`: the one validator for the catalog rules of spec 10.2. In this task it rejects a duplicate provider ID, more than 32 providers, an `[ec.identity.providers.]` table naming an unknown provider, and `revoke_scope` set for a provider whose descriptor declares no `captures`. No generic pull provider declares `captures`, so in Part 1 `revoke_scope` on any table is an error. Task 33 adds its rules to this same function. Nothing else repeats them. + - `validate_removed_keys` (Task 19) also rejects `pull_sync_ttl_sec` and `pull_sync_rate_limit` on any `[[ec.partners]]` entry, with an error that names the partner and the replacement: `refresh_after_sec`, and `max_calls_per_minute = ceil(N / 60)`, in `[ec.identity.providers.]` (spec 10.2, 10.4). `EcPartner` keeps a catch field for each, as Task 19 does for `pull_sync_concurrency`. + - `pub(crate) fn generic_pull_descriptors(partners: &[EcPartner]) -> Result, Report>`, for the pull-enabled partners + - `ProviderCatalog::from_settings(settings: &Settings) -> Result>`. It reads `settings.ec.partners` directly and never builds a `PartnerRegistry`, so no secret is checked. In Part 1 it holds generic pull providers only, with no owners, and it calls `validate_identity_catalog` on their descriptors before building. Task 32 adds the integration registry's providers. + - Test support, all `pub` in the `pub mod test_support` module behind `#[cfg(any(test, feature = "test-utils"))]`, so Task 30's adapter tests can reach them (the adapter's dev-dependency already enables `trusted-server-core/test-utils`, `adapter-fastly/Cargo.toml:45`): + - `WriteCountingEcKv` (an in-memory store that counts `Add`, `IfGenerationMatch` and `Overwrite` writes and lookups) with `pub fn writes(&self) -> WriteCounts` + - `KvIdentityGraph::write_counting(name) -> (KvIdentityGraph, Arc)`, `pub` under the same gate + - `FakeHttpClient` (Task 21), made `pub` here if Task 21 left it crate-private - `ProviderHarness` with `new`, the `with_*` setters, `request`, `request_json`, `respond`, `respond_raw` and `resolve`. Conformance checks come in Task 39. Generic pull's descriptor: @@ -6166,7 +8264,15 @@ ProviderDescriptor::builder(id) .build() ``` -Its `request` is a `GET` of `validated_pull_sync_url(partner)` with `?ec_id=` (via `build_pull_request_url`) and `with_bearer(ts_pull_token)`. Skip (`Ok(None)`) when there is no valid URL or token. Its `response` is `success_json::<{ uid: String }>()`. An empty UID or one over `MAX_UID_LENGTH` (`is_valid_eid_uid`) is `no_match`, and any other UID is `found([ResolvedId::new(source, uid)])`. The provider's settings default to `mode = AfterSend`, `max_calls_per_minute = legacy_rate_limit_per_minute(partner.pull_sync_rate_limit)`, and `refresh_after_sec = partner.pull_sync_ttl_sec` (used only when Part 2's `owner` makes it authoritative). A `[ec.identity.providers.]` table overrides these. Expose `fn default_settings_for(partner) -> ProviderSettings` and make `IdentitySettings::provider` fall back to it for `pull-` IDs. To do that, `provider()` takes an `Option` base. +Its `request` is a `GET` of `validated_pull_sync_url(partner.pull_sync_url.as_deref(), &partner.pull_sync_allowed_domains)` with `?ec_id=` (via `build_pull_request_url`) and `with_bearer(ts_pull_token)`. Skip (`Ok(None)`) when there is no valid URL or token. Its `response` keeps today's no-op answers (`pull_sync.rs:55-57`, `:447-453`): + +- A 404 is `Resolution::no_match()`, checked before `success_json`, as today's "treating as no-op". +- The body is `success_json::<{ uid: Option }>()`. `None`, an empty UID or one over `MAX_UID_LENGTH` (`is_valid_eid_uid`) is `no_match`, never `Malformed`, so a partner's "no user" answer waits the `no_match` 24 h instead of counting a failure. +- Any other UID is `found([ResolvedEid::new(&source, ResolvedUid::new(uid))])`: one entry, one UID, no `atype`, no `ext` and no EID-level fields, so the coordinator stores the one-entry record pull sync stores today (G13). + +A generic pull provider's settings are `settings.ec.identity.provider(&generic_pull_id(&partner.source_domain))` like any other provider's: `ProviderSettings::default()` unless the operator writes an `[ec.identity.providers.pull-]` table. Pacing moves there (spec 10.4): an operator who had `pull_sync_rate_limit = N` per hour sets `max_calls_per_minute = ceil(N / 60)`, and one who had `pull_sync_ttl_sec` sets `refresh_after_sec`. The changelog (Task 31) gives both conversions. + +Until Task 29 deletes `dispatch_pull_sync`, `ec/pull_sync.rs` reads the same table: its rate check becomes `rate_limiter.exceeded_per_minute(&rate_key, provider_settings.max_calls_per_minute)` (`pull_sync.rs:200`), and its marker TTL uses `provider_settings.refresh_after_sec`. Its tests set those through `settings.ec.identity.providers` instead of the removed partner fields. - [ ] **Step 1: Port today's pull sync tests** @@ -6175,62 +8281,93 @@ List them with `grep -n "fn " crates/trusted-server-core/src/ec/pull_sync.rs | s ```rust #[test] fn calls_the_partner_url_with_the_ec_id_and_bearer() { - let partner = partner_config("ssp.example.com", "https://sync.ssp.example.com/pull", "example-token-value-0123456789abcdef"); + let partner = ec_partner("ssp.example.com", "https://sync.ssp.example.com/pull", "example-token-value-0123456789abcdef"); let provider = GenericPullProvider::from_partner(&partner).expect("should build"); let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); let request = harness.request().expect("should build").expect("should not skip"); - assert_eq!(request.url().host_str(), Some("sync.ssp.example.com"), "host"); + assert_eq!(request.url().host_str(), Some("sync.ssp.example.com"), "should call the partner host"); assert_eq!( request.url().query_pairs().find(|(name, _)| name == "ec_id").map(|(_, value)| value.into_owned()), Some(TEST_EC_ID.to_owned()), "should pass the EC ID" ); - assert_eq!(request.headers()["authorization"], "Bearer example-token-value-0123456789abcdef", "bearer"); + assert_eq!(request.headers()["authorization"], "Bearer example-token-value-0123456789abcdef", "should send the bearer token"); + + let resolution = harness.respond(StatusCode::OK, &json!({ "uid": "partner-uid" })).expect("should parse the answer"); + assert_eq!(resolution.eids().len(), 1, "should return one entry"); + let eid = &resolution.eids()[0]; + assert_eq!(eid.uids().len(), 1, "should return one UID"); + assert_eq!(eid.uids()[0].id(), "partner-uid", "should return the partner's UID"); + assert_eq!( + (eid.uids()[0].atype(), eid.uids()[0].ext(), eid.inserter(), eid.matcher(), eid.mm(), eid.ext()), + (None, None, None, None, None, None), + "should set nothing beyond the UID, so it is stored as one entry with one UID" + ); + } - let resolution = harness.respond(StatusCode::OK, &json!({ "uid": "partner-uid" })).expect("should parse"); - assert_eq!(resolution.ids()[0].uid(), "partner-uid", "uid"); + #[test] + fn not_found_and_null_uid_are_no_match_as_today() { + let partner = ec_partner("ssp.example.com", "https://sync.ssp.example.com/pull", "example-token-value-0123456789abcdef"); + let provider = GenericPullProvider::from_partner(&partner).expect("should build"); + for (name, status, body) in [ + ("404", StatusCode::NOT_FOUND, json!({})), + ("null uid", StatusCode::OK, json!({ "uid": null })), + ("empty uid", StatusCode::OK, json!({ "uid": "" })), + ] { + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + harness.request().expect("should build").expect("should not skip"); + let resolution = harness.respond(status, &body).expect("should not be an error"); + assert!(resolution.is_no_match(), "should be no_match for {name}"); + } } #[test] fn provider_ids_are_derived_and_bounded() { - assert_eq!(generic_pull_id("ssp.example.com").as_str(), "pull-ssp-example-com", "simple"); + assert_eq!(generic_pull_id("ssp.example.com").as_str(), "pull-ssp-example-com", "should keep a dot-only domain readable"); let long = format!("{}.example.com", "a".repeat(70)); let id = generic_pull_id(&long); - assert!(id.as_str().len() <= 63, "bounded"); - assert_ne!(id, generic_pull_id(&format!("{}.example.com", "a".repeat(71))), "unique by hash"); + assert!(id.as_str().len() <= 63, "should bound the ID to 63 characters"); + assert_ne!(id, generic_pull_id(&format!("{}.example.com", "a".repeat(71))), "should keep long IDs unique by hash"); + assert_ne!( + generic_pull_id("a-b.example.com"), + generic_pull_id("a.b.example.com"), + "should not collide when a domain already contains a hyphen" + ); } - #[test] - fn legacy_hourly_limit_maps_to_per_minute() { - assert_eq!(legacy_rate_limit_per_minute(0), 1, "at least one"); - assert_eq!(legacy_rate_limit_per_minute(59), 1, "rounds up"); - assert_eq!(legacy_rate_limit_per_minute(65), 2, "65/hr is 2/min, as the limiter does today"); - } ``` `ProviderHarness::with_ec_id` is crate-private (only generic pull reads the EC ID). Add it next to the public setters with `pub(crate)`. +`ec_partner(source, url, token)` builds an `EcPartner` with pull sync enabled and `url`'s host allowed. Add it to `test_support.rs`. + +Also add `generic_pull_pacing_comes_from_its_provider_table`: a partner with no table gets `refresh_after_sec = 86_400` and `max_calls_per_minute = 1`, and one with `[ec.identity.providers.pull-ssp-example-com]` holding `max_calls_per_minute = 2` gets 2. In `config.rs` tests, add `removed_partner_pull_keys_are_rejected_with_their_replacements`: a partner with `pull_sync_ttl_sec` fails both validators with an error naming `refresh_after_sec`, one with `pull_sync_rate_limit` with an error naming `max_calls_per_minute`, and the control without either passes. + +In `catalog.rs`, add `validate_identity_catalog_rejects_unknown_provider_tables`, `validate_identity_catalog_rejects_revoke_scope_without_captures`, `validate_identity_catalog_rejects_duplicate_ids_and_more_than_32_providers` and `from_settings_builds_from_unresolved_settings`. Each rejection pairs a rejected config with a control that differs only in the key under test: a table for the partner's real `pull-` ID passes, and the same table with `revoke_scope = "provider"` does not. The last one builds the catalog from settings whose `api_token` holds a short secret-store key name, which `PartnerRegistry::from_config` would reject (`ec/registry.rs:279`). + - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- ec::identity::generic_pull` +Run: `cargo test-fastly -- ec::identity::generic_pull` and `cargo test-fastly -- ec::identity::catalog` Expected: compile errors. - [ ] **Step 3: Implement** Write `generic_pull.rs`, `ProviderCatalog::from_settings`, the harness basics and the counting store. The harness builds `ResolveSignals` from its setters (`SeededRandom` from `with_random_seed`, default seed `1`, default `now` 1,760,000,000). It computes kinds as spec 13 says: a source with a `with_stored` record is `Refresh`, any other is `Initial`, and sources whose kind's needs are missing are dropped, with `Ok(None)` and no provider call when none are left. It keeps the built request's call context for `respond`. `respond` builds a `ProviderResponse` from `serde_json::to_vec(body)`. +Call `validate_identity_catalog(&generic_pull_descriptors(&settings.ec.partners)?, &settings.ec.partners, &settings.ec.identity)` from `validate_settings_for_deploy` and `validate_settings_for_runtime`, after the partner check. Deploy validation calls only `PartnerRegistry::validate_config_for_deploy` today (`config.rs:259`) and must never build a `PartnerRegistry`: `from_config` checks API token lengths, and at deploy time `api_token` holds a key name. Building generic pull descriptors reads no secret (spec 10.2), so this works on unresolved settings. + - [ ] **Step 4: Run and see them pass** -Run: `cargo test-fastly -- ec::identity` and `cargo test-fastly -- ec::pull_sync` +Run: `cargo test-fastly -- ec::identity`, `cargo test-fastly -- ec::pull_sync`, `cargo test-fastly -- config::`, `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity` and `./scripts/test-cli.sh` Expected: PASS. - [ ] **Step 5: Commit** ```bash git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Model pull sync partners as GenericPullProvider" -m "Adds the provider harness basics and a KV store that counts writes by mode." +git commit --signoff -S -m "Model pull sync partners as GenericPullProvider" -m "Adds the provider harness basics and a KV store that counts writes by mode. Generic pull results stay one UID per source, a 404 or null UID stays a no-op, and one catalog validator rejects provider tables it cannot apply." -m "pull_sync_ttl_sec and pull_sync_rate_limit are removed from [[ec.partners]]: pacing comes from each pull partner's [ec.identity.providers.pull-] table, and a config that sets either key fails validation with an error naming its replacement." ``` --- @@ -6239,33 +8376,61 @@ git commit --signoff -S -m "Model pull sync partners as GenericPullProvider" -m **Files:** +- Create: `crates/trusted-server-core/src/ec/identity/runtime.rs` (`IdentityRuntime`) +- Create: `crates/trusted-server-core/src/ec/identity/request.rs` (`RequestIdentity`) - Modify: `crates/trusted-server-core/src/http_util.rs` (add `is_document_navigation`, move `is_prefetch_request` here) -- Modify: `crates/trusted-server-core/src/publisher.rs` (`is_prefetch_request` at about line 3200 becomes a re-export or its callers switch) +- Modify: `crates/trusted-server-core/src/publisher.rs` (`is_prefetch_request` at about line 3200 moves out; the navigation and page-bids merge build their `ServeContext` from the runtime) +- Modify: `crates/trusted-server-core/src/integrations/gpt_diagnostics.rs` (its call at line 274 switches to `crate::http_util::is_prefetch_request`) - Modify: `crates/trusted-server-core/src/ec/mod.rs` (`EcContext` fields and accessors) -- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`build_ec_request_state`, the publisher fallback, and the integration-proxy EC generation path) +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs`, `crates/trusted-server-core/src/ec/identify.rs` and `crates/trusted-server-core/src/ec/admin.rs` (the servability callers Task 14 left on `ProviderCatalog::empty()`) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`AppState`, `build_state_from_settings`, `build_ec_request_state`, the publisher fallback, the integration-proxy EC generation path, the admin lookup call, and the `AppState` test literal at about line 1579) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (the settings-reload branch that runs without an `AppState`, about line 526) - Modify: `crates/trusted-server-core/src/integrations/registry.rs` (integration-proxy EC generation, about line 1046, gains the real-browser check) -- Test: `http_util.rs`, `ec/mod.rs`, `adapter-fastly/src/app.rs` +- Test: `http_util.rs`, `ec/mod.rs`, `ec/identity/runtime.rs`, `auction/endpoints.rs`, `adapter-fastly/src/app.rs` **Interfaces:** +- Consumes: `ProviderCatalog::from_settings` (Task 22), `IdentityKeys::from_settings` (Task 13), `IdentityMutation` (Task 15), `ServeContext` (Task 14) - Produces: + - `pub struct IdentityRuntime`, built once per settings load. Its fields are private: the `ProviderCatalog`, the `IdentityKeys` and a clone of `IdentitySettings` (`identity()`). It is the one identity type the adapter names, so the crate-private catalog and keys never cross the crate boundary or appear in a `pub` signature (`private_interfaces` is an error under `-D warnings`). + - `pub fn from_settings(settings: &Settings) -> Result>`: `ProviderCatalog::from_settings(settings)?` narrowed with each provider table's `sources` (`with_narrowing`, Task 14), and `IdentityKeys::from_settings(settings)`. Task 32 adds a second parameter, the integration registry, for registered providers. Keep the body a short sequence of builder calls so that parameter slots in. + - `pub(crate) fn catalog(&self) -> &ProviderCatalog`, `keys(&self) -> &IdentityKeys`, `identity(&self) -> &IdentitySettings` + - `pub(crate) fn deps<'a>(&'a self, settings: &'a Settings, services: &'a RuntimeServices, random: &'a dyn RandomSource, rate_limiter: Option<&'a dyn RateLimiter>) -> IdentityDeps<'a>` (Task 21) + - A hand-written `Debug` that prints the provider count, never key material. + - `EcContext::set_identity_runtime(&mut self, runtime: Arc)` (`pub`), which the adapter calls where it builds `EcRequestState`, and `pub(crate) fn identity_runtime(&self) -> Option<&IdentityRuntime>`. Without `ec.ec_store` there is no KV and no record to serve, so no runtime is needed there. + - `pub(crate) fn opted_out(entry: &KvEntry) -> BTreeSet` (in `request.rs`): the providers with a sticky opt-out in KV `ps.o`, unless a later `oc` clears it (spec 7.1). KV is the only source: no cookie carries an opt-out. - `pub fn is_document_navigation(req: &Request) -> bool`: `Sec-Fetch-Dest: document` only, with no `Accept` fallback - `pub fn is_prefetch_request(req: &Request) -> bool`, moved unchanged and made public - - `pub(crate) struct RequestIdentity` stored on `EcContext`, with accessors: + - `pub(crate) struct RequestIdentity` stored on `EcContext`. It derives `Default` and `Clone` and has a hand-written `Debug` that prints only counts and flags (the number of mutation changes, `writer`, `is_real_browser`, `removal_lost`), because the derived `Debug` of `IdentityMutation` and `KvPartnerId` would print UIDs whenever `EcContext` is debug-printed (spec 9.3). Accessors: - `writer: bool` + - `is_real_browser: bool`, which Task 38 reads for the real-browser gate on capture - `user_agent: Option`, `dnt: bool`, `page_url: Option` (captured in phase 1) - - `s_cookie: SOpen`, `removals: OpenedRemovals` (opened lazily, at most once, Task 16 and 18 types) - - `mutation: IdentityMutation` - - `staged: StagedCookies`: what this response must set or delete. `s_cookie: Option` to set, `expire_s: bool`, `new_r_batch: Option`, `delete_r: Vec`. - - `EcContext::identity(&self) -> &RequestIdentity` and `identity_mut(&mut self) -> &mut RequestIdentity` - - `EcContext::decide_writer(&mut self, req: &Request, organic_route: bool, is_real_browser: bool)`. It sets `writer = organic_route && is_document_navigation(req) && !is_prefetch_request(req) && is_real_browser` and captures the user agent, DNT and the page URL (the request URL without query or fragment). - - `EcContext::open_identity_cookies(&mut self, keys: &IdentityKeys, catalog: &ProviderCatalog, headers: &HeaderMap)`: idempotent, and a no-op without an EC ID + - `runtime: Option>` + - `mutation: IdentityMutation`: the writer's one flush (Task 25 fills it, Task 29 writes it) + - `pending_removals: Option`: a removal write that failed before send, to try once more after send (Task 38) + - `removal_lost: u32` with `record_removal_lost(&mut self)`: removals every attempt failed to write (spec 5.2 phase 4, 5.3). Task 31 reports it. + - `capture_unpersisted: BTreeMap<(IdentityProviderId, SourceDomain), u32>` with `record_unpersisted(provider, source)`: positive capture IDs counted, never stored (spec 4.7). Task 38 fills it and Task 31 reports it. + - `pub(crate) fn EcContext::identity(&self) -> &RequestIdentity` and `pub(crate) fn identity_mut(&mut self) -> &mut RequestIdentity`. They are crate-private because `RequestIdentity` is. The adapter never calls them. + - `pub fn EcContext::decide_writer(&mut self, req: &Request, organic_route: bool, is_real_browser: bool)`, called from the adapter. It sets `writer = organic_route && is_document_navigation(req) && !is_prefetch_request(req) && is_real_browser`, records `is_real_browser`, and captures the user agent, DNT and the page URL (the request URL without query or fragment). + - `pub fn EcContext::set_real_browser(&mut self, is_real_browser: bool)`, called from the adapter The writer decision is made in the adapter where routing already decides EC generation (spec 5.1). That is after pre-route filters allow dispatch: - In the publisher fallback branch of `app.rs`, next to `let is_navigation = is_navigation_request(&req);`, call `ec.ec_context.decide_writer(&req, true, ec.is_real_browser)`. -- In the integration-proxy branch, call `decide_writer(&req, true, is_real_browser)` before `handle_proxy`, and make proxy EC generation require `is_real_browser` explicitly, not just a present KV graph (`registry.rs:1048`). `ProxyDispatchInput` gains `is_real_browser: bool` for this. -- Every other route leaves `writer = false`. +- In the integration-proxy branch, call `decide_writer(&req, true, is_real_browser)` before `handle_proxy`, and make proxy EC generation require `is_real_browser` explicitly, not just a present KV graph (`registry.rs:1048`). `ProxyDispatchInput` gains `is_real_browser: bool` for this. Task 38 reuses the same field for the real-browser gate on capture. +- Every other route leaves `writer = false` and still records `is_real_browser` through `EcContext::set_real_browser(bool)`, called where the adapter builds `EcRequestState`, so a stager's capture knows whether it passes the real-browser gate (Task 38). + +The adapter holds the runtime (spec 10.3: built once per settings load): + +- `AppState` gains `identity: Arc`. Add `fn build_identity_runtime(settings: &Settings) -> Result, Report>` in `app.rs`, called by `build_state_from_settings` and by the settings-reload branch in `main.rs` that runs when `app_state` is `None` (`load_settings_from_config_store`, about line 526), so both paths have one. Update the `AppState` test literal (about `app.rs:1579`). +- `build_ec_request_state` calls `ec_context.set_identity_runtime(Arc::clone(&state.identity))`. + +The servability callers get the real catalog from here on (spec 7.1). Task 14 passed `ProviderCatalog::empty()`. From Task 29 on, generic pull results are written with `w = pull-…`, and `is_servable` rule 3 drops any record whose writer is not in the catalog, so an empty catalog would silently stop every pull ID reaching bidders (G13). Replace it at each caller: + +- `/auction` (`auction/endpoints.rs`, where it calls `resolve_partner_ids`), the publisher navigation and page-bids merge (`publisher.rs`) and `/identify` (`ec/identify.rs`, which already takes `ec_context`): `catalog = ec_context.identity_runtime().map(IdentityRuntime::catalog)`, falling back to `ProviderCatalog::empty()` only when there is no runtime, and `opted_out = request::opted_out(&entry)`. +- The admin view (`handle_admin_ec_lookup`, `ec/admin.rs:276`) gains `runtime: Option<&IdentityRuntime>`, and the Fastly adapter passes `Some(&state.identity)`. Its `opted_out` is the same KV `ps.o` set. The other adapters return the "not supported" response before reaching it. + +`resolve_partner_ids` becomes `pub(crate)`. Its only callers are `auction/endpoints.rs` and `ec/admin.rs`, and it now takes the crate-private `ServeContext`. - [ ] **Step 1: Write the failing tests** @@ -6277,19 +8442,19 @@ In `http_util.rs`: let with_dest = Request::builder() .header("sec-fetch-dest", "document") .body(EdgeBody::empty()) - .expect("request"); + .expect("should build the request"); let accept_only = Request::builder() .header(header::ACCEPT, "text/html") .body(EdgeBody::empty()) - .expect("request"); + .expect("should build the request"); let iframe = Request::builder() .header("sec-fetch-dest", "iframe") .body(EdgeBody::empty()) - .expect("request"); - assert!(is_document_navigation(&with_dest), "document counts"); - assert!(!is_document_navigation(&accept_only), "the Accept fallback does not"); - assert!(is_navigation_request(&accept_only), "control: the broad check still accepts it"); - assert!(!is_document_navigation(&iframe), "iframes do not"); + .expect("should build the request"); + assert!(is_document_navigation(&with_dest), "should count a document destination"); + assert!(!is_document_navigation(&accept_only), "should not count the Accept fallback"); + assert!(is_navigation_request(&accept_only), "control: the broad check should still accept it"); + assert!(!is_document_navigation(&iframe), "should not count an iframe"); } ``` @@ -6305,14 +8470,15 @@ In `ec/mod.rs` tests: .header(header::USER_AGENT, "ExampleBrowser/1.0") .header("dnt", "1") .body(EdgeBody::empty()) - .expect("request") + .expect("should build the request") }; let mut context = EcContext::default(); context.decide_writer(&navigation(), true, true); - assert!(context.identity().writer(), "organic real-browser document navigation"); - assert_eq!(context.identity().page_url().map(Url::as_str), Some("https://www.example.com/a"), "no query"); - assert_eq!(context.identity().user_agent(), Some("ExampleBrowser/1.0"), "ua captured"); - assert!(context.identity().dnt(), "dnt captured"); + assert!(context.identity().writer(), "should make an organic real-browser document navigation the writer"); + assert!(context.identity().is_real_browser(), "should record the real browser"); + assert_eq!(context.identity().page_url().map(Url::as_str), Some("https://www.example.com/a"), "should drop the query and fragment"); + assert_eq!(context.identity().user_agent(), Some("ExampleBrowser/1.0"), "should capture the user agent"); + assert!(context.identity().dnt(), "should capture DNT"); for (name, organic, browser, request) in [ ("named route", false, true, navigation()), @@ -6325,30 +8491,39 @@ In `ec/mod.rs` tests: .header("sec-fetch-dest", "document") .header("sec-purpose", "prefetch") .body(EdgeBody::empty()) - .expect("request"), + .expect("should build the request"), ), ] { let mut context = EcContext::default(); context.decide_writer(&request, organic, browser); - assert!(!context.identity().writer(), "not a writer: {name}"); + assert!(!context.identity().writer(), "should not be a writer: {name}"); + assert_eq!(context.identity().is_real_browser(), browser, "should keep the real-browser flag: {name}"); } } ``` Use the prefetch headers `is_prefetch_request` actually checks. Read it before writing the test. +Also write: + +- `request_identity_debug_prints_no_uid` (a `RequestIdentity` whose mutation holds a `FillMissing` and an `OwnerRemove`: `format!("{:?}", identity)` contains no UID) +- `opted_out_reads_ps_o_and_honours_a_later_clear`: a row with `ps.a.o` set, `ps.b.o` set and `ps.b.oc` strictly later, and `ps.c` with only `oc`. Only `a` is opted out. +- in `runtime.rs`, `runtime_debug_prints_no_key_material` and `runtime_narrows_providers_by_their_sources_table` +- in `auction/endpoints.rs`, `generic_pull_record_with_writer_is_served_after_flush`: a row whose record for `ssp.example.com` has `w = pull-ssp-example-com`, and a context whose runtime's catalog holds that generic pull provider. `/auction` sends the UID. The control, with a runtime whose catalog lacks the provider, does not. +- in `ec/admin.rs`, `admin_view_serves_generic_pull_records_with_the_runtime_catalog` + - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- http_util::tests::document_navigation` and `cargo test-fastly -- ec::tests::decide_writer` +Run: `cargo test-fastly -- http_util::tests::document_navigation`, `cargo test-fastly -- ec::tests::decide_writer`, `cargo test-fastly -- ec::identity::request`, `cargo test-fastly -- ec::identity::runtime`, `cargo test-fastly -- generic_pull_record_with_writer_is_served_after_flush` and `cargo test-fastly -- admin_view_serves_generic_pull_records` Expected: compile errors. - [ ] **Step 3: Implement** -Add `RequestIdentity` (in `ec/identity/mod.rs` as `pub(crate) mod request;`, file `ec/identity/request.rs`, registered in `migration_guards.rs`) with `Default`, the accessors above, and setters used by later tasks. Add the field `identity: RequestIdentity` to `EcContext` (it already derives `Default`, so this needs `RequestIdentity: Default + Clone`). `IdentityMutation`, `SCookie`, `OpenedRemovals` and `RBatch` need `Clone`, so add `#[derive(Clone)]` where it is missing. `EcFinalizeState` must stay `Clone + Send + Sync`, and every type here is plain data, so it does. +Add `IdentityRuntime` (`pub mod runtime;` in `ec/identity/mod.rs`, re-exported as `ec::identity::IdentityRuntime`) and `RequestIdentity` (`pub(crate) mod request;`, file `ec/identity/request.rs`), both registered in `migration_guards.rs`. Give `RequestIdentity` `Default`, `Clone`, the redacting `Debug`, the accessors above, and setters used by later tasks. Add the field `identity: RequestIdentity` to `EcContext` (it already derives `Debug, Default, Clone`, `ec/mod.rs:249`, so this needs `RequestIdentity: Debug + Default + Clone`). `IdentityMutation` needs `Clone`, so add `#[derive(Clone)]` to it and its parts where it is missing. `EcFinalizeState` must stay `Clone + Send + Sync`. `IdentityRuntime` is shared through `Arc` and holds `Arc`s, which are `Send + Sync` (Task 11), so it does. -Move `is_prefetch_request` into `http_util.rs` as `pub`. Replace the body in `publisher.rs` with `use crate::http_util::is_prefetch_request;` at the top, and delete the old definition. +Move `is_prefetch_request` into `http_util.rs` as `pub`. In `publisher.rs`, delete the old definition and import it with `use crate::http_util::is_prefetch_request;` at the top. Switch `gpt_diagnostics.rs:274` from `crate::publisher::is_prefetch_request` to `crate::http_util::is_prefetch_request`; a private `use` in `publisher.rs` would not be visible on that path. -Wire `decide_writer` in `app.rs` as described above. Add `is_real_browser` to `ProxyDispatchInput` and set it from `EcRequestState`. In `IntegrationRegistry::handle_proxy`, require it before proxy EC generation. +Wire `decide_writer`, `set_real_browser` and `set_identity_runtime` in `app.rs` as described above, and the runtime into `AppState` and the reload branch. Add `is_real_browser` to `ProxyDispatchInput` and set it from `EcRequestState`. In `IntegrationRegistry::handle_proxy`, require it before proxy EC generation. Switch the servability callers to the runtime's catalog and the request's opt-outs. - [ ] **Step 4: Run and see them pass** @@ -6359,7 +8534,7 @@ Expected: PASS. Behaviour is unchanged so far, apart from integration-proxy EC g ```bash git add -A crates/ -git commit --signoff -S -m "Decide the writer request once routing allows dispatch" -m "Only an organic real-browser top-level document navigation that is not a prefetch is the writer. Integration-proxy EC generation now requires a real browser explicitly." +git commit --signoff -S -m "Decide the writer request once routing allows dispatch" -m "Only an organic real-browser top-level document navigation that is not a prefetch is the writer. Integration-proxy EC generation now requires a real browser explicitly." -m "The adapter builds one IdentityRuntime per settings load, and the auction, publisher, identify and admin readers check servability against its catalog and the row's opt-outs." ``` ### Task 24: `/identify` stops writing; cluster size rides on the writer @@ -6374,10 +8549,11 @@ git commit --signoff -S -m "Decide the writer request once routing allows dispat - Produces: - `KvIdentityGraph::compute_cluster_size(&self, ec_id: &str) -> Result>`: the prefix list and count only, no write - - `/identify` uses a stored `cluster_size` when present and `compute_cluster_size` otherwise, and never writes - - `pub(crate) fn queue_cluster_size(context: &mut EcContext, graph: &KvIdentityGraph, ec_id: &str)`, called by phase 4 (Task 29) when its live row lacks `cluster_size` + - `/identify` uses a stored `cluster_size` when present and `compute_cluster_size` otherwise, and never writes. Phase 4 (Task 29) adds a missing `cluster_size` to its one write with `compute_cluster_size` and `IdentityMutation::set_cluster_size`. -- [ ] **Step 1: Write the failing test** +`/identify`'s response shape does not change (spec 7.1). Task 14 gates the partner's own record with `is_servable`, not `resolve_partner_ids`. Its `uid` is the stored record's `first_uid()`, the first UID of the first entry, and its `eid` is built from that UID with the partner's `openrtb_atype`, exactly as `ec/identify.rs:127-135` builds it today. A multi-entry record's other UIDs and EID-level fields are never returned there. + +- [ ] **Step 1: Write the failing tests** ```rust #[test] @@ -6390,20 +8566,35 @@ git commit --signoff -S -m "Decide the writer request once routing allows dispat assert!(response_body_json(response)["cluster_size"].is_number(), "should report a size"); assert_eq!(store.writes().total(), 0, "should not write"); } + + #[test] + fn identify_returns_the_first_uid_of_a_multi_entry_record_with_the_partner_atype() { + let (graph, _store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_row_with_entry_set(&graph, "partner.example.com", &[&["example-a", "example-b"], &["example-c"]]); + + let body = response_body_json(identify_for_test(&graph, &ec_id)); + + assert_eq!(body["uid"], json!("example-a"), "should return the first UID of the first entry"); + assert_eq!( + body["eid"], + json!({ "source": "partner.example.com", "uids": [{ "id": "example-a", "atype": 3 }] }), + "should return one UID with the partner's openrtb_atype, as for a one-UID record" + ); + } ``` -`seed_row_without_cluster_size`, `identify_for_test` and `response_body_json` are thin wrappers around what the existing identify tests already do. Write them in the test module from that setup. +`seed_row_without_cluster_size`, `identify_for_test` and `response_body_json` are thin wrappers around what the existing identify tests already do. Write them in the test module from that setup. `seed_row_with_entry_set` seeds a row whose record for the source holds a set with each inner slice one entry, stored `atype: 1` on every UID and `inserter: "example.com"` on the first entry, so the test also shows that stored fields do not leak into `/identify`. Register the partner with `openrtb_atype = 3` and `bidstream_enabled = true`. -- [ ] **Step 2: Run and see it fail** +- [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- ec::identify::tests::identify_computes_cluster_size_without_writing` -Expected: FAIL with one write. +Run: `cargo test-fastly -- ec::identify::tests::identify_` +Expected: the cluster test fails with one write. The multi-entry test may already pass, since Task 14 reads `first_uid()`. Keep it: it pins the shape. - [ ] **Step 3: Implement** -Split `evaluate_cluster` into `compute_cluster_size` (list and count) and the existing write. `/identify` calls only `compute_cluster_size`. Keep `evaluate_cluster` for now. Task 29 calls it through the mutation (`IdentityMutation::set_cluster_size`), and once nothing else calls it, remove it in that task. +Split `evaluate_cluster` into `compute_cluster_size` (list and count) and the existing write. `/identify` calls only `compute_cluster_size`. Keep `evaluate_cluster` for now, because other callers may still use it. Task 29 adds the size to the mutation with `compute_cluster_size` and `IdentityMutation::set_cluster_size`, and removes `evaluate_cluster` in its Step 3 once nothing calls it. -- [ ] **Step 4: Run and see it pass** +- [ ] **Step 4: Run and see them pass** Run: `cargo test-fastly -- ec::identify` and `cargo test-fastly -- ec::kv` Expected: PASS. @@ -6412,58 +8603,51 @@ Expected: PASS. ```bash git add -A crates/trusted-server-core/src/ec -git commit --signoff -S -m "Stop writing cluster size from /identify" +git commit --signoff -S -m "Stop writing cluster size from /identify" -m "Pins the response shape for multi-entry records: the first UID with the partner's atype, as for a one-UID record." ``` -### Task 25: Finalize builds the mutation, and stagers save into cookies +### Task 25: Finalize builds the mutation, and stagers write nothing -This is the core behaviour change of Part 1. `ec_finalize_response` stops writing KV for returning visitors. On the writer request it adds changes to the mutation. On stagers it saves changes into `__Host-ts-eids-s` and `__Host-ts-eids-r-*`. +This is the core behaviour change of Part 1. `ec_finalize_response` stops writing KV for returning visitors. On the writer request it adds browser ingest to the mutation that phase 4 (Task 29) flushes. On stagers it ingests nothing and writes nothing. No identity cookie is set on any path (spec 5.2 phase 3, 9.4). **Files:** -- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (`ec_finalize_response`, `finalize_unusable_consent`, `confirm_then_recover_orphaned_ec`, `recover_orphaned_ec`) -- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`generate_if_needed` takes the browser ingest) +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (`ec_finalize_response`, `confirm_then_recover_orphaned_ec`, `recover_orphaned_ec`, and the new `browser_seed`) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`generate_if_needed` takes the browser ingest; its three callers in `ec/mod.rs` tests) - Modify: `crates/trusted-server-core/src/ec/kv.rs` (`create_if_absent` callers pass the seeded entry) -- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (pass the parsed `ts-eids` and sharedId to `generate_if_needed`) +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`ProxyDispatchInput` gains the seed, and `handle_proxy`'s `generate_if_needed` call at about line 1047 passes it) +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` and `crates/trusted-server-core/src/publisher.rs` (their test callers of `ec_finalize_response`, whose expectations change) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (build the seed for `generate_if_needed` and `ProxyDispatchInput`) - Test: `ec/finalize.rs` **Interfaces:** -- Consumes: `RequestIdentity` (Task 23), `IdentityMutation` (Task 15), the cookie functions (Tasks 16 to 18), `ProviderCatalog`, `IdentityKeys` +- Consumes: `RequestIdentity` (Task 23), `IdentityMutation` and `PartnerIdChange::FillMissing` (Task 15), `KvPartnerId::browser` (Task 6), `collect_eid_updates` with `now` (Task 7) - Produces: - - `ec_finalize_response` gains one argument, `identity: &IdentityFinalize<'_>`, which bundles `keys`, `catalog`, `identity_settings`, `now` and `random`, so the function stays within 7 arguments - - Writer, returning EC: adds `FillMissing` browser ingest (`acquired_at = now`) for `ts-eids` and sharedId, plus the flush-checked staged entries from `ts-eids-s` and every `ts-eids-r` batch, to `ec_context.identity_mut().mutation`. It writes nothing. - - Stager, returning EC: stages its changes in `staged.s_cookie` (positive) and `staged.new_r_batch` (removals). - - `pub(crate) fn emit_identity_cookies(context: &mut EcContext, identity: &IdentityFinalize<'_>, response: &mut Response)`, called at the end of every finalize path (the unusable-consent path included). It appends the `Set-Cookie` headers for `staged`, and it is the only place `__Host-ts-eids-*` cookies are set. - - `generate_if_needed(settings, kv, seed: &[PartnerIdUpdate])`: the new-visitor create carries the browser ingest, so finalize no longer writes for a new EC (spec 5.2, phase 1) - - Orphan rotation creates the new row carrying this request's browser ingest plus the removals, opt-outs, `oc` and `iv` from `ts-eids-r`, never positive staged IDs. When the create is `Written`, the response deletes the `ts-eids-r` cookies and sets a fresh `ts-eids-s` from the created row. Rotation empties the mutation and the markers. - -What each path does (spec 5.2, 5.3, 5.6): - -| Path | KV | Cookies | -| --------------------------------------------------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| New EC (writer or not) | create before routing, carrying browser ingest (`generate_if_needed`) | `ts-ec` as today. `ts-eids-s` from the created row. | -| Returning EC, writer | none (phase 4 flushes the mutation) | `ts-eids-s` only under phase 3's rules: something staged, a marker, a merge change, or a missing, failed or stale cookie | -| Returning EC, stager | none | `ts-eids-s` when it saved an entry or its KV read changed the cookie. A new `ts-eids-r` when it produced a removal. | -| Unusable consent, not withdrawn | none | a new `ts-eids-r` when it produced a removal. `ts-eids-s` expired. | -| Consent withdrawn | today's tombstone path, unchanged | `ts-eids-s` expired. Every `ts-eids-r` deleted when the tombstone write succeeded, kept when it failed. | -| Any KV read that found a tombstone | none | `ts-eids-s` expired, every `ts-eids-r` deleted | -| Any KV read that found the row missing (no recovery here) | none | `ts-eids-s` replaced by the minimal `m: 1` cookie. `ts-eids-r` kept. | - -Flush checks for positive staged entries (spec 5.3). Write each as a predicate with its own test: - -- The cookie opened with this EC's ID (guaranteed by `open`). -- `now - sa <= staged_max_age_sec`. -- The writer `w` is in the catalog and still claims the source. For an owned source it also owns it. -- `gate_eid_updates_by_consent` and the writer's `policy_allows` pass under this request's consent. -- The writer has no sticky opt-out, unless the cookie stages a clear `oc` later than that opt-out's `at`. -- Conversion: `OwnerWrite { expected: pv }` when the writer owns the source, else `FillMissing`. A `b: 1` entry becomes `FillMissing { writer: None }`. - -Removals and staged `o`, `oc` and `iv` skip every check except the EC binding (spec 5.6). + - Writer, returning EC, live row: adds one `FillMissing { source, record: KvPartnerId::browser(uid, Some(now)) }` per consent-gated browser-ingest update from `ts-eids` and sharedId to `ec_context.identity_mut().mutation`. It covers owned sources too, which seeds them for their owner (spec 7.2). It writes nothing. + - Stager, returning EC: no ingest and no write. Today's returning-visitor `sync_eid_cookie_updates` call (`ec/finalize.rs:110-112`) is gone for `/auction`, `/_ts/page-bids` and every other stager (spec 5.3). + - `ec_finalize_response` keeps its signature. It sets no `Set-Cookie` beyond today's `ts-ec` and the pull-sync marker that Task 29 removes. + - `pub struct BrowserSeed`, an opaque wrapper over `Vec`, built by `pub fn browser_seed(settings: &Settings, registry: &PartnerRegistry, consent: &ConsentContext, eids_cookie: Option<&str>, sharedid_cookie: Option<&str>, now: u64) -> BrowserSeed` in `ec/finalize.rs`. It calls `collect_eid_updates` and the private `gate_eid_updates_by_consent` (`ec/finalize.rs:268`), so it must live in that file. `BrowserSeed::empty()` is for callers with nothing to seed. `PartnerIdUpdate` (`ec/kv.rs:75`) stays `pub(crate)`. + - `pub fn generate_if_needed(settings, kv, seed: &BrowserSeed)`: the new-visitor create carries the browser ingest, so finalize no longer writes for a new EC (spec 5.2, phase 1). `ProxyDispatchInput` gains `seed: &'a BrowserSeed`, which the adapter builds with the same `browser_seed` call, so an EC minted on an integration-proxy navigation carries its ingest in the same one write. + - Orphan rotation creates the new row carrying this request's browser ingest, as today (`ec/finalize.rs:323`). It is the page load's one write: rotation empties the mutation, and phase 4 skips (spec 5.2 phase 1). If the confirm read finds the row after all, no rotation happens, that read becomes the phase 1 snapshot, and the ingest joins the mutation instead of today's immediate write (`ec/finalize.rs:383-386`). + +Browser ingest keeps writing one entry with one UID per registered source (G13). `collect_eid_updates` already picks one UID per registered source and drops unregistered sources: for a repeated source, the first valid UID of its last entry, with the sharedId cookie applied last (`ec/prebid_eids.rs:146-151`). Nothing in this task changes that rule or widens what is stored. A browser entry's extra UIDs and EID-level fields still reach bidders through the `/auction` body and the merge, as today, and never KV. The ingest reads `ts-eids` and sharedId only: the `/auction` body is no longer an ingest input (spec G10), and Task 26 removes that plumbing. + +What each path does (spec 5.1, 5.2, 5.5): + +| Path | KV | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| New EC (writer or not) | the create before routing, carrying browser ingest (`generate_if_needed`). `ts-ec` set as today. | +| Returning EC, writer, live row | none. Ingest joins the mutation, and phase 4 flushes it. | +| Returning EC, writer, missing row | today's orphan recovery: confirm, then rotate with ingest in the create. An integration-proxy writer never rotates. | +| Returning EC, stager | none | +| Unusable consent, not withdrawn | none | +| Consent withdrawn | today's tombstone path, unchanged here (Task 27 adds the after-send attempt) | +| Any KV read that found a tombstone | none, and the mutation stays empty | - [ ] **Step 1: Write the failing tests** -Add a `mod identity_flow` test module to `finalize.rs`. Build requests and contexts the way the existing finalize tests do (`grep -n "fn finalize_" crates/trusted-server-core/src/ec/finalize.rs | head -40`). Use `KvIdentityGraph::write_counting`. Write these tests. The first two are shown in full: +Add a `mod identity_flow` test module nested inside `finalize.rs`'s existing `mod tests`, so the filter `ec::finalize::tests::identity_flow` matches it. Build requests and contexts the way the existing finalize tests do (`grep -n "fn finalize_" crates/trusted-server-core/src/ec/finalize.rs | head -40`). Use `KvIdentityGraph::write_counting` (Task 22). Write these tests. The first two are shown in full: ```rust #[test] @@ -6471,6 +8655,7 @@ Add a `mod identity_flow` test module to `finalize.rs`. Build requests and conte let (graph, store) = KvIdentityGraph::write_counting("ec-store"); let ec_id = seed_live_row(&graph, KvEntry::minimal("other.example.com", "x", 1)); let mut context = returning_context(&ec_id, Writer::Yes); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); let mut response = html_response(); ec_finalize_response( @@ -6481,56 +8666,71 @@ Add a `mod identity_flow` test module to `finalize.rs`. Build requests and conte None, Some("example-shared-id"), &mut response, - &identity_finalize_for_test(), ); - assert_eq!(store.writes().total(), 0, "finalize must not write for a returning writer"); - assert!(!context.identity().mutation().is_empty(), "should queue the sharedId fill"); - assert!(set_cookie_names(&response).is_empty(), "no Set-Cookie on an unchanged returning HTML response"); + assert_eq!(store.writes().total(), 0, "should not write in finalize for a returning writer"); + let records = queued_records(context.identity().mutation()); + assert_eq!(records.len(), 1, "should queue the sharedId fill"); + assert_eq!(records[0].1.first_uid(), Some("example-shared-id"), "should queue the cookie's UID"); + assert!(set_cookie_names(&response).is_empty(), "should set no cookie on a returning HTML response"); } #[test] - fn auction_stager_saves_the_kv_gap_into_ts_eids_s() { + fn golden_browser_ingest_stays_one_uid_whatever_the_browser_sent() { let (graph, store) = KvIdentityGraph::write_counting("ec-store"); let ec_id = seed_live_row(&graph, KvEntry::minimal("other.example.com", "x", 1)); - let mut context = returning_context(&ec_id, Writer::No); - context.set_eid_sync_source(EidSyncSource::Auction); - context.set_client_eids(vec![eid("gap.example.com", "body-only-uid")]); + let mut context = returning_context(&ec_id, Writer::Yes); context.set_kv_snapshot(graph.load_snapshot(&ec_id)); - let mut response = json_response(); + let eids_cookie = ts_eids_cookie(&json!([ + { + "source": "partner.example.com", + "uids": [ + { "id": "example-a", "atype": 1, "ext": { "stype": "ppuid" } }, + { "id": "example-b", "atype": 3 } + ], + "inserter": "example.com", + "mm": 3 + }, + { "source": "unregistered.example.com", "uids": [{ "id": "example-c", "atype": 1 }] } + ])); + let mut response = html_response(); ec_finalize_response( - &settings_with_partner("gap.example.com"), + &settings_with_partner("partner.example.com"), &mut context, Some(&graph), - ®istry_with_partner("gap.example.com"), - None, + ®istry_with_partner("partner.example.com"), + Some(&eids_cookie), None, &mut response, - &identity_finalize_for_test(), ); - assert_eq!(store.writes().total(), 0, "a stager never writes KV"); - let cookie = open_set_s_cookie(&response, &ec_id).expect("should set ts-eids-s"); - assert_eq!(cookie.ids.len(), 1, "should save one entry"); - assert!(cookie.ids[0].is_browser_gap(), "as a b:1 KV-gap entry"); - assert_eq!(cookie.ids[0].uid(), "body-only-uid", "with the body's UID"); + assert_eq!(store.writes().total(), 0, "should not write in finalize"); + let records = queued_records(context.identity().mutation()); + assert_eq!(records.len(), 1, "should queue only the registered source, as today"); + let (source, record) = &records[0]; + assert_eq!(source.as_str(), "partner.example.com", "should queue the registered source"); + assert_eq!( + serde_json::to_value(record).expect("should serialize the record"), + json!({ "at": record.at, "eids": [{ "uids": [{ "id": "example-a" }] }] }), + "should store the UID collect_eid_updates picks today, with no atype, ext or EID-level field" + ); } ``` The others: -- `stager_never_saves_positive_entries_when_consent_denies_persistence` (it may still set a `ts-eids-r`) -- `unusable_consent_path_still_sets_removal_cookie_and_expires_ts_eids_s` -- `withdrawal_expires_ts_eids_s_and_deletes_removals_only_after_tombstone_lands` -- `tombstone_read_expires_ts_eids_s_and_deletes_removal_cookies` -- `missing_row_on_stager_sets_minimal_m_cookie_and_keeps_removals` -- `orphan_rotation_carries_removals_but_never_positive_staged_ids` (with a `ts-eids-s` holding a `k: 0` provider entry and a `ts-eids-r` removal: the created row has the removal's opt-out and none of the staged IDs) -- `new_visitor_create_carries_browser_ingest_in_one_write` (one `Add` write, and the row holds the sharedId) -- `flush_checks_drop_stale_unowned_disabled_and_opted_out_entries_but_never_removals` (one assertion per check, each paired with a control entry that survives) -- `config_change_discards_positive_entries_and_keeps_markers_and_removals` +- `stagers_ingest_nothing_and_write_nothing`: the same row and cookies on `/auction` (`EidSyncSource::Auction`, with body EIDs), `/_ts/page-bids` and a non-navigation proxy call. Zero writes, an empty mutation, no `Set-Cookie`. This replaces today's tests that assert a returning-visitor ingest write on those routes. +- `unusable_consent_writes_nothing_and_queues_nothing` +- `withdrawal_still_writes_its_tombstone_on_any_request`: the existing withdrawal tests keep passing, on a writer and on a stager. +- `tombstone_read_leaves_the_mutation_empty` +- `orphan_rotation_carries_browser_ingest_in_one_create`: a missing row on a writer navigation. One `Add` write, holding the sharedId as a one-UID record, and an empty mutation afterwards. +- `confirm_read_that_finds_the_row_queues_ingest_instead_of_writing` +- `integration_proxy_writer_never_rotates_on_missing_row` (an integration-proxy writer navigation and a missing row: no create, as today, spec 5.1) +- `new_visitor_create_carries_browser_ingest_in_one_write` (one `Add` write, and the row holds the sharedId as a one-UID record) +- `integration_proxy_new_ec_carries_browser_ingest` (`handle_proxy` minting an EC with a seed: one `Add` write holding the seeded UID) -`identity_finalize_for_test()`, `open_set_s_cookie`, `set_cookie_names`, `seed_live_row`, `returning_context` and the response builders go in the test module. Build them on `ec/identity/test_support.rs`. +`ts_eids_cookie`, `queued_records`, `set_cookie_names`, `seed_live_row`, `returning_context` and the response builders go in the test module. Build them on `ec/identity/test_support.rs`. `queued_records` lists the `(source, record)` of every positive change in the mutation, in order. - [ ] **Step 2: Run and see them fail** @@ -6539,134 +8739,181 @@ Expected: compile errors, then assertion failures (finalize still writes). - [ ] **Step 3: Implement** -Restructure `ec_finalize_response`: +Restructure the returning-EC branch of `ec_finalize_response` (`ec/finalize.rs:80-121`): -1. At the top, call `ec_context.open_identity_cookies(...)` once (it is idempotent and needs the EC ID). -2. Unusable consent: keep today's code, then stage any removals the route produced (Task 38 adds captures, so in Part 1 these come only from `ts-eids-r` carry-over), expire `ts-eids-s`, and call `emit_identity_cookies`. On withdrawal, delete the `ts-eids-r` cookies only when every tombstone write returned success. Check what `apply_withdrawal_tombstones` reports. If it returns nothing, make it return whether every write succeeded. -3. Returning EC with a graph: - - Collect browser ingest as today, then consent-gate it. - - If the request read KV (`kv_snapshot` is not `NotRead`), run `merge_with_kv` on the cookie and handle tombstone, missing and orphan recovery as the table says. Today's `confirm_then_recover_orphaned_ec` stays, with its `updates` argument extended to carry the `ts-eids-r` removals into the created row. - - **Writer:** add ingest, then the flush-checked staged entries, then every `ts-eids-r` batch, to the mutation. Set `staged.s_cookie` only under the phase 3 rules. Record which `ts-eids-r` cookies the phase 1 read shows applied (`r_batch_applied`) or unopenable, in `staged.delete_r`. - - **Stager:** run the KV-gap logic (Task 26 adds the `/auction` part) and save. Never touch `staged.delete_r`. - - Remove the call to `sync_eid_cookie_updates` and the `EidCookieSyncOutcome` measurement for these paths. Keep `record_eid_sync_terminal` for the new-EC create only, or delete it if nothing uses it. -4. New EC: `generate_if_needed` already wrote the seeded row. Finalize sets `ts-ec` as today and stages `ts-eids-s` from the created row. -5. Call `reconcile_pull_sync_marker` as today, then `emit_identity_cookies`. +1. Collect browser ingest from `ts-eids` and sharedId as today, with `now` (Task 7), then consent-gate it. +2. **Writer** (`ec_context.identity().writer()`): when the snapshot is a live row, add each update to the mutation as `FillMissing`. When it is `NotRead` (a failed or skipped phase 1 read), still add them: phase 4 re-reads before it writes (spec 5.2 phase 4 step 2). A missing row keeps today's orphan recovery; pass the updates into `confirm_then_recover_orphaned_ec` so the create carries them, and clear the mutation when the create is `Written`. A confirm read that finds the row sets the snapshot and queues the updates. +3. **Stager:** do nothing. Remove the `sync_eid_cookie_updates` call and the snapshot refresh around it for these paths. The `EidCookieSyncOutcome` measurement line (`EID sync measurement: ...`) goes with it for returning visitors, and Task 31's per-request counters replace it. Keep `record_eid_sync_terminal` for the new-EC create. Name the removed line and its replacement in this commit's body and in Task 31's CHANGELOG entry, because operator dashboards may read it. +4. New EC: `generate_if_needed` already wrote the seeded row. Finalize sets `ts-ec` as today and stops writing ingest for it (`ec/finalize.rs:144`). +5. Call `reconcile_pull_sync_marker` as today. Task 29 removes it. -In `ec/mod.rs`, give `generate_if_needed` a `seed: &[PartnerIdUpdate]` parameter and apply it to the `KvEntry::new(..)` it creates, via `apply_partner_id_updates`, before `create_if_absent`. In `app.rs`, parse `ec.eids_cookie` and `ec.sharedid_cookie` with `collect_eid_updates` and `gate_eid_updates_by_consent` before calling `generate_if_needed`. Both helpers are `pub(crate)` in core, so add a `pub fn browser_seed_updates(settings, registry, consent, eids_cookie, sharedid_cookie, now) -> Vec` wrapper in `ec/finalize.rs` for the adapter. - -`ec_finalize_response` callers: the Fastly adapter's `finalize_with_ec_state` (`main.rs`, about line 597) and `publisher.rs` tests. Build `IdentityFinalize` from `AppState`, which holds the catalog and keys built once per settings load (Task 29 adds them to `AppState`). Until then, construct them in `finalize_with_ec_state` from settings. +In `ec/mod.rs`, give `generate_if_needed` a `seed: &BrowserSeed` parameter and apply its updates to the `KvEntry::new(..)` it creates, via `apply_partner_id_updates`, before `create_if_absent`. In `app.rs`, build the seed with `browser_seed(settings, registry, consent, ec.eids_cookie, ec.sharedid_cookie, now)` before calling `generate_if_needed`, and pass the same seed in `ProxyDispatchInput` for `handle_proxy`'s call (`registry.rs:1047`). `ec/mod.rs`'s own test callers pass `&BrowserSeed::empty()`. - [ ] **Step 4: Run and see them pass** -Run: `cargo test-fastly -- ec::finalize` and then the full `cargo test-fastly` -Expected: PASS. Existing finalize tests that asserted a KV write on a returning navigation, `/auction` or `/_ts/page-bids` must now assert no write plus a queued mutation or a staged cookie. Update each one and list them in the commit body. +Run: `cargo test-fastly -- ec::finalize`, `cargo test-fastly -- golden_browser_ingest` and then the full `cargo test-fastly` +Expected: PASS. Existing finalize, auction and publisher tests that asserted a KV write on a returning navigation, `/auction` or `/_ts/page-bids` must now assert no write, plus a queued mutation on the writer. Update each one and list them in the commit body. - [ ] **Step 5: Commit** ```bash git add -A crates/ -git commit --signoff -S -m "Queue browser ingest in the writer's mutation and stage it on stagers" -m "Returning-visitor finalize no longer writes KV. The writer adds ingest, staged entries and removal batches to its single mutation. Stagers save positive entries in __Host-ts-eids-s and removals in a new __Host-ts-eids-r cookie. A new visitor's create now carries browser ingest, so it stays one write." +git commit --signoff -S -m "Queue browser ingest in the writer's mutation and stop stager writes" -m "Returning-visitor finalize no longer writes KV. The writer navigation adds ingest from ts-eids and sharedId to its single mutation; /auction, /_ts/page-bids and other stagers ingest nothing. A new visitor's create, from the publisher fallback or an integration proxy, and an orphan rotation carry browser ingest, so each stays one write." -m "Browser ingest still writes one UID per registered source. The EID sync measurement line no longer runs for returning visitors; the identity counters of the Part 1 metrics task replace it." ``` -### Task 26: `/auction` saves the KV gap; `/_ts/page-bids` stops ingesting +### Task 26: `/auction` and `/_ts/page-bids` stop ingesting and return the mirror payload **Files:** -- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (after the KV read at about lines 307-311) -- Modify: `crates/trusted-server-core/src/publisher.rs` (the `/_ts/page-bids` ingest at about line 6925) -- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (stager branch) -- Test: `auction/endpoints.rs`, `publisher.rs` +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (the client EID plumbing at about lines 290-302, and the response build) +- Modify: `crates/trusted-server-core/src/openrtb.rs` (`ResponseExt` at line 232 gains `ts_eids_s`) +- Modify: `crates/trusted-server-core/src/publisher.rs` (`handle_page_bids` at about line 6904, and its response body at about line 7232) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`EcContext.client_eids` loses its ingest role) +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` (the injected `window.__tsjs_prebid` config at about line 1485 gains `registeredSources`) +- Create: `crates/trusted-server-js/lib/src/core/eids_mirror.ts` (writes `__Host-ts-eids-s` from a payload) +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/eids.ts` (`fitEidCookie` at line 178 keeps registered sources first) and `prebid/index.ts` (reads the `/auction` payload) +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` (`PageBidsResponse` and its handler read the page-bids payload) +- Test: `auction/endpoints.rs`, `publisher.rs`, `integrations/prebid.rs`, and the Vitest files next to each changed TS file **Interfaces:** -- Produces: `pub(crate) fn kv_gap_entries(body_eids: &[Eid], snapshot: &EcKvSnapshot, registry: &PartnerRegistry, consent: &ConsentContext, invalidated: &BTreeSet, now: u64) -> Vec`. For each registered source in the consent-gated body set that the KV row lacks or holds only expired, it picks the UID with `collect_eid_updates`'s rules and skips a UID whose hash is in the owner's `iv`. +- Consumes: `mirror_payload` and `MirrorRead` (Task 17), the servable records (Task 14) and `to_eids`, `gate_eids_by_consent`, `ServeContext` from the runtime (Task 23) +- Produces: + - `pub(crate) fn mirror_read<'a>(context: &EcContext, serve: &ServeContext<'_>, sources: &'a mut Vec) -> MirrorRead<'a>`: `NotRead` when the request read no KV; `Cleared` when consent denies EID persistence, or the read found a tombstone, `consent.ok = false` or a missing row; else `Servable` over the servable records with `bidstream_enabled = true`, after `gate_eids_by_consent`, one `MirrorSource` per source with its writer's `cookie_priority` (0 for a browser-seeded record) and the earliest `exp` of its set (spec 6.2). The EIDs are the server entries the merge would send, before browser precedence, provenance defaults included once Task 28 adds them. + - `ResponseExt { orchestrator, ts_eids_s: Option }`, with `#[serde(skip_serializing_if = "Option::is_none")]`, holding `MirrorPayload::to_json()`. The page-bids body gains a `ts_eids_s` member under the same rule. + - `/auction`'s consent-denied early return still answers with `ts_eids_s = {"clear": true}`, so a withdrawal clears the mirror (spec 5.6). + - TS: `writeEidsMirror(payload: unknown): void` in `core/eids_mirror.ts`. On `{value, max_age}` it writes `__Host-ts-eids-s=; Path=/; Secure; SameSite=Lax; Max-Age=`, skipping the write when the cookie already holds that value. On `{clear: true}` it expires the cookie. On anything else, absent included, it does nothing. It never reads the cookie for any other purpose and never copies it anywhere (spec 6.2, "No feedback"). + - TS: `fitEidCookie(eids, registeredSources: ReadonlySet)` drops unregistered sources before any registered one, keeping the EID fidelity trim order within each group (spec 5.3). -Rules (spec 5.3): +What stops (spec 5.3, G10): -- Save only when the client set came from the body, not from the `ts-eids` cookie. -- Save nothing when the KV read failed, found the row missing, or found a tombstone. -- `/auction` returns early on consent denial before it reads KV. Then nothing is saved. -- `/_ts/page-bids` no longer ingests `ts-eids` into KV. Its `EidSyncSource::PageBids` still lets finalize run cookie upkeep from its KV read. +- `/auction` no longer ingests its body EIDs or `ts-eids` into KV. The body EIDs still feed the auction merge exactly as today. `ec_context.set_client_eids` stays only if the merge or finalize logging still reads it; otherwise remove it with `client_eids()` and their callers. +- `/_ts/page-bids` no longer ingests `ts-eids` into KV. +- `EidSyncSource::Auction` and `EidSyncSource::PageBids` no longer select an ingest path in finalize. Keep the enum only if Task 31's counters label requests with it. - [ ] **Step 1: Write the failing tests** In `auction/endpoints.rs`, next to the existing `/auction` + KV tests: -- `auction_body_only_source_is_staged_not_written`: a registered source in the body and not in KV. Assert zero writes and a `ts-eids-s` `Set-Cookie` holding it. -- `auction_cookie_sourced_client_set_stages_nothing`: no body EIDs, `ts-eids` cookie present. No `ts-eids-s`. -- `auction_with_missing_row_stages_no_gap_and_sets_m`. -- `auction_skips_source_kv_already_holds_and_fills_expired`. +- `auction_body_eids_are_not_written_to_kv`: a registered source in the body and not in KV. Zero writes, and the body's UID still reaches the bidder request. +- `auction_returns_the_mirror_of_its_servable_set`: a row with one servable record. `ext.ts_eids_s.value` decodes (base64url, no padding) to `{"v":1,"eids":[...]}` holding exactly that record's EID, and `max_age` is 86,400. +- `auction_mirror_max_age_follows_the_earliest_exp` +- `auction_mirror_leaves_out_bidstream_disabled_and_consent_gated_sources` +- `auction_returns_clear_on_consent_denial_tombstone_and_missing_row` (one case each, plus a control with a live row) +- `auction_returns_clear_when_the_servable_set_is_empty` +- `auction_without_a_kv_read_omits_the_mirror` (no `ts_eids_s` key at all in `ext`) In `publisher.rs`: - `page_bids_never_writes_kv`: with a `ts-eids` cookie holding a registered source missing from KV, zero writes. +- `page_bids_returns_the_mirror_payload` and `page_bids_without_an_auction_omits_the_mirror` + +In `integrations/prebid.rs`: + +- `injected_config_lists_registered_sources`: the `window.__tsjs_prebid` config holds `registeredSources` with every `[[ec.partners]]` source domain, sorted, and nothing else from the partner table. + +In Vitest: + +- `eids_mirror.test.ts`: writes the value with the given `Max-Age` and the `__Host-` attributes; skips an unchanged value; expires on `clear`; ignores an absent or malformed payload; never touches `ts-eids`. +- `eids.test.ts`: `keeps_registered_sources_before_unregistered_ones` (a set over 3,072 bytes whose registered source comes last: the cookie keeps it and drops unregistered ones first), and a control showing the order within each group is unchanged. +- In `prebid/index.test.ts` and `gpt/index.test.ts`: a response carrying `ts_eids_s` calls `writeEidsMirror` with it, and one without it does not. - [ ] **Step 2: Run and see them fail** -Run: `cargo test-fastly -- auction::endpoints::tests::auction_` and `cargo test-fastly -- publisher::tests::page_bids_never_writes_kv` -Expected: failures (today `/auction` and page-bids write). +Run: `cargo test-fastly -- auction::endpoints::tests::auction_`, `cargo test-fastly -- publisher::tests::page_bids_`, `cargo test-fastly -- injected_config_lists_registered_sources`, and `cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run` +Expected: failures (today `/auction` and page-bids write, and there is no mirror). - [ ] **Step 3: Implement** -In finalize's stager branch, when `eid_sync_source == Some(EidSyncSource::Auction)` and `client_eids` came from the body (add a `client_eids_from_body: bool` to `EcContext`, set by `handle_auction` where it chooses body or cookie), call `kv_gap_entries` and save each as `SEntry::staged_browser`. In `publisher.rs`, remove the page-bids path that leads to a KV write. With Task 25 in place, finalize no longer writes for a returning EC, so check that page-bids has no other direct write (`grep -n "upsert\|sync_eid\|create" crates/trusted-server-core/src/publisher.rs`). +In `auction/endpoints.rs`, remove the ingest of body and cookie EIDs; keep the merge input. Build the payload with `mirror_read` and `mirror_payload` once the KV read is done, and put it in `ResponseExt`. In `publisher.rs`, remove the page-bids path that leads to a KV write (with Task 25 in place finalize no longer writes for a returning EC, so check that page-bids has no other direct write: `grep -n "upsert\|sync_eid\|create" crates/trusted-server-core/src/publisher.rs`) and add `ts_eids_s` to the body. In `prebid.rs`, add `registeredSources` to the injected config. In TS, add `core/eids_mirror.ts`, call it from the `/auction` response handler in `prebid/index.ts` and the page-bids handler in `gpt/index.ts`, and pass the injected `registeredSources` (an empty set when absent, which keeps today's trim) to `fitEidCookie`. - [ ] **Step 4: Run and see them pass** -Run: `cargo test-fastly -- auction::` and `cargo test-fastly -- publisher::` +Run: `cargo test-fastly -- auction::`, `cargo test-fastly -- publisher::`, `cargo test-fastly -- integrations::prebid`, and in `crates/trusted-server-js/lib`: `mise exec node@24.12.0 -- npx vitest run`, `mise exec node@24.12.0 -- npm run lint` and `mise exec node@24.12.0 -- npm run format` Expected: PASS. - [ ] **Step 5: Commit** ```bash -git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Stage the /auction KV gap and stop /_ts/page-bids KV ingest" +git add -A crates/trusted-server-core/src crates/trusted-server-js/lib/src +git commit --signoff -S -m "Return the ts-eids-s mirror from /auction and stop stager ingest" -m "/auction and /_ts/page-bids no longer write browser EIDs to KV. Both return a ts_eids_s payload built from the servable server set, which the shim writes as the client-only __Host-ts-eids-s cookie, or a clear on denied consent, a tombstone or a missing row." -m "The shim's ts-eids trim keeps registered sources before unregistered ones, so the writer navigation's ingest still sees every registered source /auction used to ingest." ``` -### Task 27: The removal overflow write +### Task 27: The withdrawal tombstone's after-send attempt **Files:** -- Modify: `crates/trusted-server-core/src/ec/kv.rs` (new `apply_removals_now`) -- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (`emit_identity_cookies`) -- Test: `ec/kv.rs`, `ec/finalize.rs` +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (the withdrawal path at about lines 470-505, `apply_withdrawal_tombstones` at about line 527) +- Modify: `crates/trusted-server-core/src/ec/mod.rs` (`EcContext` records the failed IDs) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (the after-send hook at about lines 517 and 536) +- Test: `ec/finalize.rs`, `adapter-fastly/src/main.rs` **Interfaces:** -- Produces: `KvIdentityGraph::apply_removals_now(&self, ec_id: &str, mutation: &IdentityMutation, context: &ApplyContext<'_>) -> RemovalWriteOutcome` with `RemovalWriteOutcome { Written, AlreadyApplied, Failed }`. It reads the row, applies only the batch's removals, opt-outs, clears and `iv` (through `mutation::apply` on a removals-only mutation), and writes with `IfGenerationMatch`. On a generation mismatch it re-reads and retries, up to 5 attempts in total, as the withdrawal path does. Any other error (a per-key rate limit included) stops at once with `Failed`. +- Produces: + - `EcContext::failed_withdrawals(&self) -> &[String]` (`pub(crate)`), the withdrawn EC IDs whose before-send tombstone returned `EcKvSnapshot::Failed` after its five CAS attempts (`ec/kv.rs:1349`). The withdrawal closure records each one where it logs the failure today (`ec/finalize.rs:495`). + - `pub fn retry_failed_withdrawals(context: &mut EcContext, kv: &KvIdentityGraph)`: after send, calls `tombstone_existing_from_snapshot(ec_id, EcKvSnapshot::NotRead)` once for each failed ID, so it re-reads first (spec 5.1). It is one more run of the same path, with its own five CAS attempts and completion marker, never a second retry. A second failure is logged at `warn` with the EC ID through `log_id` only and counted (Task 31). A success updates the context's snapshot when the ID is the active one, as the before-send path does. -`emit_identity_cookies` calls `plan_r_batch`. On `RPlan::Overflow` it calls `apply_removals_now` before the response is sent. `Written` and `AlreadyApplied` set no removal cookie. `Failed` sets `split_r_batch`'s cookies. This is the second and last page-load exception to G4 (spec 6.2). +This is the withdrawal's existing exception to G4, extended by one after-send run (spec 5.1). Nothing else rides on it in Part 1. Task 38's captured global opt-out reuses it. - [ ] **Step 1: Write the failing tests** -- `overflow_batch_is_written_before_send_with_no_cookie`: a batch of 12 removals, a live row. Exactly one `IfGenerationMatch` write and no `ts-eids-r` `Set-Cookie`. -- `overflow_write_failure_falls_back_to_split_cookies`: a failing store. Every removal is present across the new cookies, and no other cookie is rewritten. -- `overflow_retries_only_on_generation_mismatch`: a store that fails the CAS twice and then succeeds gives 3 attempts. A store that returns a non-CAS error gives 1 attempt. -- `four_openable_cookies_force_overflow`. +In `ec/finalize.rs`: -- [ ] **Step 2: Run, see them fail, implement, see them pass** +- `failed_withdrawal_is_recorded_for_after_send`: a store that fails every tombstone write. After finalize, `failed_withdrawals()` holds the active ID. +- `successful_withdrawal_records_nothing` +- `after_send_retry_writes_the_tombstone_once`: a store that fails five times and then succeeds. Finalize records the ID; `retry_failed_withdrawals` writes the tombstone and the completion marker; the row reads as a tombstone. +- `after_send_retry_is_the_last_attempt`: a store that always fails. The retry makes one run of at most five CAS attempts, logs, and leaves `failed_withdrawals()` unchanged for the counters. +- `withdrawal_of_two_ids_retries_only_the_failed_one`: the cookie ID's tombstone fails and the active ID's succeeds. The retry touches only the cookie ID. -Run: `cargo test-fastly -- overflow` -Expected: FAIL, then PASS after implementing. Model the retry loop on `ec/kv.rs`'s withdrawal tombstone loop (`grep -n "fn write_withdrawal_tombstone\|attempt" crates/trusted-server-core/src/ec/kv.rs`). +In the adapter: -- [ ] **Step 3: Commit** +- `after_send_runs_the_withdrawal_retry_before_identity_work`: with a failed withdrawal recorded, the after-send hook calls the retry and runs no provider dispatch, since the row is now a tombstone or still unwritten. + +- [ ] **Step 2: Run and see them fail** + +Run: `cargo test-fastly -- ec::finalize::tests::failed_withdrawal`, `cargo test-fastly -- ec::finalize::tests::after_send_retry` and `cargo test-fastly -- after_send_runs_the_withdrawal_retry` +Expected: compile errors. + +- [ ] **Step 3: Implement** + +Record each `Failed` outcome in the withdrawal closure. Add `retry_failed_withdrawals` next to `apply_withdrawal_tombstones`. In `main.rs`, call it first in the after-send hook, before `run_edgezero_pull_sync_after_send` (which Task 29 replaces with phase 4), whenever the request has a graph and a failed withdrawal. A withdrawal request never runs providers or flushes a mutation, so the order matters only for logs. + +- [ ] **Step 4: Run and see them pass** + +Run: `cargo test-fastly -- ec::finalize` and `cargo test-fastly` +Expected: PASS. + +- [ ] **Step 5: Commit** ```bash -git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Write overflowing removal batches before send" -m "When a removal batch does not fit one cookie, or four removal cookies are already present, it is applied to KV with the withdrawal path's CAS retries. If that write fails it is split across new cookies, never coarsened." +git add -A crates/ +git commit --signoff -S -m "Retry a failed withdrawal tombstone once after send" -m "A tombstone whose five CAS attempts fail before send runs the same path once more after send, by which time the per-key write limit has usually passed. A second failure is logged and counted." ``` -### Task 28: The server set in the auction merge +### Task 28: The server set and provider denials in the auction merge **Files:** - Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (`merge_auction_eids`, about line 505) - Modify: `crates/trusted-server-core/src/publisher.rs` (navigation and page-bids merge, about lines 5547-5568) -- Modify: `crates/trusted-server-core/src/ec/eids.rs` (`to_eids` adds provenance) -- Test: `auction/endpoints.rs`, `publisher.rs` +- Modify: `crates/trusted-server-core/src/ec/eids.rs` (`to_eids` fills provenance defaults) +- Create: `crates/trusted-server-core/src/ec/identity/denials.rs` +- Test: `auction/endpoints.rs`, `publisher.rs`, `ec/eids.rs`, `denials.rs` **Interfaces:** -- Produces: `merge_auction_eids(client: &[Eid], server: &ServerSet, registry: &PartnerRegistry) -> Vec`. `ServerSet` holds, per source, the servable record from the `ts-eids-s` cookie view (staged entries included) overlaid on KV, minus stubs, removed entries (`ts-eids-r`) and expired entries. Build it with `pub(crate) fn server_set(context: &EcContext, serve: &ServeContext<'_>) -> ServerSet`. +- Consumes: `ProviderDescriptor::provenance(source)` and the `with_provenance` default (Task 8), `uid_hash` (Task 13), `ProviderCatalog::claimants`, `is_capture_only` and `sources_of` (Task 14), the servability check, `ResolvedPartnerId { entries, writer, .. }` and `to_eids`, which already emits one `Eid` per stored entry (Task 14), and `opted_out` (Task 23) +- Produces: + - `pub(crate) struct ServerSet<'a>`, built by `pub(crate) fn server_set<'a>(entry: Option<&'a KvEntry>, serve: &ServeContext<'_>) -> ServerSet<'a>`: per source, the servable record of the KV row the request read, with its writer. It never holds anything from `ts-eids-s` (spec 7.1). No KV read, or a row that is not live and consenting, gives an empty set. + - `pub(crate) struct Denials`, built by `pub(crate) fn denials(entry: &KvEntry, catalog: &ProviderCatalog) -> Denials` in `denials.rs`, read from KV only (spec 7.1, "Provider denials apply to browser candidates"): + - per source, the union of the `ps.iv` and `ps.ov` hashes of every provider in `catalog.claimants(source)` + - the set of sources whose browser value is dropped whole: every source the owner of which has a sticky `ps.o`, and every source of a capture-only provider (`is_capture_only`) with a sticky `ps.o`, owned or not (`opted_out`, Task 23, decides "sticky") + - `pub(crate) fn filter_browser_candidates(client: &[Eid], denials: &Denials) -> Vec`: drops each entry of a dropped-whole source; otherwise drops each UID whose `uid_hash` is in that source's hashes, then each entry left with no UID. It runs before the precedence table and before `gate_eids_by_consent`. Without a KV read there are no denials (a stager without a read has no browser candidates to check against, and the writer always reads, Task 29). + - `merge_auction_eids(client: &[Eid], server: &ServerSet<'_>, denials: &Denials, registry: &PartnerRegistry) -> Vec` + - `pub(crate) fn fill_provenance(eids: &mut [Eid], writer: Option<&IdentityProviderId>, source: &SourceDomain, catalog: &ProviderCatalog)`, called on a provider-written record's output + +Part 1's catalog holds only generic pull providers, which claim one source each, own nothing and capture nothing, so in production the denials hold only `iv` and `ov` hashes, and only once a provider writes them. The tests build test catalogs with owners and capture-only providers (`test_support`), so every rule lands and is tested here, and Task 36 adds only the owned precedence row. Precedence (spec 7.1): @@ -6675,88 +8922,125 @@ Precedence (spec 7.1): | Owned | the owner's valid value (`w == owner`, not expired), else the browser's | the owner's valid value, else a browser-seeded value if the source does not expire | | Unowned | the browser's (today) | the server value (today) | -Part 1 has no owners, so only the unowned row applies. Part 2 (Task 36) adds owner denials of browser candidates. A server value uses its stored `atype`, else the partner's, plus its stored UID-level `ext`. A value a provider wrote also gets that provider's EID-level `inserter`, `matcher` and `mm` from `descriptor.provenance(source)`. Sources with `bidstream_enabled = false` are left out. The merged set goes through `gate_eids_by_consent` and into `x-ts-eids` exactly as today. +This task implements the unowned row. Task 36 adds the owned row. -- [ ] **Step 1: Write the failing tests** +How a server value goes out (spec 7.1): + +- The source's whole entry set, one `Eid` per stored entry, in stored order, after the browser entries. Task 14's `to_eids` already does this, with each UID's stored `atype`, else the partner's `openrtb_atype`, and its stored UID-level `ext`, and each entry's stored EID-level fields. A one-UID record is one entry, exactly as today (`ec/eids.rs:64-76`). +- For a record a provider wrote (`w` names a configured provider), the provider's `with_provenance` default for the source (`descriptor.provenance(source)`) applies only to an entry that sets none of `inserter`, `matcher` and `mm` (spec 4.3). Then each field the default sets is filled. An entry that sets any of the three is the vendor's complete statement and is left as stored, so vendor and config provenance never mix in one entry. A browser-seeded or version 1 record (no `w`) gets no default. +- An unowned server set goes out only when the browser did not send the source (after the denials). It is never mixed with browser entries. + +Sources with `bidstream_enabled = false` are left out. The merged set goes through `gate_eids_by_consent` and into `x-ts-eids` exactly as today, with the same 4 KB cap and `x-ts-eids-truncated` when an entry is cut. `encode_eids_header` and `build_eids_header` are unchanged and still truncate per entry (Task 14), so the header can carry part of a stored set; `user.eids` always carries it whole (spec 7.1). + +Browser entries that pass the denials keep today's path untouched: they come first, in the browser's order, with the browser's own EID-level fields, and the merge never edits one beyond dropping a denied UID. + +Task 26's mirror payload reads the same `ServerSet` (before browser precedence), so it carries the provenance defaults too. -- `server_set_includes_staged_ts_eids_s_entry_for_unowned_source_browser_did_not_send` -- `browser_value_wins_for_unowned_source` (control: today's rule) -- `removed_entry_in_ts_eids_r_is_not_sent_before_it_reaches_kv` -- `expired_entry_is_not_sent` -- `stored_atype_and_ext_are_emitted` -- `x_ts_eids_matches_what_was_sent` +- [ ] **Step 1: Write the failing tests** -Compare whole `Vec` values with every field written out, with no `..Default::default()`. +Every test name in this task starts with `merge_`, `denial_` or `golden_merge_`, so the filters below run them all: + +- `merge_browser_value_wins_for_unowned_source` (control: today's rule) +- `merge_server_value_goes_out_when_the_browser_did_not_send_the_source` +- `merge_never_reads_the_mirror_cookie`: the same request with and without a `__Host-ts-eids-s` cookie gives the same `Vec`. +- `merge_expired_entry_is_not_sent` +- `merge_stored_atype_and_ext_are_emitted` +- `merge_x_ts_eids_matches_what_was_sent` +- `merge_entry_set_goes_out_one_eid_per_entry_in_stored_order_after_browser_entries` +- `merge_provenance_applies_only_to_an_entry_with_no_eid_level_fields` (three entries of a provider-written set: one with no EID-level field gets the whole default; one with `inserter` set keeps its `inserter` and gets no `matcher` or `mm`; and a browser-seeded record gets none) +- `merge_header_still_truncates_per_entry` (a stored set whose entries pass 4 KB: `x-ts-eids` holds the entries that fit, `x-ts-eids-truncated` is set, and `user.eids` holds the whole set) +- `denial_drops_a_browser_uid_in_a_claimants_iv_for_an_unowned_source` +- `denial_drops_a_browser_uid_in_a_claimants_ov` +- `denial_filters_uid_by_uid_and_drops_an_emptied_entry`: a browser entry with UIDs A and B, where only A is in `iv`, goes out with B alone; a second entry holding only A is dropped. +- `denial_ignores_hashes_of_a_provider_that_does_not_claim_the_source` (control) +- `denial_drops_every_owned_source_while_the_owner_is_opted_out` +- `denial_drops_every_source_of_an_opted_out_capture_only_provider`: owned and unowned sources alike. Control: after a later `oc`, the browser values go out again. +- `denial_mirror_id_fed_back_through_the_body_is_filtered`: a UID that only the mirror once held, now in `ps.iv`, sent back in the `/auction` body, is not sent. +- `golden_merge_one_uid_row_goes_out_as_on_main` (G13): the merge of an empty client set with a row of one-UID records gives the exact `Vec` today's `merge_auction_eids(None, Some(to_eids(..)))` gives. Write the expected value from `main`'s code, and do not edit it afterwards. +- `golden_merge_browser_entries_are_forwarded_unchanged` (G13): a client set with repeated sources, several UIDs and EID-level fields, no denials and an empty server set: the merge returns the client set as today's `merge_auction_eids` does, minus entries without UIDs. + +Put the `publisher.rs` cases (the navigation and page-bids merge) under the same names with a `merge_publisher_` prefix. Compare whole `Vec` values with every field written out, with no `..Default::default()`. - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- merge` and `cargo test-fastly -- auction::endpoints` -Expected: FAIL, then PASS. +Run: `cargo test-fastly -- merge_`, `cargo test-fastly -- denial_`, `cargo test-fastly -- golden_merge_`, `cargo test-fastly -- ec::eids` and `cargo test-fastly -- auction::endpoints` +Expected: FAIL, then PASS. The existing `merge_auction_eids_*` tests and the PBS and APS goldens (`pbs_request_serialization_goldens_cover_disabled_and_deterministic_enabled_signing` at `integrations/prebid.rs:9981`, and `aps_request_forwards_eid_provenance_fields_and_repeated_sources` at `integrations/aps.rs:2312`, the closest APS test; there is no APS golden) still pass with their expected values unchanged. Port the existing merge tests to the new signature without changing what they assert. - [ ] **Step 3: Commit** ```bash git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Merge the server identity set into auction EIDs" +git commit --signoff -S -m "Merge the server identity set into auction EIDs and apply provider denials" -m "A server value goes out as its whole entry set, one EID per stored entry, from KV only. A provider's provenance default applies only to an entry that sets no EID-level field. Browser candidates lose each UID a claiming provider invalidated or removed on opt-out, and every source of an opted-out owner or capture-only provider. Browser entries and one-UID records otherwise go out exactly as before." ``` ### Task 29: Phase 4: the single flush after send, replacing pull sync **Files:** -- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (`run_edgezero_pull_sync_after_send` and its callers at about lines 482-560 and 618-668) -- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`EcFinalizeState` gains the writer flag and the response status; `AppState` gains `ProviderCatalog` and `IdentityKeys`) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (the post-route part of `edgezero_main` at about lines 506-557 becomes `finalize_send_and_flush`; `apply_edgezero_ec_finalize`, `run_edgezero_pull_sync_after_send` and `prepare_pull_sync_after_send` at about lines 595-666) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`AppState` gains `identity_graph: Option`, used by `build_ec_request_state` (about line 450) and the admin lookup (about line 585) in place of per-request `maybe_identity_graph` calls; `app_state_for_settings` at about line 1462 gains a graph-taking twin for tests) - Create: `crates/trusted-server-core/src/ec/identity/flush.rs` (`run_after_send`) -- Modify: `crates/trusted-server-core/src/ec/pull_sync.rs` (delete `dispatch_pull_sync`, `build_pull_sync_context` and `PullSyncContext`. Keep `entry_is_pull_complete` and the helpers generic pull uses) -- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (writer phase 3: dry runs and pending markers) -- Test: `flush.rs`, `adapter-fastly/src/main.rs` +- Modify: `crates/trusted-server-core/src/ec/pull_sync.rs` (delete `dispatch_pull_sync`, `build_pull_sync_context` and `PullSyncContext`; keep the helpers generic pull uses) +- Delete: `crates/trusted-server-core/src/ec/pull_sync_marker.rs`, with `COOKIE_TS_EC_PULL_COMPLETE` (`constants.rs:5`), the `EcContext` marker field and its accessors (`ec/mod.rs:187-212`, `:283`, `:366`, `:454`, `:581-616`), `validate_pull_sync_marker` and `reconcile_pull_sync_marker` calls (`ec/finalize.rs`, `publisher.rs:4495`), and the marker probe tests (`publisher.rs` at about line 8014) +- Modify: `crates/trusted-server-core/src/publisher.rs` (the preload decision at about lines 4819-4831 drops `marker_valid`) +- Modify: `crates/trusted-server-core/src/ec/kv.rs` and `ec/identify.rs` (delete `evaluate_cluster` once no caller is left, Step 3) +- Test: `flush.rs`, `adapter-fastly/src/main.rs`, `adapter-fastly/src/app.rs`, `publisher.rs` **Interfaces:** -- Produces: `pub fn run_after_send(input: AfterSendInput<'_>) -> FlushOutcome`. `AfterSendInput` holds `settings`, `catalog`, `keys`, `services`, `graph: &KvIdentityGraph`, `context: &EcContext`, `status: StatusCode` and `rate_limiter`. `FlushOutcome` is `Skipped | NoChange | Written | Dropped(&'static str) | WriteFailed`. +- Consumes: `eligibility` with `Phase::AfterSend` and `outcome_to_changes` (Task 20), `dispatch` (Task 21), `mutation::apply` (Task 15), `write_removals` and `log_removal_lost` (Task 18), `compute_cluster_size` (Task 24) +- Produces: + - `pub fn run_after_send(input: AfterSendInput<'_>) -> FlushOutcome`. `pub struct AfterSendInput<'a> { runtime: &'a IdentityRuntime, settings: &'a Settings, services: &'a RuntimeServices, graph: &'a KvIdentityGraph, context: &'a mut EcContext, status: StatusCode, rate_limiter: &'a dyn RateLimiter }`. The catalog and keys come from the runtime (Task 23), so no crate-private type is in the signature (P1). `#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FlushOutcome { Skipped, NoChange, Written, Dropped(&'static str), WriteFailed { removals: RemovalWriteOutcome } }`. + - In the adapter's `main.rs`: `fn finalize_send_and_flush(input: PostRouteInput<'_>, response: HttpResponse, send: &mut dyn FnMut(HttpResponse)) -> Option` with `struct PostRouteInput<'a> { runtime: &'a IdentityRuntime, settings: &'a Settings, ec_state: &'a mut EcFinalizeState, graph: &'a dyn Fn() -> Option, rate_limiter: &'a dyn RateLimiter }`. It takes the runtime and settings rather than `&AppState`, because the reload branch has no `AppState`. It runs finalize (Task 25) with the graph from `graph`, captures the status, hands the finalized response to `send`, runs the withdrawal retry (Task 27), then `run_after_send` with `ec_state.services`, and returns its outcome (`None` when there is no EC state). `edgezero_main` calls it with `send_edgezero_response` as `send`, `&state.identity` and `|| state.identity_graph.clone()` as `graph` (or `maybe_identity_graph(&settings)` on the reload branch, with that branch's runtime from Task 23's `build_identity_runtime`) and `FastlyRateLimiter::new(RATE_COUNTER_NAME)`. `send` is a callback, not a return value, so phase 4 still runs after the response is sent. + - In `main.rs` tests: `fn drive_request_for_test(state: &Arc, request: HttpRequest) -> (HttpResponse, Option)`. It routes the request with the router the existing `app.rs` tests use (`block_on(router.oneshot(request))`), then calls `finalize_send_and_flush` with a `send` that keeps the response, `|| state.identity_graph.clone()` and an allow-all rate limiter. Tests build `state` with `app_state_with_graph_for_test(settings, graph)` in `app.rs`, passing the graph from `KvIdentityGraph::write_counting` (Task 22), so routing, finalize and phase 4 all count against one store. Tasks 30 and 39b drive every request-level test through it. +- `AppState` already holds the runtime (Task 23). This task adds only `identity_graph`. -Phase 3 additions in finalize (writer, returning EC, status 2xx or 304; spec 5.2): +**Phase 1 reads on every returning writer navigation** (spec 5.2 phase 1, 6.3 step 1). The publisher preload (`publisher.rs`, `auction_needs_row` and the preload at about lines 3110-3117 and 4819-4831) reads the row whenever `ec_context.identity().writer()` and the EC is returning, in addition to today's auction rule, and without the pull-complete marker's skip. That one read finds orphans, answers eligibility and filters browser candidates (Task 36), so nothing needs a cookie copy of the row. `/auction` and `/_ts/page-bids` keep their reads unchanged. -- For each provider that is eligible on the merged view in phase `AfterSendDryRun`: - - **Without private state:** build its `ResolveInput` with `dry_run = true` and call `request`. `Some` sets a pending marker `ps..p = now` in `staged.s_cookie`. `None` sets `ps.n = now + refresh_after_sec` in the mutation. - - **With private state:** no dry run. Set the marker when it is due on the cookie view. -- Unowned dry runs return `Some` only when the phase 1 read showed the source missing and the pull-complete marker is absent (spec 5.6). Enforce this in the coordinator before the dry run. The provider does not decide it. -- Section 6.3's phase-1 read triggers must hold. The adapter reads KV in phase 1 (the existing preload) whenever any trigger is true. Add `pub(crate) fn phase1_read_needed(context: &EcContext, catalog, settings, auction_needs_row: bool) -> bool` in `ec/identity/request.rs`, and make the publisher preload decision (`publisher.rs`, `auction_needs_row` and the preload at about lines 3110-3117 and 4819-4831) consult it in addition to today's rule. +Phase 4 (`run_after_send`, spec 5.2 steps 1 to 5): -Phase 4 (`run_after_send`, spec 5.2, steps 1 to 5): +1. If the request is not a returning-EC writer: `Skipped`. If `status` is not 2xx or 304 (spec 5.1), no after-send provider is due, and the mutation is cut down with `IdentityMutation::retain_for_error_status()` (added here): it keeps every removal, opt-out and clear, and the `ps.n` and `ps.f` of every outcome other than `found`, and drops browser ingest and every positive result with its state and scheduling. In Part 1 the mutation holds only browser ingest, so this empties it; Task 37's hold-mode outcomes are what it keeps. +2. Collect the after-send providers: each provider `eligibility(.., Phase::AfterSend)` passes on the phase 1 view (`EligibilityView::from_kv`, Task 20). Without a phase 1 view (a failed read), use an empty view: step 3 re-reads before anything dispatches. If the mutation is empty and none is due: `Skipped`. No read. +3. If phase 1 read the row (`Present` with a generation) and no after-send provider is due, use that snapshot. Otherwise call `graph.get(ec_id)`. A tombstone, `consent.ok = false`, a missing row or a failed read: `Dropped(reason)`, and stop. No provider is called before this check, so no EC ID is disclosed before consent is confirmed (as `ec/pull_sync.rs:100-110` does today). +4. Recompute eligibility steps 4 and 5 for each after-send provider from the live row, rebuild its request from it (the live `ps.st` for a stateful provider), dispatch with `coordinator::dispatch(.., budget: None)`, and turn the outcomes into changes added to the mutation. +5. If the live row has no `cluster_size`, add it with `compute_cluster_size` and `set_cluster_size`. +6. `mutation::apply` against the live entry. If `changed`, write once with `IfGenerationMatch(generation)`: `Written`, or `NoChange` when nothing changed. Never retry the whole flush. +7. On a failed write (a precondition failure, Fastly's per-key rate limit or any other error), log at `warn` with `log_id`. When `mutation.removals()` (Task 15) is not empty, run `write_removals(graph, ec_id, &removals, &context)` (Task 18): a re-read, the removal rules of section 5.4, and up to five attempts. `Failed` there calls `log_removal_lost` and `RequestIdentity::record_removal_lost` (Task 23) once per lost removal. Return `WriteFailed { removals }` with that outcome, or `NothingToWrite` when the mutation held no removal. -1. If the request is not a returning-EC writer, or `status` is not 2xx or 304: `Skipped`. Hold-mode answers already live in the cookie. -2. If the mutation is empty and no marker was set this request: `Skipped`. No read. -3. If phase 1 read the row (`Present` with a generation) and no marker is set, use that snapshot. Otherwise call `graph.get(ec_id)`. A tombstone, `consent.ok = false`, a missing row or a failed read means drop and stop. -4. Recompute eligibility steps 4 to 6 from the live row for every marked provider. Rebuild each request from the live row (the live state for stateful providers). Dispatch with `coordinator::dispatch`. Turn the outcomes into changes and add them to a copy of the mutation. A marked provider not dispatched for a reason the live row does not show (the burst cap, or the breaker in Part 2) gets `n = max(n, now + 60)`. -5. If the live row has no `cluster_size`, compute it with `compute_cluster_size` and `set_cluster_size`. -6. `mutation::apply` against the live entry. If `changed`, write once with `IfGenerationMatch(generation)`. Never retry. Log failure at `warn` with `log_id`, and return `WriteFailed`. +`mutation::apply` writes each source's record whole (Task 15), so a multi-entry record the mutation does not touch keeps its `eids` byte for byte, and a version 1 row upgraded on write serializes as Task 6's golden expects. -Remove `run_edgezero_pull_sync_after_send` and `prepare_pull_sync_after_send`. Their callers call `run_after_send`. The adapter captures the response status before `send_edgezero_response` consumes the response (`main.rs` around lines 510-517) and passes it in. Build the KV graph after send exactly as today (`require_identity_graph`). +Remove `run_edgezero_pull_sync_after_send` and `prepare_pull_sync_after_send`. Their callers call `finalize_send_and_flush`. Build the KV graph after send exactly as today (`require_identity_graph`). - [ ] **Step 1: Write the failing tests** In `flush.rs`: -- `writer_flushes_mutation_once_with_generation_match` (1 `IfGenerationMatch` write) +- `writer_flushes_mutation_once_with_generation_match` (1 `IfGenerationMatch` write, no read after phase 1) - `writer_with_nothing_to_do_reads_and_writes_nothing` (0 reads, 0 writes) -- `non_2xx_or_304_status_drops_the_mutation` +- `non_2xx_or_304_status_drops_ingest_and_dispatches_nothing` (and `retain_for_error_status_keeps_removals_and_backoff` in `mutation.rs`: a mutation with a `FillMissing`, an `OwnerWrite` from a `found`, an `OwnerRemove` and a `no_match` backoff keeps only the last two) - `tombstone_on_reread_drops_mutation_and_calls_no_provider` (the fake HTTP client sees zero calls) -- `marked_generic_pull_provider_dispatches_after_reread_and_its_uid_lands_in_the_same_write` -- `cas_conflict_is_not_retried` (1 attempt, `WriteFailed`) -- `burst_capped_marked_provider_gets_sixty_second_backoff` +- `failed_phase_one_read_rereads_before_writing` +- `due_generic_pull_provider_dispatches_after_reread_and_its_uid_lands_in_the_same_write` (and the record it lands is a one-entry record with `w = pull-…`) +- `hold_provider_on_a_navigation_without_an_auction_runs_in_phase_four` +- `cas_conflict_is_not_retried_as_a_whole` (1 flush attempt, `WriteFailed`) +- `failed_flush_writes_its_removals_alone`: a mutation holding a `FillMissing`, an `OwnerRemove` and an opt-out, and a store that refuses the flush once. One flush attempt, then one removal write holding only the removal and the opt-out. +- `failed_removal_retry_counts_removal_lost`: a store that refuses every write. Five removal attempts, then `removal_lost` counts each removal. +- `burst_capped_provider_gets_sixty_second_backoff_in_the_write` - `missing_cluster_size_is_added_to_the_one_write` +- `untouched_entry_set_is_preserved_by_the_flush` (a live row holding a multi-entry record and a sharedId fill in the mutation: the written row's record equals the live one, field for field) + +In `publisher.rs`: `returning_writer_navigation_reads_kv_without_an_auction` (one read, and a control: a stager navigation without an auction reads nothing), replacing the marker probe tests. -In the adapter (`main.rs` tests): replace `pull_sync_noop_states_skip_post_send_graph_factory` with `after_send_noop_states_skip_post_send_graph_factory`, which checks that no graph is built when there is nothing to flush. +In the adapter (`main.rs` tests): replace `pull_sync_noop_states_skip_post_send_graph_factory` with `after_send_noop_states_skip_post_send_graph_factory`, which checks that no graph is built when there is nothing to flush. Add `finalize_send_and_flush_sends_before_it_flushes`: a `send` callback that records the store's write count when it runs sees 0, and the outcome is `Written` with 1 write. Add `drive_request_for_test_counts_routing_finalize_and_flush_against_one_store`: a returning writer navigation with one change gives one read and one `IfGenerationMatch` write on the injected store. Add `no_response_sets_or_reads_the_pull_complete_cookie`. - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- flush` and `cargo test-fastly` +Run: `cargo test-fastly -- flush`, `cargo test-fastly -- returning_writer_navigation_reads_kv` and `cargo test-fastly` Expected: FAIL, then PASS. -- [ ] **Step 3: Delete pull sync's dispatcher** +- [ ] **Step 3: Delete pull sync's dispatcher and the marker** -Delete `dispatch_pull_sync`, `build_pull_sync_context`, `PullSyncContext`, `InFlightPull` and `pull_rate_limit_key` from `ec/pull_sync.rs`. Port every remaining pull sync test whose behaviour is unchanged to `generic_pull.rs` or `flush.rs` before deleting it. For each behaviour change in spec 10.4, write a test named after it: +Delete `dispatch_pull_sync`, `build_pull_sync_context`, `PullSyncContext`, `InFlightPull`, `pull_rate_limit_key` and `AllowAllRateLimiter` (its tests move with them; `drive_request_for_test` uses core's `test_support` allow-all limiter, or adds one there) from `ec/pull_sync.rs`, and `pull_sync_marker.rs` with every caller listed above. Then delete `KvIdentityGraph::evaluate_cluster` (`ec/kv.rs:1484`) and its two tests: Task 24 moved `/identify` to `compute_cluster_size`, and phase 4 writes the size through the mutation. `grep -rn "evaluate_cluster\|pull_sync_marker\|PULL_COMPLETE" crates/` must come back empty. Port every remaining pull sync test whose behaviour is unchanged to `generic_pull.rs` or `flush.rs` before deleting it. For each behaviour change in spec 10.4, write a test named after it: - `generic_pull_runs_only_on_returning_writer` - `new_visitor_reaches_pull_partners_from_second_page_load` @@ -6773,60 +9057,83 @@ Expected: PASS. ```bash git add -A crates/ -git commit --signoff -S -m "Flush identity changes once after send and run pull partners there" -m "Phase 4 replaces the pull sync after-send hook. It reuses the phase 1 snapshot when no provider is marked, otherwise re-reads the row, dispatches marked providers, and applies the whole mutation in one IfGenerationMatch write that is never retried. Pull partners now run only on a returning visitor's writer navigation, and back off through ps." +git commit --signoff -S -m "Flush identity changes once after send and run pull partners there" -m "Phase 4 replaces the pull sync after-send hook. It reuses the phase 1 snapshot when no after-send provider is due, otherwise re-reads the row, dispatches the due providers, and applies the whole mutation in one IfGenerationMatch write. A failed flush is not retried, but its removals are written alone with the shared CAS loop, and any that still fail are counted as lost." -m "Every returning writer navigation now reads KV in phase 1, so the pull-complete marker cookie is removed. Pull partners run only on a returning visitor's writer navigation and back off through ps. evaluate_cluster is removed." ``` -### Task 30: Prove the one-write invariant, reads and G10 +### Task 30: Prove the one-write invariant, reads, G10 and G13 **Files:** -- Create: `crates/trusted-server-adapter-fastly/tests/identity_writes.rs` (or a `#[cfg(test)] mod identity_writes` in `app.rs` if the adapter has no integration-test harness; check how existing adapter tests drive a full request: `grep -rn "fn .*end_to_end\|TrustedServerApp" crates/trusted-server-adapter-fastly/src | head`) +- Modify (tests only): `crates/trusted-server-adapter-fastly/src/main.rs`, a `#[cfg(test)] mod identity_writes` nested in its `mod tests`. The adapter is a binary crate with no `src/lib.rs`, so a `tests/` file could not import its items; every test here drives a request through Task 29's `drive_request_for_test`. +- Modify (tests only): `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` or `eids.test.ts`, for the `ortb2` half of G10, the registered-first trim and the `ts-eids` golden - Test only **Interfaces:** -- Consumes: `WriteCountingEcKv`, the fake HTTP client, and the adapter's request-driving test helpers +- Consumes: `KvIdentityGraph::write_counting` and `WriteCountingEcKv` (Task 22), the fake HTTP client (Task 21), and Task 29's `drive_request_for_test` and `app_state_with_graph_for_test`. Core's `test_support` items are `pub` behind `test-utils`, which the adapter's dev-dependency on core enables. - [ ] **Step 1: Write the scenario table as tests** -One test per row of spec 13's "One-write invariant" table. Each drives a full request through the adapter (routing, finalize, phase 4) and asserts the exact write counts by mode: - -| Test | Writes | -| ----------------------------------------------------------- | ------------------------------------------------- | -| `new_visitor_navigation_writes_once` | 1 `Add` | -| `new_visitor_collision_remints_and_writes_once` | 1 `Add` that lands (a precondition failure first) | -| `returning_visitor_with_changes_writes_once` | 1 `IfGenerationMatch` | -| `returning_visitor_without_changes_writes_nothing` | 0 | -| `revalidation_304_with_changes_writes_once` | 1 | -| `returning_navigation_answered_3xx_4xx_5xx_writes_nothing` | 0 each | -| `new_visitor_prefetch_writes_once` | 1 `Add` | -| `returning_prefetch_writes_nothing` | 0 | -| `new_visitor_without_sec_fetch_dest_writes_once` | 1 `Add` | -| `returning_iframe_or_no_sec_fetch_dest_writes_nothing` | 0 | -| `integration_proxy_navigation_minting_ec_writes_once` | 1 `Add` | -| `auction_page_bids_identify_proxy_and_assets_write_nothing` | 0 each | -| `consent_withdrawal_writes_tombstone_and_marker` | today's count, per ID | -| `overflowing_removal_batch_writes_before_send` | 1 `IfGenerationMatch` | -| `cas_conflict_on_flush_is_one_attempt` | 1 failed, no retry | +One test per row of spec 13's "One-write invariant" table that Part 1 can exercise. Each drives a full request through `drive_request_for_test` (routing, finalize, phase 4) and asserts the exact write counts by mode. The table's capture rows need Task 38, so they land in Task 39b as `captured_revocation_writes_with_cas`, `global_revoke_from_real_browser_writes_as_withdrawal` and `non_browser_capture_writes_nothing`. Part 1 has no hold mode, so the 3xx, 4xx and 5xx row is 0 here, and Task 39b adds its hold-mode half: + +| Test | Writes | +| ----------------------------------------------------------- | ------------------------------------------------------------- | +| `new_visitor_navigation_writes_once` | 1 `Add` | +| `new_visitor_collision_remints_and_writes_once` | 1 `Add` that lands (a precondition failure first) | +| `returning_visitor_with_changes_writes_once` | 1 `IfGenerationMatch` | +| `returning_visitor_without_changes_writes_nothing` | 0 | +| `revalidation_304_with_changes_writes_once` | 1 | +| `returning_navigation_answered_3xx_4xx_5xx_writes_nothing` | 0 each | +| `new_visitor_prefetch_writes_once` | 1 `Add` | +| `returning_prefetch_writes_nothing` | 0 | +| `new_visitor_without_sec_fetch_dest_writes_once` | 1 `Add` | +| `returning_iframe_or_no_sec_fetch_dest_writes_nothing` | 0 | +| `integration_proxy_navigation_minting_ec_writes_once` | 1 `Add` | +| `auction_page_bids_identify_proxy_and_assets_write_nothing` | 0 each | +| `consent_withdrawal_writes_tombstone_and_marker` | today's count, per ID, plus one after-send run when it failed | +| `cas_conflict_on_flush_is_one_attempt` | 1 failed flush, then its removals alone (up to 5), or none | Also write: -- **Reads:** `returning_visitor_with_recent_cookie_and_nothing_due_reads_zero_times_on_navigation`, and `auction_and_page_bids_keep_one_read`. -- **G10:** `body_only_source_reaches_kv_on_next_navigation` (an `/auction` with a body-only registered source, then a navigation: the source is in KV after the navigation's single write), and `gap_too_large_for_cookie_is_saved_again_by_next_auction`. -- **Failed flush:** `refused_flush_keeps_staged_entries_for_next_navigation`. -- **Orphans:** `deleted_row_is_found_within_kv_revalidate_sec_and_rotated` and `stager_that_sees_missing_row_sets_m_and_next_writer_recovers`. -- **Withdrawal:** `withdrawal_on_auction_after_navigation_send_makes_phase_four_drop_and_call_nothing`. +- **Reads:** `returning_writer_navigation_reads_once`, `due_after_send_provider_adds_one_reread`, and `auction_and_page_bids_keep_one_read`. Each counts reads with the same store. +- **No server identity cookie:** `no_response_sets_an_identity_cookie_other_than_ts_ec`: across every scenario above, no `Set-Cookie` names `ts-eids`, `__Host-ts-eids-s`, any `ts-eids-r` or `ts-ec-pull-complete`. +- **The server never reads the mirror:** `forged_mirror_cookie_changes_nothing`: the same navigation and `/auction`, once without `__Host-ts-eids-s` and once with a forged, stale or malformed one. The bidder requests, the `/identify` answer and every KV write are identical. +- **G10:** `registered_source_beyond_todays_trim_reaches_kv_on_next_navigation` (the shim's `ts-eids` from Task 26's trim, carrying a registered source today's trim would drop: the next navigation's single write holds it) and `ortb2_only_eid_reaches_kv` (below). +- **Failed flush:** `refused_flush_is_rederived_by_the_next_navigation` (a refused write, then a second navigation with the same cookies: the second write holds the ingest) and `refused_flush_writes_its_removals_alone`. +- **Orphans:** `deleted_row_is_found_by_the_next_writer_and_rotated` and `stager_that_sees_missing_row_returns_clear_and_next_writer_recovers`. +- **Withdrawal:** `withdrawal_on_auction_after_navigation_send_makes_phase_four_drop_and_call_nothing`, and `withdrawal_returns_clear_for_the_mirror`. +- **Servability (P2):** `generic_pull_uid_written_after_send_is_served_on_the_next_auction`: a returning writer navigation whose phase 4 lands a generic pull UID (`w = pull-…`), then an `/auction`: `user.ext.eids` carries the UID, and `ext.ts_eids_s` mirrors it. This is Task 23's `generic_pull_record_with_writer_is_served_after_flush` driven end to end. + +**R8: an EID present only in Prebid's `ortb2` config reaches `ts-eids` and KV (spec 13, G10).** Lockr writes its ID5 there. Two halves: + +1. In the shim's Prebid tests, add `ortb2_config_only_eid_reaches_ts_eids_and_the_auction_body` if the existing tests do not already cover it (`grep -n "ortb2" crates/trusted-server-js/lib/test/integrations/prebid/*.ts`). A bidder request whose `ortb2.user.ext.eids` holds an `id5-sync.com` entry that `getUserIdsAsEids()` does not return: `buildRequests` writes it into `ts-eids` and the `/auction` body. Assert the exact cookie value, and copy it into the Rust test as a fixture. +2. `ortb2_only_eid_reaches_kv`: a returning writer navigation carrying that exact `ts-eids` value, with `id5-sync.com` registered and absent from KV. After the one write, KV holds the UID. + +**Complete provider EIDs (G13, spec 13).** Part 1 writes no multi-entry record, but reads, preserves and serves one (spec 14, step 1), so these seed version 2 records directly: + +- `entry_set_round_trips_from_kv_to_bidders`: a row holding a two-entry, three-UID record with every UID-level and EID-level field, for a registered unowned source the browser does not send. One `/auction`: `user.ext.eids` and `x-ts-eids` carry both entries after the browser entries, in stored order, every field intact, and the mirror payload holds the same entries. +- `entry_set_survives_an_unrelated_flush`: the same row, then a navigation that flushes a sharedId fill. The written row's record is unchanged. +- `expired_entry_set_is_never_served_or_mirrored`. +- `version_1_row_is_served_and_upgraded_by_the_next_write`: a version 1 row with `uid` records. `/auction` serves them as today; the next writer navigation with a change writes version 2. + +**Behaviour that predates this spec (G13, spec 13).** Golden tests pin each path G13 must not change. Write each expected value from the code on `main`: run the test body in a worktree of `main` (`git worktree add`) and copy what it prints into the assertion, then never edit it. Where the row shape differs only by the version 2 encoding (Task 6), the assertion compares the decoded records: + +- `golden_browser_ingest_writes_the_first_uid`: a navigation whose `ts-eids` holds a registered source with two UIDs, `inserter`, `mm` and EID-level `ext`, and an unregistered source. The created or flushed row holds the same UID for the same sources as `main`'s row for the same input, apart from the record metadata Part 1 adds (`at`, `rv`). +- `golden_one_uid_row_goes_out_unchanged`: a row of one-UID records, `/auction`, browser sends nothing. `user.ext.eids` and `x-ts-eids` equal `main`'s, byte for byte. +- `golden_auction_body_forwarding_is_unchanged`: a body with repeated sources and EID-level fields and an empty KV row. `user.ext.eids` equals `main`'s. +- The `ts-eids` cookie's content and EID-level `ext` removal belong to the shim, and Part 1 changes only its trim order. Check that the shim's Prebid tests pin both (`grep -n "MAX_EID_COOKIE_BYTES\|ts-eids" crates/trusted-server-js/lib/test/integrations/prebid/*.ts`), and add `ts_eids_cookie_golden` with a pinned value of a set with no unregistered source, which Task 26's trim leaves exactly as today. +- The PBS and APS golden requests named in Task 28 pass unchanged. - [ ] **Step 2: Run them** -Run: `cargo test-fastly -- identity_writes` +Run: `cargo test-fastly -- identity_writes`, `cargo test-fastly -- golden_`, then `cd crates/trusted-server-js/lib && mise exec node@24.12.0 -- npx vitest run` Expected: PASS. A failure here is a real bug in Tasks 23 to 29. Fix it in the task that owns the behaviour, and do not loosen the test. - [ ] **Step 3: Commit** ```bash -git add -A crates/trusted-server-adapter-fastly -git commit --signoff -S -m "Test the one-write-per-page-load invariant end to end" +git add -A crates/trusted-server-adapter-fastly crates/trusted-server-js/lib/test +git commit --signoff -S -m "Test the one-write-per-page-load invariant end to end" -m "Also pins the paths complete provider EIDs must not change, the ortb2-only EID reaching KV, entry sets read and served whole, version 1 rows upgraded on write, and a server that never reads the ts-eids-s mirror." ``` ### Task 31: Metrics, docs and the Part 1 PR @@ -6834,26 +9141,42 @@ git commit --signoff -S -m "Test the one-write-per-page-load invariant end to en **Files:** - Modify: `crates/trusted-server-core/src/ec/finalize.rs` or `ec/identity/request.rs` (metrics) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (emit the line after phase 4) - Modify: `docs/guide/edge-cookies.md`, `docs/guide/configuration.md`, `docs/guide/ec-setup-guide.md`, `docs/guide/api-reference.md`, `CHANGELOG.md`, `trusted-server.example.toml` - [ ] **Step 1: Metrics** -Find how the repo emits request metrics or counters today (`grep -rn "record_eid_sync_terminal\|metric\|counter" crates/trusted-server-core/src/ec/finalize.rs | head`). Emit the same way, with no IDs and no cookie values: +Find how the repo emits request metrics or counters today (`grep -rn "record_eid_sync_terminal\|metric\|counter" crates/trusted-server-core/src/ec/finalize.rs | head`). Today that is a structured `log::info!` line (`EID sync measurement: ...`), which Task 25 removed for returning visitors (the new-EC `record_eid_sync_terminal` stays). These counters replace it. Emit them the same way, as one `log::info!("identity request measurement: ...")` line per request after phase 4, with no IDs and no cookie values: -- the inbound `Cookie` header size in bytes -- the number and total sealed size of `__Host-ts-eids-r-*` cookies per request +- the inbound `Cookie` header size in bytes (spec 14 step 1) - KV reads and writes per request, labelled by route kind (writer, stager, new EC) -- phase 4 outcome (`FlushOutcome` as a label) +- the phase 4 outcome (`FlushOutcome` as a label) +- `removal_lost` (Task 23), and whether a withdrawal tombstone needed its after-send run and whether that run failed (Task 27) -Add one test that asserts the counts are recorded for a writer navigation. +Task 38 adds `capture_unpersisted` to the same line. + +Add `identity_measurement_records_counts_for_a_writer_navigation` (a writer navigation with one change: the measurement holds the `Cookie` header size, one read, one write and `Written`), and a control `identity_measurement_holds_no_identifier` that checks the formatted line contains neither the EC ID nor any UID or cookie value. - [ ] **Step 2: Docs** -- `edge-cookies.md`: add sections for `__Host-ts-eids-s` and `__Host-ts-eids-r-*` (purpose, sealed, host-only, `HttpOnly`, one-year lifetime, 2,048 and 512 byte caps, never read by the shim, never sent to the origin or Prebid Server). Also describe the writer-request rule in plain words: only a full page navigation writes KV, once, after the page is sent. -- `configuration.md`: the `[ec.identity]` table, with every key, its default and its meaning, from spec 10.1. Note that `ec.pull_sync_concurrency` is deprecated. -- `ec-setup-guide.md`: pull partners now run from a visitor's second page load and back off on errors. -- `api-reference.md`: `/_ts/api/v1/identify` and `/_ts/page-bids` no longer write KV, and `/identify` omits expired IDs. -- `CHANGELOG.md` `### Changed`: one entry each for fewer KV writes (with the writer-request rule), `/auction`, `/_ts/page-bids` and `/identify` no longer writing, pull partners from the second page load, and the two new cookies. Mark the rollback note: after this release, do not roll back past it once providers write metadata (spec 14). +- `edge-cookies.md`: add a section for `__Host-ts-eids-s`: a client-only mirror of the server IDs that the shim writes from `/auction` and `/_ts/page-bids` responses for page scripts, with `Path=/; Secure; SameSite=Lax`, not `HttpOnly`, a `Max-Age` of at most one day and never past the earliest ID expiry, and a 2,048-byte cap that leaves out whole sources by `cookie_priority`. The server never reads it, and it is stripped from origin and Prebid Server requests (Tasks 1 to 4). Note that `ts-ec-pull-complete` is gone. Describe the writer-request rule in plain words: only a full page navigation from a returning visitor writes KV, once, after the page is sent. Say that the shim's `ts-eids` trim now keeps registered sources first. +- `configuration.md`: the `[ec.identity]` table, with every key, its default and its meaning, from spec 10.1. Name `revoke_scope` as applying from Part 2. Name the removed keys and their replacements: `ec.pull_sync_concurrency` becomes `[ec.identity].max_concurrency`; `pull_sync_ttl_sec` becomes `refresh_after_sec` and `pull_sync_rate_limit = N` becomes `max_calls_per_minute = ceil(N / 60)`, both in `[ec.identity.providers.pull-]` (Tasks 19 and 22). Validation rejects the old keys. The capture provider checklist lands with capture, in Task 41. +- `ec-setup-guide.md`: pull partners now run from a visitor's second page load, back off on errors, and are paced by their provider table. +- `api-reference.md`: `/_ts/api/v1/identify`, `/auction` and `/_ts/page-bids` no longer write KV, and `/identify` omits expired IDs. `/identify`'s response shape is unchanged: one UID per source, even for a record that holds several. Document `ext.ts_eids_s` in the `/auction` response and `ts_eids_s` in the page-bids body, with the `value`, `clear` and absent cases (spec 6.2). +- `configuration.md`, a "Rolling back" subsection after the `[ec.identity]` table (spec 14). This release moves KV to schema version 2, and a release before it cannot read a version 2 row, so a rollback past it is unsupported once it has written. For a later vendor release rolled back to this one: remove `owner` and every vendor `[ec.identity.providers.*]` table from the config first, because this release rejects an owner or a provider table it does not know; entry sets stay readable but are not served while their writer is not configured; and a write prunes an unconfigured provider's `ps` to `o`, `oc` and `ov`, so opt-outs survive while `ps.iv` and `ps.st` do not. +- `CHANGELOG.md` `### Changed`: one entry each for: + - fewer KV writes, with the writer-request rule + - `/auction`, `/_ts/page-bids` and `/identify` no longer writing + - pull partners from the second page load, paced by their provider table + - the removed config keys and their replacements, and that validation now rejects the old ones + - the `ts-eids-s` mirror payload and the shim's registered-first trim + - the removed `ts-ec-pull-complete` cookie + - KV schema version 2, and that rolling back past this release is unsupported + - the removed `EID sync measurement: ...` log line for returning visitors and its replacement, the `identity request measurement: ...` line, because operator dashboards may read it + + Link the "Rolling back" subsection. + +Update `trusted-server.example.toml` with a commented `[ec.identity]` table and one commented `[ec.identity.providers.pull-ssp-example-com]` table. Run `cd docs && npm run format` and the root markdown prettier command from `AGENTS.md` gate 8. @@ -6861,18 +9184,16 @@ Run `cd docs && npm run format` and the root markdown prettier command from `AGE ```bash git add -A crates/ docs/ CHANGELOG.md trusted-server.example.toml -git commit --signoff -S -m "Document the single identity write and the new identity cookies" +git commit --signoff -S -m "Document the single identity write and the ts-eids-s mirror" ``` - [ ] **Step 4: Part 1 gate and PR** -Run `/check-ci`. Every gate must pass, including `scripts/lint-wasm-clock.sh` (new core code must not call `std::time::{Instant,SystemTime}::now`. Use `checked_current_timestamp()` and `web_time` where you need a clock) and the parity crate. Open the PR against `feat/1246-identity-cookie-strips` with `pr-creator`. Title: "Write identity KV once per page load". The description lists the behaviour changes from spec 14 step 1 and the `test-utils` feature reuse. - ---- +Run `/check-ci`. Every gate must pass, including `scripts/lint-wasm-clock.sh` (new core code must not call `std::time::{Instant,SystemTime}::now`. Use `checked_current_timestamp()` and `web_time` where you need a clock), the parity crate, `./scripts/test-cli.sh`, and the shim's `npm run lint`, which CI's `format-typescript` job runs beside the format check. Open the implementation PR against the spec branch with `pr-creator`. Title: "Write identity KV once per page load". The description lists the behaviour changes from spec 14 step 1, the removed config keys, the `test-utils` feature reuse, and the rollback limit. # Part 2: Providers (spec 14, step 2) -Part 2 lets integrations register providers, adds ownership, hold mode, passive capture, provider state and sticky opt-outs, the circuit breaker, and the consent decoding that strict policies need. No vendor ships here: ID5, LiveRamp, Lockr and UID2 each get their own spec and plan (spec 2.2). This part proves the framework with the fictional `exampleid` and capture providers from spec 4.6, registered only in tests. +Part 2 lets integrations register providers, and adds ownership, hold mode, revoke-only passive capture (the capture removal write, its `capture_unpersisted` and `removal_lost` counters, the real-browser gate, and `revoke_scope` defaulting to `"provider"`), provider state and sticky opt-outs, the circuit breaker, and the consent decoding that strict policies need. It also brings the first full-set writes (G13): Part 1 already reads, preserves, serves and validates complete entry sets, and here registered providers start returning several entries or UIDs per source, or EID-level fields. Every path that predates the spec (browser ingest, `ts-eids`, batch sync, generic pull and the `/identify` response shape) keeps its Part 1 golden tests (the `golden_` tests) unchanged. No vendor ships here: ID5, LiveRamp, Lockr and UID2 each get their own spec and plan (spec 2.2). This part proves the framework with the fictional `exampleid` and capture providers from spec 4.6, registered only in tests. Before you start: `git switch feat/1246-identity-single-write && git switch -c feat/1246-identity-providers`. @@ -6880,62 +9201,73 @@ Before you start: `git switch feat/1246-identity-single-write && git switch -c f **Files:** -- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`IntegrationRegistration`, the builder around line 641, `with_plan` around line 842) +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`IntegrationRegistration`, the builder around line 641, `with_plan` around line 842, and a new `from_registrations`) - Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (`from_settings` adds registered providers) -- Test: `integrations/registry.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/runtime.rs` (`IdentityRuntime::from_settings` takes the integration registry) +- Modify: `crates/trusted-server-core/src/config.rs` (`identity_descriptors_for_deploy`) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`build_identity_runtime` passes the registry `build_state_from_settings` already builds, at about line 226) +- Create: `crates/trusted-server-core/src/ec/identity/examples.rs` (`#[cfg(test)] pub(crate) mod examples;` in `mod.rs`) +- Test: `integrations/registry.rs`, `ec/identity/catalog.rs` **Interfaces:** +- Consumes: `IdentityRuntime` (Task 23), `ProviderCatalog::from_settings` and `validate_identity_catalog` (Task 22) - Produces: - `IntegrationRegistrationBuilder::with_identity_provider(self, provider: Arc) -> Self` - `IntegrationRegistrationBuilder::try_with_identity_provider(self, provider: Result>) -> Result>`. A failure becomes `TrustedServerError::Integration { integration: , message: }`. - `IntegrationRegistry::identity_providers(&self) -> Vec>` - - In `with_plan`: every capture path and prefix must lie under one of the registering integration's `IntegrationProxy::routes()`. Otherwise it is a `Configuration` error naming the integration. - - `ProviderCatalog::from_settings(settings, partner_registry, integration_registry)` adds the registered providers after generic pull, and rejects duplicate IDs. + - `pub(crate) fn IntegrationRegistry::from_registrations(registrations: Vec) -> Result>`. `with_plan` gathers its registrations as today (`prebid::register_for_plan`, `aps::register_for_plan` and `crate::integrations::builders()`, `integrations/mod.rs:292`) and then calls it. The merge and validation loop moves into it, so tests can pass a test integration that no settings can add. + - In `from_registrations`: every capture path and prefix must lie under one of the registering integration's `IntegrationProxy::routes()`. Otherwise it is a `Configuration` error naming the integration. + - `ProviderCatalog::from_settings(settings, registered: &[Arc])` adds the registered providers after generic pull and runs Task 22's `validate_identity_catalog` over every descriptor, so a duplicate ID is reported by the one validator. `IdentityRuntime::from_settings(settings, integrations: &IntegrationRegistry)` passes `integrations.identity_providers()`. The adapter's `build_identity_runtime` passes `®istry`, both from `build_state_from_settings` and from the settings-reload branch. + - `pub(crate) fn identity_descriptors_for_deploy(settings: &Settings) -> Result, Report>`: the generic pull descriptors built from `settings.ec.partners` (Task 22, secret-free) plus one line per identity integration. No integration has a line in this PR. Task 33 adds the mechanism for one and the test line. + - `examples.rs` (`#[cfg(test)]`): `ExampleIdConfig`, `ExampleIdState`, `ExampleIdProvider` and the capture-only `ExampleCaptureProvider`, copied verbatim from spec 4.6, plus `example_config()`. Tasks 33, 38 and 39 import from here. - [ ] **Step 1: Write the failing tests** ```rust #[test] fn try_with_identity_provider_converts_a_constructor_error_once() { - let failing: Result> = + let failing: Result> = Err(IdentityProviderError::config("bad partner id")); let error = IntegrationRegistration::builder("exampleid") .try_with_identity_provider(failing) - .expect_err("should fail"); + .expect_err("should fail on a constructor error"); assert!( matches!(error.current_context(), TrustedServerError::Integration { integration, .. } if integration == "exampleid"), "should name the integration" ); let built = IntegrationRegistration::builder("exampleid") - .try_with_identity_provider(Ok(ExampleTestProvider::new())) - .expect("should register") + .try_with_identity_provider(ExampleIdProvider::new(&example_config())) + .expect("should register a valid provider") .build(); - assert_eq!(built.identity_providers.len(), 1, "control: one provider registered"); + assert_eq!(built.identity_providers.len(), 1, "should hold one registered provider"); } #[test] fn capture_rule_outside_the_integrations_routes_is_rejected_at_startup() { - let settings = settings_with_capture_integration("/integrations/other/api"); - let error = IntegrationRegistry::with_plan(&settings, empty_plan()).expect_err("should reject"); + let error = IntegrationRegistry::from_registrations(vec![capture_registration("/integrations/other/api")]) + .expect_err("should reject a capture rule outside the integration's routes"); assert!(format!("{error}").contains("examplecapture"), "should name the integration"); - let ok = settings_with_capture_integration("/integrations/examplecapture/api/token"); - IntegrationRegistry::with_plan(&ok, empty_plan()).expect("control: a rule under its own routes"); + + IntegrationRegistry::from_registrations(vec![capture_registration("/integrations/examplecapture/api/token")]) + .expect("should accept a capture rule under the integration's own routes"); } ``` -`ExampleTestProvider` and the capture integration are test-only types defined in this module's tests. The capture integration registers an `IntegrationProxy` with routes under `/integrations/examplecapture/` and a capture provider whose path comes from the helper's argument. Follow how existing registry tests register a stub integration (`grep -n "from_request_filters\|fn stub_" crates/trusted-server-core/src/integrations/registry.rs | head`). +`capture_registration(path)` is a test helper in this module. It builds an `IntegrationRegistration` for `examplecapture` with an `IntegrationProxy` whose routes sit under `/integrations/examplecapture/`, and an `ExampleCaptureProvider` whose capture path is `path`. Follow how existing registry tests build a stub integration (`grep -n "from_request_filters\|fn stub_" crates/trusted-server-core/src/integrations/registry.rs | head`). + +In `catalog.rs`: `registered_provider_with_a_generic_pull_id_is_rejected_by_the_shared_validator`. A registered provider whose ID equals a generic pull ID fails with Task 22's duplicate-ID error text. The control registers `exampleid` next to one pull partner and builds. - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- integrations::registry::tests::try_with_identity_provider` and `cargo test-fastly -- integrations::registry::tests::capture_rule` -Expected: FAIL, then PASS. Add `identity_providers: Vec>` to `IntegrationRegistration` and merge it in `with_plan`, as proxies are merged. A route matches a capture path when the path equals the route's path or, for a wildcard route, starts with its prefix. Mirror how `handle_proxy` matches routes today. +Run: `cargo test-fastly -- integrations::registry::tests` and `cargo test-fastly -- ec::identity::catalog` +Expected: FAIL, then PASS. Add `identity_providers: Vec>` to `IntegrationRegistration` and merge it in `from_registrations`, as proxies are merged. A route matches a capture path when the path equals the route's path or, for a wildcard route, starts with its prefix. Mirror how `handle_proxy` matches routes today. Existing `with_plan` tests must pass unchanged. - [ ] **Step 3: Commit** ```bash -git add -A crates/trusted-server-core/src +git add -A crates/ git commit --signoff -S -m "Let integrations register identity providers" ``` @@ -6945,34 +9277,36 @@ git commit --signoff -S -m "Let integrations register identity providers" - Modify: `crates/trusted-server-core/src/settings.rs` (`EcPartner` gains `owner`) - Modify: `crates/trusted-server-core/src/ec/registry.rs` (`PartnerConfig.owner`) -- Modify: `crates/trusted-server-core/src/config.rs` (`validate_identity_integration`, the identity validation pass in deploy and runtime validation) -- Modify: `crates/trusted-server-core/src/ec/identity/config.rs` (`validate_identity_catalog`) -- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (owners from `[[ec.partners]].owner`, plus `sources` narrowing) -- Modify: `crates/trusted-server-core/src/ec/identity/generic_pull.rs` (an owned pull partner uses `pull_sync_ttl_sec` as `refresh_after_sec`) -- Test: `config.rs`, `ec/identity/config.rs` +- Modify: `crates/trusted-server-core/src/config.rs` (`validate_identity_integration`, and `identity_descriptors_for_deploy` gains its test line) +- Modify: the module that holds Task 22's `validate_identity_catalog` (the ownership rules) +- Modify: `crates/trusted-server-core/src/ec/identity/catalog.rs` (owners from `[[ec.partners]].owner`; Task 14's `sources` narrowing already applies) +- Test: `config.rs`, and the `validate_identity_catalog` test module **Interfaces:** +- Consumes: Task 22's `validate_identity_catalog(descriptors, partners, identity)`, which already rejects a duplicate provider ID, more than 32 providers (generic pull included), an `[ec.identity.providers.]` table naming an unknown provider, and `revoke_scope` set for a provider whose descriptor declares no `captures`. This task adds rules to that one function. It does not add a second validator. - Produces: - - `EcPartner.owner: Option`, `#[serde(default, skip_serializing_if = "Option::is_none")]` + - `EcPartner.owner: Option` and `PartnerConfig.owner: Option`, `#[serde(default, skip_serializing_if = "Option::is_none")]` - `fn validate_identity_integration(settings: &Settings, id: &str) -> Result, Report>`. It parses `[integrations.]` as `T` and, when enabled, calls `build_descriptor`. It never builds a provider and never reads secrets. - - `pub(crate) fn validate_identity_catalog(descriptors: &[ProviderDescriptor], partners: &[EcPartner], identity: &IdentitySettings) -> Result<(), Report>`, with every error of spec 10.2: + - New rules in `validate_identity_catalog`, every remaining error of spec 10.2: - `owner` names a provider that is not enabled - `owner` names a provider whose (narrowed) descriptor lacks that source - an enabled provider claims a source with no `[[ec.partners]]` entry - two enabled providers claim a source one of them owns (`sources` narrowing applied first) - `sources` names an undeclared source, or narrows to nothing - - two providers have the same ID - - `[ec.identity.providers.]` names an unknown provider - - more than 32 enabled providers, generic pull included - - The validation runs in `validate_settings_for_deploy` (next to `PartnerRegistry::validate_config_for_deploy`) and in `validate_settings_for_runtime`. The deploy path collects descriptors from a new `identity_descriptors_for_deploy(settings)` list, which has one `validate_identity_integration::` line per identity integration. It is empty in this PR except under `#[cfg(test)]`, where the test `ExampleIdConfig` is added, so the list's shape is exercised. + - a capture-only provider (one that declares `captures` and no `resolves`) has no `[ec.identity.providers.].sources` list: it must name the sources it is the delivery path for (spec 10.2) + - `owner` names a capture-only provider: a capture-only provider cannot own a source, because it stores no positive ID (spec 4.7) + - The deploy path (`validate_settings_for_deploy`, next to `PartnerRegistry::validate_config_for_deploy`) calls `validate_identity_catalog` on `identity_descriptors_for_deploy(settings)`. It never builds a `PartnerRegistry`, because `PartnerRegistry::from_config` validates API tokens that are still key names at deploy time. The runtime path validates through `IdentityRuntime::from_settings` (Task 32). `identity_descriptors_for_deploy` has one `validate_identity_integration::` line per identity integration. No production integration has one in this PR. Under `#[cfg(test)]` it has one for `examples::ExampleIdConfig` (Task 32), so the list's shape is exercised. - [ ] **Step 1: Write the failing tests** -One test per error above, each paired with a control config that passes. Also: +One test per new rule above, each paired with a control config that passes. Also: - `config_without_ec_identity_is_accepted` - `build_descriptor_runs_in_deploy_validation_without_secrets`: a config whose secret field holds a key name, not a value, still validates +- `capture_settings_on_a_provider_without_captures_are_rejected_at_deploy`: through `validate_settings_for_deploy`, `revoke_scope = "global"` on the resolve-only `exampleid` fails, naming the provider. The rule is Task 22's; this test proves the deploy path reaches it with an integration descriptor. The control is the same key on `ExampleCaptureProvider`'s descriptor, with its `sources` listed, which validates. +- `capture_only_provider_without_sources_is_rejected` and `capture_only_provider_cannot_own_a_source`, each with a control. +- `capture_enabled_has_no_effect`: a provider table holding `capture_enabled = true` loads (Task 19 keeps provider tables lenient) and changes nothing: a capture still stores no positive ID. For example: @@ -6981,20 +9315,28 @@ For example: fn owner_must_name_an_enabled_provider_that_declares_the_source() { let descriptors = vec![descriptor("exampleid", &["exampleid.example.com"])]; let ok = vec![partner("exampleid.example.com", Some("exampleid"))]; - validate_identity_catalog(&descriptors, &ok, &IdentitySettings::default()).expect("control"); + validate_identity_catalog(&descriptors, &ok, &IdentitySettings::default()) + .expect("should accept an owner that declares the source"); let unknown = vec![partner("exampleid.example.com", Some("missing"))]; - let error = validate_identity_catalog(&descriptors, &unknown, &IdentitySettings::default()).expect_err("unknown owner"); + let error = validate_identity_catalog(&descriptors, &unknown, &IdentitySettings::default()) + .expect_err("should reject an owner that is not enabled"); assert!(format!("{error}").contains("missing"), "should name the owner"); - let wrong_source = vec![partner("exampleid.example.com", None), partner("other.example.com", Some("exampleid"))]; - validate_identity_catalog(&descriptors, &wrong_source, &IdentitySettings::default()).expect_err("owner lacks source"); + let wrong_source = vec![ + partner("exampleid.example.com", None), + partner("other.example.com", Some("exampleid")), + ]; + validate_identity_catalog(&descriptors, &wrong_source, &IdentitySettings::default()) + .expect_err("should reject an owner whose descriptor lacks the source"); } ``` +`descriptor` and `partner` are test helpers in that module. `partner` builds the partner type Task 22's signature takes. + - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- ec::identity::config` and `cargo test-fastly -- config::` +Run: `cargo test-fastly -- validate_identity_catalog`, `cargo test-fastly -- owner`, and `cargo test-fastly -- config::` Expected: FAIL, then PASS. - [ ] **Step 3: CLI check** @@ -7015,26 +9357,30 @@ git commit --signoff -S -m "Add source ownership and validate identity provider - Modify: `crates/trusted-server-core/src/consent/gpp.rs` (around `decode_us_sale_opt_out` at line 125, and the section-6 skip at line 112) - Modify: `crates/trusted-server-core/src/consent/types.rs` (decoded fields, and a redacting `Debug` for `ConsentContext`) -- Modify: `crates/trusted-server-core/src/consent/mod.rs` (`stricter_of`) - Modify: `crates/trusted-server-core/src/ec/identity/policy.rs` (new blocks, and the fingerprint bits) -- Test: `consent/gpp.rs`, `consent/mod.rs`, `policy.rs` +- Test: `consent/gpp.rs`, `consent/types.rs`, `policy.rs` **Interfaces:** +- Consumes: Task 20's `fingerprint_bits` (TCF purposes, the vendor bit, the US sale opt-out, GPC and DNT, as a plain hex bitmask), which this task extends - Produces: - Decoded on the GPP form: `us_sharing_opt_out: Option`, `us_targeted_advertising_opt_out: Option`, `us_gpc: Option` and `us_known_child: Option` (US National section), and the section 6 US Privacy string read through the existing US Privacy decoder. Use the `iab_gpp` fields the spec names: `sharing_opt_out` (US National, California) and `targeted_advertising_opt_out` (US National and the state sections that define it). Check the `iab_gpp` version in `Cargo.lock` and read its section structs before writing the extraction. - `policy_allows` additionally blocks a strict policy on the sharing opt-out, the targeted-advertising opt-out, GPP's GPC field, and a known child without consent (a known child blocks every provider, whatever the relaxations). - - `pub(crate) fn fingerprint_bits(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> String`: a canonical string of the TCF purpose bits and vendor bit the policy reads, the US opt-outs it does not relax, and GPC and DNT unless relaxed. Never the raw consent string. - - `pub(crate) fn stricter_of(cookie: &ConsentContext, body: &ConsentContext) -> ConsentContext`: field by field, any opt-out, missing purpose or missing vendor consent in either wins + - A GPP string that decodes but has no applicable section, such as the header-only `DBAA` (spec 9.1), keeps its decoded form with every new field `None`. It carries no opt-out, so it does not block `strict()` by itself. Task 12 owns the fail-closed rule (a raw string with no decoded form blocks). The new decoders must not turn an empty section list into a missing decoded form. Where TCF applies, the policy still needs TCF consent from a TC string or a GPP TCF section. + - `fingerprint_bits(policy, consent, signals)` appends two bits after Task 20's: the sharing and the targeted-advertising opt-out, each 0 when the policy relaxes it (spec 8). GPP's GPC field sets the existing GPC bit. Each signal keeps its own named bit position (`const FP_US_SALE: u32`, and so on, in `policy.rs`), so Task 36 can test the US opt-out bits by mask. Never the raw consent string. + - Pub(crate) consts `FP_US_OPT_OUT_MASK` (sale, sharing, targeted advertising, GPC) for Task 36. - A hand-written `impl fmt::Debug for ConsentContext` that prints `[redacted]` for every raw string - [ ] **Step 1: Write the failing tests** - In `gpp.rs`: one decode test per new field, per section that defines it, using GPP strings built the way existing tests build them. Each asserts the decoded value, next to a control string where the field is not set. - In `policy.rs`: `strict_blocks_sharing_targeted_gpc_field_and_known_child`, and `targeted_advertising_opt_out_without_sale_opt_out_blocks_strict`. -- In `consent/mod.rs`: `stricter_of_takes_any_opt_out_or_missing_consent_from_either_side`. +- In `policy.rs`: `header_only_gpp_does_not_block_strict_where_tcf_does_not_apply`. A request outside TCF jurisdiction with only the GPP string `DBAA` passes `strict()`. Two controls: the same string with `gdpr_applies` and no TC string fails for lack of TCF consent, and an undecodable GPP string fails closed, as Task 12 has it. +- In `gpp.rs`: `header_only_gpp_decodes_with_no_section_fields`. `DBAA` decodes, `section_ids` is empty, and every new field is `None`. - `consent_context_debug_redacts_raw_strings`: `format!("{context:?}")` does not contain the TC string. - `fingerprint_is_stable_across_cmp_resaves_and_changes_with_a_purpose`: two TC strings with identical decisions but different `LastUpdated` give the same bits, and dropping purpose 4 changes them. +- `fingerprint_bits_name_each_opt_out_separately`: setting only the sharing opt-out and setting only the targeted-advertising opt-out give two different bitmasks, both different from neither, and each sets only its own bit. +- `fingerprint_keeps_part_1_bit_positions`: a consent that sets only Part 1 signals gives the same hex value as Task 20's function did (pinned). - [ ] **Step 2: Run, see them fail, implement, see them pass** @@ -7045,7 +9391,7 @@ Expected: FAIL, then PASS. Do not change any existing auction consent decision. ```bash git add -A crates/trusted-server-core/src/consent crates/trusted-server-core/src/ec/identity -git commit --signoff -S -m "Decode the GPP opt-outs strict identity policies need" -m "Adds the US sharing and targeted-advertising opt-outs, GPP section 6, and the US National GPC and known-child fields. ConsentContext's Debug output no longer prints raw consent strings." +git commit --signoff -S -m "Decode the GPP opt-outs strict identity policies need" -m "Adds the US sharing and targeted-advertising opt-outs, GPP section 6, and the US National GPC and known-child fields. A header-only GPP string with no applicable section still does not block a strict policy. ConsentContext's Debug output no longer prints raw consent strings." ``` ### Task 35: Rate-limiter `record` and `count`, and the circuit breaker @@ -7053,31 +9399,36 @@ git commit --signoff -S -m "Decode the GPP opt-outs strict identity policies nee **Files:** - Modify: `crates/trusted-server-core/src/ec/rate_limiter.rs` (trait) +- Modify: `crates/trusted-server-core/src/ec/batch_sync.rs` (`MockRateLimiter`, about line 328, implements the new methods) +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` (`FakeRateLimiter`) - Modify: `crates/trusted-server-adapter-fastly/src/rate_limiter.rs` (`FastlyRateLimiter`) -- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (breaker in dispatch, step 7) -- Test: all three +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (breaker in dispatch, eligibility step 6) +- Test: `rate_limiter.rs` (adapter), `coordinator.rs` **Interfaces:** - Produces: - - `RateLimiter::record(&self, key: &str) -> Result<(), Report>` and `RateLimiter::count(&self, key: &str) -> Result>`. Both are trait methods without defaults, so every implementation must decide. Fastly uses `counter.increment(key, 1)` and `counter.lookup_count(key, CounterDuration::SixtySecs)`. + - `RateLimiter::record(&self, key: &str) -> Result<(), Report>` and `RateLimiter::count(&self, key: &str) -> Result>`. Both are trait methods without defaults, so every implementation must decide: `FastlyRateLimiter`, `MockRateLimiter` in `ec/batch_sync.rs`, and `FakeRateLimiter`. (`AllowAllRateLimiter` in `ec/pull_sync.rs` is gone since Task 29.) Fastly uses `counter.increment(key, 1)` and `counter.lookup_count(key, CounterDuration::SixtySecs)`, and maps errors the way `exceeded` does. + - `pub struct FakeRateLimiter` in `test_support.rs` (public under `test-utils`, as the module's other items), storing counts in a `Mutex>` - Breaker: before dispatch, `count("identity-fail:") >= breaker_failures_per_minute` gives `CallOutcome::BreakerOpen` (not dispatched, `ps` unchanged). After dispatch, only `TransportFailure` outcomes (transport errors, timeouts, 429 and 5xx) call `record("identity-fail:")`. `HoldCutOff` is a budget cut, not a provider failure, and never records (spec 4.7). - - A limiter error is logged, and the breaker is treated as closed. Skip the burst cap check on that request too, failing closed for disclosure: no dispatch. + - Limiter errors. If `count` fails, log it and treat the breaker as closed. If the burst-cap check fails, log it and do not dispatch that provider on this request. If `record` fails, log it and keep the outcome. - [ ] **Step 1: Write the failing tests** -In core, with a `FakeRateLimiter` in `test_support.rs` that stores counts in a `Mutex`: +In core, with `FakeRateLimiter`, which can also be set to fail each method: - `breaker_opens_at_threshold_and_skips_every_visitor` - `four_xx_and_malformed_never_open_the_breaker`: 1,000 visitors with 400 responses leave `count` at 0 - `hold_cut_off_does_not_record_a_failure` - `burst_cap_is_checked_after_every_other_eligibility_step`: an ineligible provider never increments the burst counter +- `breaker_count_error_treats_the_breaker_as_closed` +- `burst_cap_error_skips_the_dispatch` -In the adapter: a unit test for `record` and `count` against Viceroy's rate counter, following the existing `mod tests` in `rate_limiter.rs`. +In the adapter: the existing `mod tests` in `rate_limiter.rs` tests only pure helpers and never calls the counter, because Viceroy 0.17's Edge Rate Limiting host calls cannot be relied on to count (this plan has not confirmed that they do). Keep that pattern. Put the two counter calls behind a private `trait CounterOps { fn increment(&self, key: &str) -> ...; fn lookup_count(&self, key: &str) -> ...; }`, implemented for `RateCounter`, and test `record` and `count` against a fake `CounterOps`: `record_increments_once`, `count_reads_the_sixty_second_window`, and `counter_errors_map_to_kv_store_errors`. - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- rate_limiter` and `cargo test-fastly -- ec::identity::coordinator` +Run: `cargo test-fastly -- rate_limiter`, `cargo test-fastly -- breaker`, `cargo test-fastly -- burst_cap` and `cargo test-fastly -- ec::batch_sync` Expected: FAIL, then PASS. - [ ] **Step 3: Commit** @@ -7087,82 +9438,91 @@ git add -A crates/ git commit --signoff -S -m "Add a per-provider circuit breaker on transport failures" ``` -### Task 36: Provider state, sticky opt-outs, re-grants and owner denials +### Task 36: Owner precedence, opt-out re-grants and the admin clear **Files:** - Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (opt-out clearing on re-grant) -- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` and `publisher.rs` (owner denials filter browser candidates in the merge) +- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` and `publisher.rs` (the owned precedence row) - Modify: `crates/trusted-server-core/src/ec/admin.rs` (an operator clear of `ps.o`) -- Test: `coordinator.rs`, `auction/endpoints.rs`, `ec/admin.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` (`Settings::ADMIN_ENDPOINTS`, about line 3346) +- Modify: the admin route tables of all four adapters: `crates/trusted-server-adapter-fastly/src/app.rs` (routes at about line 1149, the module doc table at lines 25-26), `crates/trusted-server-adapter-axum/src/app.rs` (about line 365), `crates/trusted-server-adapter-cloudflare/src/app.rs` (about line 640) and `crates/trusted-server-adapter-spin/src/app.rs` (`named_fallback_paths`, about line 224, whose array length grows by one) +- Test: `coordinator.rs`, `auction/endpoints.rs`, `publisher.rs`, `ec/admin.rs`, `settings.rs`, adapter `app.rs` tests **Interfaces:** +- Consumes: Task 28's `ServerSet`, `Denials` and `filter_browser_candidates`, which already drop denied browser UIDs and every source of an opted-out owner or capture-only provider; Task 34's `fingerprint_bits` and `FP_US_OPT_OUT_MASK`; Task 15's `opt_out_clear`, which also empties `ps.ov` - Produces: - - `pub(crate) fn regrant_clears_opt_out(descriptor: &ProviderDescriptor, opt_out: &KvOptOut, consent: &ConsentContext, keys: &IdentityKeys, signals: PolicySignals) -> bool` (spec 8): + - `pub(crate) fn regrant_clears_opt_out(descriptor: &ProviderDescriptor, opt_out: &KvOptOut, consent: &ConsentContext, signals: PolicySignals) -> bool` (spec 8): - **GVL vendor:** a TC string whose `LastUpdated` is after `o.at` and that grants the vendor and purposes. - - **Any provider, under GPP:** the applicable US section shows none of the opt-outs the policy reads, where the stored `fp` showed one. Compare the current `fingerprint_bits`-derived fingerprint's opt-out part. - - A jurisdiction change, a newly present GPP string, or a relaxed signal never clears. - - When it clears, the writer adds `opt_out_clear = Some(now)` to that provider's `ProviderStateChange`. Clears are writer-only, so they ride the one flush. - - Merge (spec 7.1, "Owner denials apply to browser candidates"): before the precedence table and before `gate_eids_by_consent`, drop the browser's candidate for an owned source when its UID's keyed hash is in the owner's `iv` (KV `ps.iv` ∪ `ts-eids-r` `iv`), or when the owner has a sticky opt-out (KV ∪ `ts-eids-r`). Unowned sources are unaffected. - - The owned rows of the precedence table become active: the owner's valid value wins over the browser's. When the browser did not send the source, a browser-seeded value is used only if the source does not expire. - - Admin: `DELETE /_ts/admin/ec/{id}/providers/{provider}/opt-out`, which clears `ps.o` by writing `oc = now` through the existing admin write path. Add it to the admin route table next to the EC lookup, behind the same Basic-auth coverage, and add the route to the admin coverage test. + - **Any provider, under GPP:** parse `o.fp` as a `u64` (Task 6 validates it as hex). Clear when `fp & FP_US_OPT_OUT_MASK` is not 0 and the current `fingerprint_bits` has none of the US opt-out bits `fp` recorded set: `current & fp & FP_US_OPT_OUT_MASK == 0`. Other bits may differ; the spec clears "whatever else changed" (spec 13). + - A jurisdiction change, a newly present GPP string, or a relaxed signal never clears. A `fp` that does not parse never clears. + - When it clears, the writer adds `opt_out_clear = Some(now)` to that provider's `ProviderStateChange`. Clears are writer-only on the resolve path, so they ride the one flush; Task 38 adds the capture-only clear through the capture removal write. + - The owned row of the precedence table becomes active (spec 7.1). The owner's valid value (`w == owner`, not expired) is its whole entry set, and it replaces every browser entry for the source, all of them when the browser repeated the source. The two are never mixed. Task 28's output rules then send it, one `Eid` per stored entry in stored order, after the browser entries. Without an owner's valid value, the browser's value goes out after Task 28's denials. When the browser did not send the source, a browser-seeded value is used only if the source does not expire. A browser-seeded record never beats the browser's current value. + - Admin: `DELETE /_ts/admin/ec/{id}/providers/{provider}/opt-out`, which clears `ps.o` by writing `oc = now` through the existing admin write path, behind the same Basic-auth coverage. The clear empties `ps.ov` (Task 15). Add the path to `Settings::ADMIN_ENDPOINTS` so its sync test (about `settings.rs:8191`) covers it. Add the route to all four adapters. The Fastly adapter calls the core handler; Axum, Cloudflare and Spin return the same "not available on this adapter" response they give the EC lookup. Keep the Fastly reserved-path test's `/_ts/admin/ec/{ec_id}/extra` case (app.rs, about line 2078) as a path that must not route. - [ ] **Step 1: Write the failing tests** - `tcf_regrant_after_opt_out_clears_it` and `jurisdiction_change_never_clears_it` -- `gpp_section_without_opt_out_clears_when_fingerprint_had_one` -- `invalidated_uid_in_auction_body_is_not_sent_before_or_after_removal_reaches_kv` -- `owner_opt_out_drops_browser_value_for_every_owned_source` +- `gpp_regrant_clears_when_none_of_the_recorded_opt_outs_remain`: the opt-out was stored with the sale bit set, and the current section shows no opt-out. It clears even with purpose 4 also changed. Two controls: a current section that still shows the sale opt-out does not clear, and a stored `fp` with no US opt-out bit never clears by GPP. +- `regrant_clear_empties_ov`: after the clear flushes, `ps.ov` is empty and the browser UID it held goes out again. - `owner_valid_value_beats_browser_value` +- `owner_valid_set_replaces_every_repeated_browser_entry`: the browser sends the owned source twice, and the owner's stored set has two entries with three UIDs and EID-level fields. Exactly the owner's two entries go out, in stored order, with every field written out. +- `targeted_opt_out_blocks_server_ids_flush_and_mirror` (spec 13): a targeted-advertising opt-out without a sale opt-out gives no server-side IDs in `user.eids`, no flush of that provider's entries, and none in the `ts-eids-s` mirror payload - `browser_seeded_value_never_beats_browser_current_value` - `expiring_owned_source_never_serves_browser_seeded_value` -- `unowned_source_unaffected_by_owner_denials` (control) -- `admin_clears_provider_opt_out` +- `unowned_source_keeps_the_browser_first_rule` (control) +- `admin_clears_provider_opt_out`, `admin_opt_out_route_is_in_admin_endpoints`, and, per adapter, a test that the `DELETE` path routes (Fastly) or returns the not-available response (the others) - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- ec::identity` and `cargo test-fastly -- auction::endpoints` and `cargo test-fastly -- ec::admin` -Expected: FAIL, then PASS. +Run: `cargo test-fastly -- regrant`, `cargo test-fastly -- owner`, `cargo test-fastly -- targeted_opt_out`, `cargo test-fastly -- browser_`, `cargo test-fastly -- ec::admin`, `cargo test-fastly -- admin`, `cargo test-axum`, `cargo test-cloudflare` and `cargo test-spin` +Expected: FAIL, then PASS. The merge tests compare whole `Vec` values with every field written out, with no `..Default::default()`. Task 28's `merge_` and `denial_` tests and the Part 1 `golden_` tests for browser entries must pass unchanged. - [ ] **Step 3: Commit** ```bash -git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Apply owner precedence, owner denials and opt-out re-grants" +git add -A crates/ +git commit --signoff -S -m "Apply owner precedence and clear provider opt-outs on re-grant" ``` -### Task 37: Hold mode +### Task 37: Hold mode on the writer navigation **Files:** - Modify: `crates/trusted-server-core/src/publisher.rs` (before the navigation auction dispatch, about lines 4886-4889) -- Modify: `crates/trusted-server-core/src/auction/endpoints.rs` (`handle_auction`, before the merge) -- Modify: `crates/trusted-server-core/src/publisher.rs` (the `/_ts/page-bids` auction, about lines 7077-7090) -- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (`run_hold`) -- Test: `coordinator.rs`, `auction/endpoints.rs`, `publisher.rs` +- Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (`run_hold`, `HoldResults`) +- Modify: `crates/trusted-server-core/src/ec/identity/mutation.rs` (tag hold outcomes for `retain_for_error_status`, Task 29) +- Test: `coordinator.rs`, `publisher.rs`, `mutation.rs` **Interfaces:** -- Produces: `pub(crate) fn run_hold(deps: &IdentityDeps<'_>, context: &mut EcContext) -> HoldResults`. It runs eligible `Hold`-phase providers within `hold_budget_ms` from the first dispatch, after recomputing eligibility from the KV row this request read (pre-disclosure revalidation, spec 4.7). It returns the answers as server-set additions for this auction and stages them: - - A stateless provider's IDs go into `staged.s_cookie` as `k: 0` with `pv`, its `ps` changes are marked `sg: 1`, and its removals and opt-outs go into `staged.new_r_batch`. On the writer they also join the mutation. - - A stateful provider (writer only) puts its whole `ProviderResult` in the mutation only. Never in the cookie. - - A cut-off call sets `n = now + hold_retry_sec` and counts no failure. -- `handle_auction` and the navigation take `IdentityDeps` inside the existing deps or context struct so neither passes 7 arguments. -- `/_ts/page-bids` runs hold mode only when its auction runs, because only then does it read KV. +- Consumes: `IdentityDeps` (Task 21), `eligibility` with `Phase::Hold` and `outcome_to_changes` (Task 20), `dispatch` with a budget (Task 21), the `ServerSet` (Task 28), and `IdentityMutation::retain_for_error_status` (Task 29) +- Produces: + - `pub(crate) struct HoldResults { additions: BTreeMap>, writer: BTreeMap }`: the answers this auction's server set adds, per source, with the provider that wrote each, so Task 28's output rules can apply that provider's provenance default. `ServerSet::with_hold_results(&mut self, results: HoldResults)` overlays them, replacing the stored value for each source they name. + - `pub(crate) fn run_hold(deps: &IdentityDeps<'_>, context: &mut EcContext) -> HoldResults`. On a returning writer navigation that runs an auction, it dispatches every provider `eligibility(.., Phase::Hold)` passes on the phase 1 view, within `hold_budget_ms` measured from the first dispatch, each waiting `min(hold_timeout_ms, remaining budget)` (spec 4.7). The phase 1 read already confirmed a live, consenting row, so no EC ID is disclosed before consent is confirmed. + - Each outcome becomes changes through `outcome_to_changes` and joins the writer's mutation, stateful providers included. No answer goes into a cookie (spec 4.7, "Results"). + - Each `found` set also feeds this auction through `HoldResults`: one `Eid` per entry, in the order the provider returned it, after the browser entries, with the provider's provenance default applied (Task 28). + - A cut-off call is `HoldCutOff`: `n = now + hold_retry_sec`, and no failure is counted. + - The mutation tags each change and state change with whether it came from a `found` outcome, so `retain_for_error_status` (Task 29) keeps a hold call's removals, opt-outs and the `ps.n`/`ps.f` of every outcome other than `found`, and drops its positive answers, when the origin answers with a status other than 2xx or 304 (spec 5.1). +- Nothing else runs hold mode. `/auction`, `/_ts/page-bids` and every other stager run no resolving provider (spec 5.3). A hold-mode provider on a writer navigation without an auction runs after send in phase 4 (Task 20's `Phase::AfterSend` rule, Task 29), with `after_send_timeout_ms`. +- The navigation takes `IdentityDeps` inside its existing deps or context struct, so it passes no more than 7 arguments. - [ ] **Step 1: Write the failing tests** -With the fake HTTP client's latency control: +With the fake HTTP client's latency control. Every test name contains `hold`: - `auction_starts_at_the_hold_budget_when_a_provider_is_slow` (the elapsed fake time equals the budget, and the auction ran) -- `hold_answer_feeds_this_auction_and_is_saved_with_k0` -- `cut_off_call_sets_hold_retry_without_counting_a_failure` -- `stateful_hold_provider_never_runs_on_a_stager` -- `stateful_hold_answer_stays_out_of_the_cookie` -- `writer_navigation_with_non_2xx_status_keeps_hold_answers_in_its_cookie` -- `cookie_only_no_match_stops_a_recall_on_the_next_auction` (the merged view) -- `kv_opt_out_blocks_dispatch_even_when_the_cookie_lacks_it` +- `hold_answer_feeds_this_auction_and_joins_the_mutation` +- `full_set_hold_answer_is_sent_in_order_and_flushed_whole`: two entries, three UIDs, no EID-level fields on the second entry and `inserter` on the first. `user.ext.eids` carries both entries in order. The first entry goes out exactly as returned. The second, which sets none of `inserter`, `matcher` or `mm`, gets the provider's provenance default. The one flush stores the set as returned, without the default. +- `hold_sets_no_cookie`: no `Set-Cookie` beyond today's on the navigation response. +- `stateful_hold_answer_flushes_its_state_with_its_ids` +- `cut_off_hold_call_sets_hold_retry_without_counting_a_failure` +- `no_hold_on_a_stager`: `/auction` and `/_ts/page-bids` for the same visitor, with the hold provider due, make no provider call. +- `hold_provider_moves_after_send_on_a_navigation_without_an_auction`: dispatched in phase 4 with `after_send_timeout_ms`, and a miss is a transport failure, not a cut-off. +- `writer_navigation_with_non_2xx_status_flushes_hold_removals_and_backoff_only`: a 404 navigation whose hold calls returned one `found`, one `no_match` and one `invalidate`. The one write holds the `no_match` `ps.n` and the removal, and no ID from the `found`. +- `hold_no_match_on_a_writer_stops_recalls_from_the_spa_session`: after the writer's flush, a run of `/auction` and `/_ts/page-bids` calls never calls the provider before `ps.n`. +- `hold_kv_opt_out_blocks_dispatch` - [ ] **Step 2: Run, see them fail, implement, see them pass** @@ -7173,57 +9533,81 @@ Expected: FAIL, then PASS. Use the platform's time source the auction orchestrat ```bash git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Run hold-mode identity providers before the auction" +git commit --signoff -S -m "Run hold-mode identity providers before the writer navigation's auction" -m "Answers feed that auction and join the writer's single flush. Stagers run no provider, and a navigation without an auction runs hold-mode providers after send instead." ``` -### Task 38: Passive capture +### Task 38: Revoke-only passive capture **Files:** -- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`handle_proxy`, about lines 1031-1061) +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` (`handle_proxy`, about lines 1031-1065) - Modify: `crates/trusted-server-core/src/ec/identity/coordinator.rs` (`run_capture`) -- Test: `integrations/registry.rs`, `coordinator.rs` +- Modify: `crates/trusted-server-core/src/ec/finalize.rs` (`ec_finalize_response`: a requested global revoke takes the withdrawal branch) +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (`finalize_send_and_flush` runs the pending removal retry after send) +- Modify: the identity measurement line of Task 31 (`capture_unpersisted`) +- Modify: `crates/trusted-server-core/src/integrations/lockr.rs` (log only the upstream path, about line 204) +- Test: `integrations/registry.rs`, `coordinator.rs`, `ec/finalize.rs`, `integrations/lockr.rs`, `adapter-fastly/src/main.rs` **Interfaces:** -- Produces: `pub(crate) fn run_capture(deps: &IdentityDeps<'_>, context: &mut EcContext, method: &Method, path: &str, request_consent: Option, response_status: StatusCode, response_headers: &HeaderMap, response_body: &[u8])` +- Consumes: `ProxyDispatchInput.is_real_browser` (Task 23), `decode_body` (Task 21), `outcome_to_changes` (Task 20), `PartnerIdChange` and `ProviderStateChange` (Task 15), `write_removals`, `RemovalWriteOutcome` and `log_removal_lost` (Task 18), `RequestIdentity::pending_removals`, `record_removal_lost` and `record_unpersisted` (Task 23), `ProviderSettings.revoke_scope` (Task 19), `regrant_clears_opt_out` (Task 36), and `ExampleCaptureProvider` and `capture_registration` (Task 32) +- Produces: + - `pub(crate) struct CaptureCall<'a> { method: &'a Method, path: &'a str, is_real_browser: bool, status: StatusCode, headers: &'a HeaderMap, body: &'a [u8] }`. One struct, so `run_capture` stays under 7 arguments. There is no request-body consent: the request body is never read (spec 4.7). + - `pub(crate) fn run_capture(deps: &IdentityDeps<'_>, graph: Option<&KvIdentityGraph>, context: &mut EcContext, call: &CaptureCall<'_>)` + - `EcContext::request_global_revoke(&mut self)` and `EcContext::global_revoke_requested(&self) -> bool`. `run_capture` sets it only for a real browser. + - `pub fn retry_pending_removals(context: &mut EcContext, graph: &KvIdentityGraph)`: after send, one more `write_removals` for a capture removal write that failed before send. `handle_proxy` per request, only when some provider's `Captures::matches(method, path)`: -1. **Before `handle`:** if a matching descriptor declares `with_request_consent` and the request has a body, buffer it with `collect_body_bounded` (`integrations/mod.rs:182`, 256 KiB). Extract only the declared JSON pointers, decode them into a `ConsentContext` with the same decoders the cookie path uses, and re-wrap the bytes with `EdgeBody::from(bytes)` before calling `handle`. A body over the cap or not valid JSON gives `request_consent = None` and allows no positive IDs. -2. **After `handle`,** on 2xx only: buffer the response up to `max_response_bytes`. A larger body is passed through uncaptured. Decode a copy with `decode_body`, call `capture` per matching provider, and re-wrap the original bytes so the browser receives the response unchanged. -3. **Apply:** - - Removals and opt-outs always apply, whatever the consent. - - Positive IDs apply only when eligibility steps 1 to 3 pass under `stricter_of(cookie, body)` and the provider has no sticky opt-out, unless this result clears it. A capture-only provider without a GVL ID that returns IDs under consenting cookie and body consent stages `ps.oc = now` beside them. - - Capture never touches `n`, `f` or the breaker. - - A state change in a capture result is ignored with a warning. - - Results are staged on `EcContext` and written by finalize: positive into `ts-eids-s`, removals into `ts-eids-r`. On a writer request they also join the mutation. +1. **After `handle`,** on 2xx only: buffer the response up to `max_response_bytes`. A larger body is passed through uncaptured. Decode a copy with `decode_body`, call `capture` per matching provider, and re-wrap the original bytes with `EdgeBody::from(bytes)` so the browser receives the response unchanged. The request body is never read or buffered. +2. **The real-browser gate** (spec 4.7). When `call.is_real_browser` is false, nothing is written and nothing is staged: the removals, opt-outs and global revokes are dropped and logged at debug with the provider ID and source only, and the positive IDs are counted in `capture_unpersisted`. So a script replaying a copied `ts-ec` cannot strip or tombstone a row. +3. **Build the removal mutation** for a real browser. From each capture result, through `outcome_to_changes` on the request's KV view, keep only: + - its removals: `OwnerRemove` for `invalidate` and `also_invalidate`, and the opt-out's `WriterRemove`s, including a capture-only provider's browser records of its sources (Task 20). The hashes are derived when the mutation applies (Task 15). + - its opt-out: `ps.o = {at: now, fp}` and `st = Clear` + - a clear: when the result has positive IDs, eligibility steps 1 to 3 pass under the request's consent, and the provider has a sticky `ps.o` that only a capture can clear (a capture-only provider without a GVL ID, spec 8): `opt_out_clear = Some(now)` + - nothing else. Capture never touches `n`, `f` or the breaker. A state change in a result is ignored with a warning. Each positive source is counted in `capture_unpersisted` (`record_unpersisted(provider, source)`) and never stored. +4. **The capture removal write** (spec 5.3), before the proxied response is sent: `write_removals(graph, ec_id, &mutation, &context)` (Task 18). It re-reads, applies the removal rules of section 5.4, and writes with a CAS, up to five attempts, retrying only on a generation mismatch. `NothingToWrite` covers a tombstone, a missing row and a removal already applied. On `Failed`, the mutation is kept in `RequestIdentity::pending_removals`, and `finalize_send_and_flush` calls `retry_pending_removals` once after send. When that fails too, `log_removal_lost` logs at error with the provider ID and source only, and `record_removal_lost` counts each removal. Without a graph, there is nothing to write. +5. **On an integration-proxy writer navigation** the capture still goes through the capture removal write, never into the writer's mutation (spec 5.2 phase 3). The writer's own phase 4 flush then fails its CAS if the capture write landed first, which is the existing rule (spec 5.2 phase 4 step 2), and its removals are retried alone. +6. **Revoke scope** (spec 8). A captured `opted_out` with `revoke_scope = "provider"` (the default) is the provider opt-out above. With `"global"` and a real browser, `run_capture` calls `context.request_global_revoke()` and writes nothing else for that result. `ec_finalize_response` treats a requested global revoke exactly like `ec_consent_withdrawn`: the row is tombstoned through `tombstone_existing_from_snapshot`, the EC cookie is expired, the request's mutation is dropped, and a failed tombstone gets Task 27's after-send run. This is the withdrawal's existing page-load exception, not a new one. +7. **Lockr proxy logging** (spec 9.3). In `integrations/lockr.rs`, log only the upstream path: `log::info!("Forwarding to Lockr API path: {target_path}")`. The query string can carry identifiers and never reaches a log. + +Add `capture_unpersisted` (per provider and source, counts only) to Task 31's measurement line. - [ ] **Step 1: Write the failing tests** -Using a test capture integration (from Task 32) and its provider: +Using `capture_registration` and `ExampleCaptureProvider` (Task 32), and `KvIdentityGraph::write_counting` (Task 22). Every test name contains `capture`: - `capture_runs_only_for_matching_rules` -- `gzip_and_brotli_responses_are_decoded_for_capture_and_forwarded_unchanged` (byte-equal body to the browser) -- `only_declared_consent_pointers_are_read_from_the_request_body` (a body with an email field. The provider sees no body, and a log capture shows no email) -- `stricter_body_consent_blocks_positive_ids_but_not_removals` -- `revoke_style_opt_out_removes_every_record_the_provider_wrote_including_stubbed_sources` -- `capture_removal_survives_a_concurrent_auction_rewriting_ts_eids_s` (two responses built from the same inbound cookies: the `ts-eids-r` cookie from the capture is still present after the `/auction` response's `ts-eids-s` is applied) -- `revoke_then_reconsent_capture_ends_with_new_ids_and_oc_in_kv` (across a navigation, and again after a failed flush) +- `capture_decodes_gzip_and_brotli_responses_and_forwards_them_unchanged` (byte-equal body to the browser) +- `capture_never_reads_the_request_body` (a request body with an email field: the body reaches the upstream unchanged, the provider sees none, and a log capture shows no email) +- `capture_stores_no_positive_id_and_counts_it`: a capture returning two sources stores nothing for them, and `capture_unpersisted` holds one count for each, with no value. +- `capture_removal_is_written_before_send`: an `also_invalidate` from a real browser gives one CAS write before the response is sent, and the row's `ps.iv` holds the record's hashes. +- `capture_removal_applies_whatever_the_consent`: the same capture under denied consent writes the same removal. +- `capture_removal_write_retries_only_on_generation_mismatch` (a store that fails the CAS twice then succeeds: 3 attempts; a non-CAS error: 1 attempt) +- `failed_capture_removal_write_runs_once_more_after_send`, and `capture_removal_lost_after_both_attempts_is_counted_and_logged_without_a_value` +- `capture_removal_write_skips_a_tombstone_a_missing_row_and_an_applied_removal` (0 writes each) +- `capture_opt_out_removes_every_record_the_provider_wrote` +- `capture_only_opt_out_also_removes_browser_records_of_its_sources` +- `capture_only_provider_clears_its_opt_out_on_a_consenting_positive_result`: a later 2xx capture with IDs writes `ps.oc = now` and no ID. Two controls: the same capture under denied consent writes no clear, and a provider with a GVL ID never clears by capture. +- `capture_global_revoke_from_a_real_browser_tombstones_the_row_and_clears_the_ec_cookie`: `revoke_scope = "global"`. Exactly one tombstone write and the EC cookie's expiry header. The control with `"provider"` removes only that provider's records and keeps the row live. +- `capture_from_a_non_browser_writes_nothing`: `is_real_browser = false` with removals, an opt-out and a `"global"` revoke. Zero KV writes, no tombstone, no cookie. +- `capture_with_replayed_ec_cookie_changes_no_kv_row`: a non-browser request with a copied `ts-ec` and a revoke-style response. The counting store records zero writes. +- `capture_on_a_proxy_writer_navigation_uses_the_removal_write_not_the_mutation` - `capture_never_touches_n_f_or_the_breaker` - `capture_state_change_is_ignored` -- `oversized_proxied_body_passes_through_uncaptured` +- `capture_skips_an_oversized_proxied_body` +- In `integrations/lockr.rs`: `lockr_proxy_log_omits_query`, using a log capture, with a request whose query holds an identifier - [ ] **Step 2: Run, see them fail, implement, see them pass** -Run: `cargo test-fastly -- capture` -Expected: FAIL, then PASS. +Run: `cargo test-fastly -- capture`, `cargo test-fastly -- lockr_proxy_log` and `cargo test-fastly -- ec::finalize` +Expected: FAIL, then PASS. Today's withdrawal tests in `ec/finalize.rs` must pass unchanged. - [ ] **Step 3: Commit** ```bash -git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Capture identity IDs from proxied provider responses" +git add -A crates/ +git commit --signoff -S -m "Capture identity revocations from proxied provider responses" -m "Capture is revoke-only: a real browser's captured removals, opt-outs and clears are written to KV on the proxied request with a CAS loop, retried once after send, and counted as lost if both fail. Positive IDs are counted, never stored. A non-browser client's capture writes nothing. revoke_scope = \"global\" runs the consent-withdrawal tombstone from a real browser." -m "The Lockr proxy now logs only the upstream path, not its query." ``` ### Task 39: The full provider harness and conformance checks @@ -7231,14 +9615,20 @@ git commit --signoff -S -m "Capture identity IDs from proxied provider responses **Files:** - Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` -- Create: `crates/trusted-server-core/src/ec/identity/examples.rs` (`#[cfg(test)]` only): the spec 4.6 `ExampleIdProvider` and the capture example, as reference providers +- Modify: `crates/trusted-server-core/src/ec/identity/examples.rs` (created in Task 32): the harness tests - Test: `examples.rs` **Interfaces:** -- Produces everything in spec 13's harness listing: `ProviderHarness::{with_setup, with_client_ip, with_user_agent, with_page_url, with_region, with_hashed_email, with_consent, with_stored, with_state, with_now, with_random_seed, as_dry_run, request, request_json, respond, respond_raw, capture, resolve, state_of, check_conformance, check_conformance_with_state}`, `HarnessSetup`, `TestConsent`, `CaptureFixture`, `ConformanceCases`, `StoredId::{for_test, written_by, expiring_at}`, `HashedEmail::for_test`, and `check_descriptor_ignores_secrets`. All of them sit behind `#[cfg(any(test, feature = "test-utils"))]` and are reachable from outside the crate as `trusted_server_core::ec::identity::test_support::*`, so the module is `pub` under the gate. +- Produces everything in spec 13's harness listing: `ProviderHarness::{with_setup, with_client_ip, with_user_agent, with_page_url, with_region, with_hashed_email, with_consent, with_stored, with_state, with_now, with_random_seed, request, request_json, respond, respond_raw, capture, resolve, state_of, output_eids, check_conformance, check_conformance_with_state, try_check_conformance, try_check_conformance_with_state}`, `HarnessSetup`, `TestConsent`, `CaptureFixture`, `ConformanceCases`, `ConformanceFailure`, `StoredId::{for_test, with_eids, written_by, expiring_at}`, `HashedEmail::for_test`, and `check_descriptor_ignores_secrets`. The module and every item are `pub` under `#[cfg(any(test, feature = "test-utils"))]` (it has been since Task 14), and reachable from outside the crate as `trusted_server_core::ec::identity::test_support::*`. + - `StoredId::for_test` builds a one-entry, one-UID `KvPartnerId` with `KvPartnerId::new` and goes through `StoredId::from_record(source, &record)` (Task 9), so `eids()` holds one entry and `uid()` is its first UID. + - `StoredId::with_eids(self, eids: Vec) -> Self` replaces the set with a full one, and sets `uid()` to its first UID (spec 13), so a test can give a refreshing provider a stored full set. + - `HashedEmail::for_test(bytes: [u8; 32]) -> HashedEmail` calls Task 9's gated `HashedEmail::from_bytes`. There is one constructor, so the two cannot drift. + - `ProviderHarness::output_eids(&self, resolution: &Resolution) -> Vec` applies `entry_sets` (Task 10) and the descriptor's provenance default (Task 28) as the bidstream would. Provenance is applied only here; `Resolution`'s getters show what the provider set. + - `#[derive(Debug, derive_more::Display)] #[non_exhaustive] pub enum ConformanceFailure`, one variant per conformance bullet below (`UndeclaredInput`, `UndeclaredHost`, `CookieHeader`, `UndeclaredSource`, `LimitHit`, `NotImplemented`, `StatelessState`, `CaptureStateChange`, `MissingExpiry`, `BareStringStateAccepted`, and the rest), each naming what failed and never a value. It implements `core::error::Error`. + - `try_check_conformance(&self, cases: &ConformanceCases) -> Result<(), Report>` and `try_check_conformance_with_state::`. `check_conformance` and `check_conformance_with_state` call them and `expect("should conform: ...")`, for provider authors. `cargo test-fastly` runs on `wasm32-wasip1`, where a panic aborts the whole test binary and `catch_unwind` never returns `Err`, so the harness's own negative tests use the `try_` forms. No `panic!` anywhere in this code: the workspace denies `clippy::panic`. -`check_conformance` asserts every bullet of spec 13's conformance list: +`try_check_conformance` checks every bullet of spec 13's conformance list: - inputs hold only what the descriptor declares - a resolver returns `Some` for at least one listed setup, and its `response` never returns the `not_implemented` marker @@ -7246,17 +9636,16 @@ git commit --signoff -S -m "Capture identity IDs from proxied provider responses - no undeclared input or state read is recorded - no state is returned without `with_private_state()` - requests go only to declared hosts, over HTTPS, with no `Cookie` header and with valid headers -- IDs are only for declared sources, at most one per source -- `expiry()` and `refresh()` report what was set, and a fixture ID of an expiring source carries an expiry +- entries are only for declared sources, and no fixture's result hits a limit of spec 4.5: no dropped UID or field, no discarded entry, and no set over `MAX_ENTRY_SET_BYTES`. The harness runs `entry_sets` (Task 10) on each result and fails when `drops` or `oversized` is not empty. +- `ResolvedEid::expiry()` and `refresh()` report what was set, and a fixture set of an expiring source carries an expiry - `capture` never returns a state change - a bare-JSON-string stored state is never treated as valid -- the dry run is discarded unsent, and `as_dry_run()` fails for a stateful provider -`check_conformance` fails with a pointer to `check_conformance_with_state` for a stateful provider. `check_conformance_with_state::` also round-trips `S` through `replacing_state` and `input.state()`. +`try_check_conformance` returns an error with a pointer to `try_check_conformance_with_state` for a stateful provider. `try_check_conformance_with_state::` also round-trips `S` through `replacing_state` and `input.state()`. - [ ] **Step 1: Write the tests (the spec's examples are the first harness cases)** -In `examples.rs`, copy `ExampleIdConfig`, `ExampleIdProvider` and the capture-only example verbatim from spec 4.6, then: +In `examples.rs`, next to the Task 32 copies of spec 4.6: ```rust #[test] @@ -7266,20 +9655,27 @@ In `examples.rs`, copy `ExampleIdConfig`, `ExampleIdProvider` and the capture-on .with_client_ip("192.0.2.10") .with_user_agent("ExampleBrowser/1.0"); - let request = harness.request().expect("should build a request").expect("should not skip the request"); + let request = harness + .request() + .expect("should build a request") + .expect("should not skip the request"); assert_eq!(request.url().host_str(), Some("api.exampleid.example.com"), "should call the declared host"); let resolution = harness .respond(StatusCode::OK, &json!({ "uid": "example-uid", "signature": "example-signature" })) .expect("should parse the response"); - assert_eq!(resolution.ids()[0].uid(), "example-uid", "should return the uid"); + assert_eq!( + resolution.eids()[0].uids()[0].id(), + "example-uid", + "should return the uid" + ); let state: ExampleIdState = harness.state_of(&resolution).expect("should replace state"); assert_eq!(state.signature, "example-signature", "should keep the signature"); } #[test] fn example_provider_conforms() { - let provider = ExampleIdProvider::new(&example_config()).expect("should build"); + let provider = ExampleIdProvider::new(&example_config()).expect("should build the provider"); ProviderHarness::new(&provider).check_conformance_with_state::( &ConformanceCases::default().with_resolve_case( HarnessSetup::default().with_client_ip("192.0.2.10").with_user_agent("ExampleBrowser/1.0"), @@ -7292,21 +9688,40 @@ In `examples.rs`, copy `ExampleIdConfig`, `ExampleIdProvider` and the capture-on #[test] fn conformance_catches_an_undeclared_read() { let provider = UndeclaredUserAgentReader::new(); - let result = std::panic::catch_unwind(|| { - ProviderHarness::new(&provider).check_conformance( + let error = ProviderHarness::new(&provider) + .try_check_conformance( &ConformanceCases::default().with_resolve_case(HarnessSetup::default(), StatusCode::OK, &json!({})), - ); - }); - assert!(result.is_err(), "should fail a provider that reads an undeclared input through .ok()"); + ) + .expect_err("should fail a provider that reads an undeclared input through .ok()"); + assert!( + matches!(error.current_context(), ConformanceFailure::UndeclaredInput { .. }), + "should report the undeclared input" + ); } #[test] fn descriptor_ignores_secrets() { check_descriptor_ignores_secrets(&example_config(), &example_config()); } + + #[test] + fn output_eids_applies_the_provenance_default_only_to_bare_entries() { + let provider = ProvenanceTestProvider::new(); + let harness = ProviderHarness::new(&provider); + let resolution = harness + .resolve(&HarnessSetup::default()) + .expect("should resolve") + .expect("should not skip the request"); + let eids = harness.output_eids(&resolution); + assert_eq!(eids[0].inserter.as_deref(), Some("vendor-inserter"), "should keep the entry's own fields"); + assert_eq!(eids[0].matcher, None, "should not mix the default into an entry that sets a field"); + assert_eq!(eids[1].inserter.as_deref(), Some("default-inserter"), "should apply the default to a bare entry"); + } ``` -Add negative conformance tests the same way for: a `Cookie` header, a request to an undeclared host, two IDs for one source, a capability declared but not overridden (the `not_implemented` marker), a stateless provider returning state, a capture returning a state change, and an expiring source without an expiry. Each uses a tiny test provider written in the test module, and each asserts the harness panics. +Add negative conformance tests the same way, each with a tiny test provider in the test module and each asserting `try_check_conformance(...)` returns the matching `ConformanceFailure` variant: a `Cookie` header, a request to an undeclared host, an entry for an undeclared source, a fixture result that hits a spec 4.5 limit (a 33rd UID for one source, and a blank `inserter`), a capability declared but not overridden (the `not_implemented` marker), a stateless provider returning state, a capture returning a state change, a stateful provider that accepts a bare-JSON-string stored state, and an expiring source without an expiry. + +Several entries and UIDs for one source are valid (G13), so add a positive control too: `conformance_accepts_a_complete_set_within_the_limits`. A test provider returns two entries for one source, three UIDs in all, with `inserter`, `matcher`, `mm`, an EID-level `ext` and a UID-level `ext`, and `try_check_conformance` returns `Ok`. Add `stored_full_set_reaches_a_refreshing_provider`: `with_stored(StoredId::for_test(..).with_eids(..))` gives the provider `stored(source).eids()` equal to the set. - [ ] **Step 2: Run, see them fail, implement, see them pass** @@ -7315,8 +9730,8 @@ Expected: FAIL, then PASS. - [ ] **Step 3: Check the feature gate from outside the crate** -Run: `cargo check -p trusted-server-adapter-fastly --target wasm32-wasip1 --tests --features trusted-server-core/test-utils` (or whichever adapter already enables `test-utils` in its dev-dependencies) -Expected: builds. This proves a provider crate can use the harness. +Run: `cargo check -p trusted-server-adapter-fastly --target wasm32-wasip1 --tests` (the Fastly adapter's dev-dependency already enables `trusted-server-core/test-utils`, `crates/trusted-server-adapter-fastly/Cargo.toml:45`), then `cargo clippy-fastly` +Expected: both pass. This proves a provider crate can use the harness, and that the `pub` gated items raise no `dead_code` warning. - [ ] **Step 4: Commit** @@ -7325,26 +9740,76 @@ git add -A crates/trusted-server-core/src git commit --signoff -S -m "Add the provider harness and conformance checks" ``` +### Task 39b: Complete provider EIDs from a registered provider, end to end + +Part 1 built and tested multi-entry records at unit level and in the invariant tests (Tasks 6, 10, 15, 17, 28 and 30). This task proves that a provider registered through `try_with_identity_provider` gets a complete set from its response to KV, the mirror payload and the bidstream, and that nothing that predates the spec moved. + +**Files:** + +- Modify: `crates/trusted-server-core/src/ec/identity/test_support.rs` (`MultiEntryTestProvider`, `pub` under `test-utils`) +- Test: the Fastly adapter's request-level test module from Task 30 (`crates/trusted-server-adapter-fastly/src/main.rs` tests), using `drive_request_for_test` (Task 29) + +**Interfaces:** + +- Consumes: Tasks 32, 37, 38, the counting store (Task 22), and Task 29's `drive_request_for_test` and Task 30's request-level helpers +- Produces: no production code. `MultiEntryTestProvider`, a test-only resolver for `exampleid.example.com` that returns two entries: the first with UIDs `uid-a` (`atype: 1`, a UID-level `ext`) and `uid-b`, plus `inserter`, `matcher` and `mm`; the second with `uid-c` and an EID-level `ext`, and no `inserter`, `matcher` or `mm`. Its descriptor declares `with_provenance` with an `inserter` and `matcher` default for the source. `MultiEntryTestProvider::stateful()` is the same provider declared `with_private_state()`, for the stateful case. It lives in core `test_support` because the adapter tests cannot see core's `#[cfg(test)]` items. + +- [ ] **Step 1: Write the failing tests** + +Each test registers `MultiEntryTestProvider` through `IntegrationRegistration::builder("exampleid").try_with_identity_provider(...)`, owning the source, and drives requests through `drive_request_for_test`: + +- `registered_provider_full_set_reaches_kv_mirror_and_bidstream_in_order`: a writer navigation in hold mode. The flush writes the record with `eids` holding both entries as returned, `w = exampleid` and a fresh `rv`. `user.ext.eids` and `x-ts-eids` carry both entries in the order returned, after the browser entries. The first entry keeps exactly its own `inserter`, `matcher` and `mm`. The second, which sets none of them, gets the provenance default. The next `/auction`'s `ext.ts_eids_s` decodes to the same two entries, defaults included. +- `one_entry_one_uid_result_is_written_as_one_entry`: the same provider configured to return only `uid-a` with no EID-level fields. The KV record is `{w, rv, at, eids: [{uids: [{id: "uid-a", atype: 1, ext}]}]}`, the one shape of spec 6.1. +- `later_result_without_an_entry_removes_it`: a refresh that returns only the first entry leaves exactly that entry in KV and in the bidstream. A source the result does not name keeps its record. +- `stateful_result_with_an_oversized_set_keeps_only_its_removals`: a set over `MAX_ENTRY_SET_BYTES` from `MultiEntryTestProvider::stateful()`. The positive part, its state included, is rejected, and its `also_invalidate` still applies. +- `identify_returns_the_first_uid_of_a_full_record`: `/_ts/api/v1/identify` returns `uid: "uid-a"` and an `eid` with the partner's `openrtb_atype`, the same shape as for a one-UID record. +- `rollback_to_part_1_config_keeps_the_owners_opt_out`: a row with the provider's `ps.o`, `ps.ov` and a multi-entry record, flushed once by a build whose catalog no longer has the provider. The provider's `ps` keeps `o`, `oc` and `ov` and loses `n`, `f`, `iv` and `st`. The record stays readable but is not served (spec 14). There is no test of a rollback past Part 1: a release before it cannot read schema version 2, and that rollback is unsupported. +- `captured_revocation_writes_with_cas`, `global_revoke_from_real_browser_writes_as_withdrawal` and `non_browser_capture_writes_nothing`: the capture rows of spec 13's one-write table that Task 30 leaves to this task, driven through a capture registration (Task 32). The first gives one CAS write before send (up to five attempts), and one more after send only when it failed. +- `returning_navigation_answered_404_flushes_only_hold_removals_and_backoff`: the hold-mode half of the 3xx, 4xx and 5xx row. + +Then rerun Part 1's `golden_` tests for paths that predate the spec with the multi-entry provider registered on another source: browser ingest writes one-UID records, `ts-eids` content is unchanged, `/auction` body forwarding is unchanged, and a one-UID row goes out as on `main`. Do not edit those tests. If one fails, the change is wrong, not the test. + +Compare whole `Vec` values and whole `KvPartnerId` values with every field written out, with no `..Default::default()`. + +- [ ] **Step 2: Run, see them fail, implement, see them pass** + +Run: `cargo test-fastly -- registered_provider`, `cargo test-fastly -- one_entry_one_uid`, `cargo test-fastly -- later_result`, `cargo test-fastly -- stateful_result`, `cargo test-fastly -- identify_returns`, `cargo test-fastly -- rollback_to_part_1`, `cargo test-fastly -- captured_revocation`, `cargo test-fastly -- global_revoke_from_real_browser`, `cargo test-fastly -- returning_navigation_answered_404`, `cargo test-fastly -- non_browser_capture` and `cargo test-fastly -- golden_` +Expected: the new tests FAIL until `MultiEntryTestProvider` exists, then PASS. A failure after that points at a gap in Tasks 32 to 38, not here. Fix it in the task that owns the code, and note the fix in this task's commit message. + +- [ ] **Step 3: Commit** + +```bash +git add -A crates/ +git commit --signoff -S -m "Prove complete provider EIDs end to end through a registered provider" +``` + ### Task 40: Batch sync never replaces an owned value **Files:** -- Modify: `crates/trusted-server-core/src/ec/batch_sync.rs` (around line 247) +- Modify: `crates/trusted-server-core/src/ec/batch_sync.rs` (`handle_batch_sync` at about line 114, the upsert around line 247) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (`run_batch_sync`, about line 737, passes `&state.identity`) - Test: `ec/batch_sync.rs` +**Interfaces:** + +- Produces: `pub fn handle_batch_sync(kv: &KvIdentityGraph, registry: &PartnerRegistry, rate_limiter: &dyn RateLimiter, identity: &IdentityRuntime, req: Request) -> Result, Report>`. `IdentityRuntime` is `pub` (Task 23), so the adapter can pass it. The handler reads the owned sources from its catalog. + - [ ] **Step 1: Write the failing test** `batch_sync_update_to_owned_source_applies_only_when_absent`: with the source owned by a provider, a batch update applies when the source is absent, and is `Unchanged` (no write) when a value exists. A control update to an unowned source keeps today's behaviour. +`batch_sync_never_touches_an_owners_entry_set`: the owner's record holds two entries. A batch update to the source is `Unchanged`, and the row's `eids` are byte-equal before and after. A batch write that does apply (an absent owned source, or an unowned source) writes a one-entry, one-UID record without `w` or `rv`, as Task 6's `KvPartnerId::new` builds it. + - [ ] **Step 2: Run, see it fail, implement, see it pass** -Run: `cargo test-fastly -- ec::batch_sync` -Expected: FAIL, then PASS. The batch sync handler needs to know the owners. Pass `&ProviderCatalog` (or just the owned source set) through its existing deps. +Run: `cargo test-fastly -- ec::batch_sync` and `cargo test-fastly -- batch_sync` +Expected: FAIL, then PASS. Existing batch sync tests pass an `IdentityRuntime` with an empty catalog and are otherwise unchanged. - [ ] **Step 3: Commit** ```bash -git add -A crates/trusted-server-core/src/ec/batch_sync.rs +git add -A crates/ git commit --signoff -S -m "Keep batch sync from replacing an owned source's value" ``` @@ -7353,7 +9818,7 @@ git commit --signoff -S -m "Keep batch sync from replacing an owned source's val **Files:** - Create: `docs/guide/identity-providers.md` (provider author guide) -- Modify: `docs/guide/configuration.md` (`[[ec.partners]].owner`, per-provider tables), `docs/guide/edge-cookies.md`, `docs/.vitepress` sidebar config (find it with `grep -rn "edge-cookies" docs/.vitepress`), `CHANGELOG.md` +- Modify: `docs/guide/configuration.md` (`[[ec.partners]].owner`, per-provider tables, `revoke_scope`), `docs/guide/edge-cookies.md` (the capture provider checklist, complete provider EIDs, rollback), `docs/.vitepress` sidebar config (find it with `grep -rn "edge-cookies" docs/.vitepress`), `CHANGELOG.md`, `trusted-server.example.toml` - [ ] **Step 1: Write the author guide** @@ -7364,25 +9829,42 @@ Base it on spec 4.2 to 4.6 and 13, using only the fictional `exampleid` provider - declaring inputs and reading them with `?` or `.ok()` - building requests - parsing responses -- returning a `Resolution` +- returning a `Resolution` of `ResolvedEid` entries, each with one or more `ResolvedUid`s: several entries and UIDs per source, UID-level `atype` and `ext`, and EID-level `inserter`, `matcher`, `mm` and `ext` +- the limits of spec 4.5: the browser parser's field rules, plus per-source caps the browser parser does not have (8 entries and 32 UIDs per source, 4,096 bytes per set). Each `ext` may hold 2,048 bytes and each `inserter` or `matcher` 255 bytes, but the field caps do not guarantee the set cap: a set whose legal fields exceed 4,096 bytes is discarded whole, never trimmed. A discarded set leaves the stored set as it is, and a result all of whose sets are discarded counts as malformed. +- that a result replaces an owned source's whole set, while for an unowned source it only fills an absent or expired record +- `with_provenance`: a default applied at output only to an entry that sets none of `inserter`, `matcher` and `mm`. An entry that sets any of them is the vendor's complete statement. A blank provenance `inserter` or `matcher` fails `build()`. +- what vendor `ext` data means for privacy: it is stored only in KV, never in `ts-eids`, and reaches bidders, `x-ts-eids` and the `ts-eids-s` mirror under the same consent gates as the UIDs (spec 9.3). The mirror is readable by every page script. A vendor spec that stores such data says what it holds. - private state -- capture +- capture: revoke-only, so a capture's removals, opt-out and clear are written to KV and its positive IDs are only counted in `capture_unpersisted`; `revoke_scope`; the real-browser gate on capture effects; and that a capture-only provider must list its `sources` and cannot own one - registration with `try_with_identity_provider` -- testing with `ProviderHarness` and `check_conformance` +- testing with `ProviderHarness`, `check_conformance` and `output_eids` - what the coordinator does for you (consent, transport, backoff, KV) Link to the spec for the protocol. - [ ] **Step 2: Config and changelog** -Document `owner`, `mode`, `timeout_ms`, `sources` narrowing, `refresh_after_sec`, `max_calls_per_minute`, `breaker_failures_per_minute` and `cookie_priority`, each with its default and a validation error example. In `CHANGELOG.md` under `### Added`, add the provider framework, ownership, hold mode, passive capture and the breaker. Under `### Changed`, add stricter consent decoding for identity providers. +Document `owner`, `mode`, `timeout_ms`, `after_send_timeout_ms`, `sources` narrowing, `refresh_after_sec`, `max_calls_per_minute`, `breaker_failures_per_minute`, `cookie_priority` and `revoke_scope`, each with its default and a validation error example. Add the commented `[ec.identity.providers.]` capture keys to `trusted-server.example.toml`, as spec 10.1 shows them, with `revoke_scope = "provider"`. + +In `docs/guide/edge-cookies.md`: + +- Add the capture provider checklist from spec 14 as the steps an operator follows before enabling a capture provider: + - each captured source has an `[[ec.partners]]` entry with `bidstream_enabled = true` and the right `openrtb_atype` + - a source moving to an owner comes off pull sync first + - the provider's `sources` list holds only the sources it is the delivery path for + - the live acceptance check through `/_ts/admin/eids` and `/_ts/admin/ec`, including that `removal_lost` stays at zero +- Say what `revoke_scope = "global"` does (a captured opt-out from a real browser runs the consent-withdrawal tombstone), and that a non-browser client's capture never changes the KV row. +- Add a short section on complete provider EIDs: what a multi-entry record looks like in the admin lookup, and that `/identify` still returns one UID per source. +- Add the rollback steps of spec 14. Before rolling back to Part 1, remove every `owner` and vendor `[ec.identity.providers.*]` table, because Part 1's validation rejects them. After the rollback, multi-entry records stay readable but are not served while their writer is unconfigured, and browser ingest may replace them. Each provider's `ps.o`, `ps.oc` and `ps.ov` are kept, and the rest of its `ps` is pruned. Rolling back past Part 1 is unsupported, because a release before it cannot read schema version 2. + +In `CHANGELOG.md` under `### Added`, add the provider framework, ownership, hold mode, revoke-only passive capture (with `revoke_scope` and the capture removal write), complete provider EIDs, the `capture_unpersisted` and `removal_lost` counters and the breaker. Under `### Changed`, add stricter consent decoding for identity providers, and that the Lockr proxy logs only the upstream path. Run the docs and markdown format commands. - [ ] **Step 3: Commit** ```bash -git add docs/ CHANGELOG.md +git add docs/ CHANGELOG.md trusted-server.example.toml git commit --signoff -S -m "Document the identity provider framework for operators and authors" ``` @@ -7392,51 +9874,53 @@ Run `/check-ci`, then `cargo test-fastly-reuse`, `./scripts/test-cli.sh`, the pa - [ ] **Step 5: PR** -Open the PR against `feat/1246-identity-single-write` with `pr-creator`. Title: "Add server-side identity providers". The description lists spec 14 step 2's contents, says no vendor ships in it, and names the follow-up vendor specs (ID5, LiveRamp, Lockr, UID2). +Open the PR against `feat/1246-identity-single-write` with `pr-creator`. Title: "Add server-side identity providers". The description lists spec 14 step 2's contents, says no vendor ships in it, says that capture is revoke-only and that each capture provider is enabled only after its checklist has passed on a live service, and names the follow-up vendor specs (ID5, LiveRamp, Lockr, UID2). --- ## Spec coverage -| Spec section | Tasks | -| --------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| 2.1 G1, G2, G11 (trait, coordinator, ergonomics) | 8 to 11, 20, 21, 39 | -| 2.1 G3 (hold or after-send) | 19, 20, 29, 37 | -| 2.1 G4 (one write) | 25, 26, 27, 29, 30 | -| 2.1 G5 (cookie first, then KV) | 16, 17, 25, 28 | -| 2.1 G6 (ownership) | 33, 36 | -| 2.1 G7 (expiry, refresh) | 14, 15, 20, 36 | -| 2.1 G8 (pull sync on the trait) | 22, 29 | -| 2.1 G9 (HEM slot) | 9 (always empty in v1) | -| 2.1 G10 (no lost EIDs) | 25, 26, 30 | -| 2.1 G12 (removals win) | 15, 18, 25, 27, 38 | -| 3.4 platform facts (status capture, real-browser gate, clock failure) | 14, 23, 29 | -| 4.2 trait and additivity rules | 11, 39 | -| 4.3 descriptor | 8, 32 (capture-path check) | -| 4.4 inputs | 9, 23 (phase-1 captures) | -| 4.5 requests, responses, results | 10, 20 | -| 4.6 example provider and registration | 32, 39 | -| 4.7 eligibility, dispatch, hold timing, breaker, capture | 20, 21, 35, 37, 38 | -| 5.1 writer and stagers; withdrawal exception | 23, 25 | -| 5.2 phases 1 to 4 | 23, 25, 29 | -| 5.3 stagers, KV gap, flush checks | 25, 26 | -| 5.4 `IdentityMutation` | 15 | -| 5.5 orphans, collisions, deletion | 25, 30 (the deletion framework itself is deferred; its tombstone rule is a note for that work) | -| 5.6 removal, marker and cookie rules | 15, 17, 18, 25, 29 | -| 6.1 KV record, rollback rule, admin redaction | 6, 14 | -| 6.2 cookies, sealing, size caps, overflow, outbound strips | 1 to 4, 16, 17, 18, 27 | -| 6.3 read order | 17, 29 | -| 7.1 merge, servability, `/identify` consent | 14, 28, 36 | -| 7.2 ownership semantics, batch sync | 33, 36, 40 | -| 8 errors and outcomes, sticky opt-outs, fingerprint | 13, 20, 34, 36 | -| 9.1 consent | 12, 34 | -| 9.2 outbound requests | 1 to 3, 21 | -| 9.3 logging and redaction | 9, 10, 34, every task's logging | -| 9.4 caching | 25 (cookies only when needed; the privacy rule already applies) | -| 10.1 to 10.3 configuration, validation, registration | 19, 32, 33 | -| 10.4 pull sync on the trait | 22, 29 | -| 11 adapters | 3 (strips on every adapter), 31 | -| 12 vendor examples | Out of scope (spec 2.2). The fictional examples in 39 prove the API | -| 13 testing | each task, 30, 39 | -| 14 rollout | Parts 0, 1, 2 | -| 15 open questions | Not implemented. Track as issues after Part 1 ships | +| Spec section | Tasks | +| --------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| 2.1 G1, G2, G11 (trait, coordinator, ergonomics) | 8 to 11, 20, 21, 39 | +| 2.1 G3 (hold or after-send) | 19, 20, 29, 37 | +| 2.1 G4 (one write; withdrawal, removal-retry and capture-removal exceptions) | 18, 25, 26, 27, 29, 30, 38, 39b | +| 2.1 G5 (KV, mirrored in `ts-eids-s`) | 17, 26, 28 | +| 2.1 G6 (ownership) | 33, 36 | +| 2.1 G7 (expiry, refresh) | 14, 15, 20, 36 | +| 2.1 G8 (pull sync on the trait) | 21 (hourly rotation), 22, 29 | +| 2.1 G9 (HEM slot) | 9 (always empty in v1) | +| 2.1 G10 (no lost EIDs; registered-first trim) | 25, 26, 30 (including an `ortb2`-only EID) | +| 2.1 G12 (removals win) | 15, 18, 27, 28 (denials), 29 (removal-only retry), 36, 38 | +| 2.1 G13 (complete provider EIDs, nothing that predates the spec changes) | 6, 7, 8, 9, 10, 13, 14, 15, 17, 22, 24, 25, 26, 28, 30, 36, 37, 39, 39b, 40 | +| 3.4 platform facts (status capture, real-browser gate, clock failure) | 14, 23, 29, 38 | +| 4.1 file layout | 5 and each task's Files list | +| 4.2 trait and additivity rules | 11, 39 | +| 4.3 descriptor, provenance default | 8, 28, 32 (capture-path check), 39 (`output_eids`) | +| 4.4 inputs, per-kind availability | 9, 10 (`CaptureInput`), 23 (phase-1 captures) | +| 4.5 requests, responses, results, entry-set limits, discarded sets | 10, 20, 39 | +| 4.6 example provider and registration | 32, 39, 39b | +| 4.7 eligibility, dispatch, hold timing, breaker, revoke-only capture, real-browser gate | 20, 21, 35, 37, 38 | +| 5.1 writer and stagers, status rule, withdrawal exception and its after-send run | 23, 25, 27, 29, 38, 39b | +| 5.2 phases 1 to 4, phase 1 read on every writer, removal-only retry | 23, 25, 29 | +| 5.3 stagers, the capture removal write, the mirror payload, registered-first trim | 18, 25, 26, 38 | +| 5.4 `IdentityMutation`, whole-set writes, `iv` and `ov` at apply, `ps` pruning | 15 | +| 5.5 orphans, collisions, deletion | 25, 30 (the deletion framework itself is deferred; its tombstone rule is a note for that work) | +| 5.6 rules that keep removals winning | 15, 18, 26, 29, 38 | +| 6.1 KV schema version 2, version 1 read, `ps`, admin redaction | 6, 13, 14, 39b | +| 6.2 the `ts-eids-s` mirror, its payload, cap and strips | 1 to 4, 17, 26 | +| 6.3 read order | 26, 29 | +| 7.1 merge, entry-set output, provider denials, servability, `/identify` shape | 14, 24, 28, 36 | +| 7.2 ownership semantics, whole-set replacement, batch sync | 33, 36, 40 | +| 8 errors and outcomes, sticky opt-outs, re-grants, revoke scope, plain fingerprint | 15, 20, 34, 36, 38 | +| 9.1 consent, header-only GPP | 12, 34 | +| 9.2 outbound requests | 1 to 3, 21 | +| 9.3 logging, redaction, vendor `ext`, Lockr path-only log | 9, 10 (`Resolution` `Debug`), 23 (`RequestIdentity` `Debug`), 34, 38, 41, every task's logging | +| 9.4 caching | 4, 25 (no identity cookie on HTML responses) | +| 10.1 to 10.3 configuration, removed keys, validation, registration | 4, 19, 22 (catalog validator, removed partner keys), 32, 33 | +| 10.4 pull sync on the trait, pull-complete marker removed | 21 (hourly rotation), 22, 29 | +| 11 adapters | 3 (strips on every adapter), 36 (admin route on every adapter); parity gate in 22 and 34 | +| 12 vendor examples | Out of scope (spec 2.2). The fictional examples in 32, 39 and 39b prove the API | +| 13 testing | each task, 30, 39, 39b | +| 14 rollout, rollback, capture checklist | Parts 1 and 2; 31 (metrics, rollback note), 39b, 41 (checklist, rollback) | +| 15 open questions | Not implemented. Track as issues after Part 1 ships | diff --git a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md index e4df65fdac..4074a4ae3b 100644 --- a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md +++ b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md @@ -1,10 +1,17 @@ # Server-side identity providers -Status: Draft, revision 12. Revised after review rounds 1 to 11. Section 17 -maps each finding to its fix. - -Issue: none filed yet. Related: [#1245](https://github.com/IABTechLab/trusted-server/issues/1245) -(LiveRamp server-to-server), [#90](https://github.com/IABTechLab/trusted-server/issues/90) +Status: Draft, revision 15. Revised after review rounds 1 to 14. Revision 14 +made `ts-eids-s` a client-only cookie, at the user's request. Revision 15 +removes the `ts-eids-r` removal cookies, runs providers only on writer +navigations, makes capture revoke-only and moves KV to schema version 2, +accepting breaking changes. Section 16 lists the decisions; the +[review log](2026-10-20-server-side-identity-providers-review-log.md) maps +each earlier finding to its fix. + +Issue: none filed yet. Related: +[#1245](https://github.com/IABTechLab/trusted-server/issues/1245) +(LiveRamp server-to-server), +[#90](https://github.com/IABTechLab/trusted-server/issues/90) and [#353](https://github.com/IABTechLab/trusted-server/issues/353) (ID5), [#1246](https://github.com/IABTechLab/trusted-server/issues/1246) (Lockr IDs to KV), [#777](https://github.com/IABTechLab/trusted-server/issues/777) @@ -33,6 +40,11 @@ Terms used throughout: - **EID / UID**: an OpenRTB extended identifier, `{source, uids[]}`, and one entry of its `uids`, `{id, atype?, ext?}` (OpenRTB 2.6 §3.2.27, §3.2.28). +- **EID-level fields**: an EID's `inserter`, `matcher`, `mm` and its own + `ext`, as opposed to a UID's `atype` and `ext`. +- **Entry set**: every EID entry stored for one source, written by one writer + and replaced or removed whole (section 6.1). A provider may return several + entries per source, each with several UIDs. - **Identity provider**: a vendor that turns request signals into an EID, such as ID5, LiveRamp ATS, Lockr AIM or UID2. - **Source**: the EID `source` domain a provider's IDs carry, such as @@ -48,20 +60,23 @@ Terms used throughout: `/_ts/page-bids`, and integration proxy calls. - **Writer request**: the one request in a page load allowed to write KV (section 5.1). -- **Stager**: every other request. It reads cookies and KV and saves changes - into `ts-eids-s` for the next writer request to write. -- **KV gap**: the registered sources a browser sent to `/auction` that the - visitor's KV row does not have yet. +- **Stager**: every other request. It reads cookies and KV, runs no + resolving provider and persists no positive ID. Its only KV write is the + capture removal write, for a revocation a proxied call carries (section + 5.3). - **Hold mode / after-send mode**: whether a provider is called before the auction, with the auction waiting for it, or after the response has been sent. - **Orphan recovery**: what TS does when a visitor's EC cookie has no KV row (`ec/finalize.rs:282`). - **`ts-eids`**: the existing first-party cookie the Prebid shim writes with - the browser's EIDs. Unchanged by this spec. -- **`ts-eids-s`**: a new server-owned cookie, named `__Host-ts-eids-s`, that - caches provider-collected IDs and holds staged changes. Defined in section - 6.2. + the browser's EIDs. Unchanged by this spec, except that its trim keeps + registered sources first (section 5.3). +- **`ts-eids-s`**: a new client-only cookie, named `__Host-ts-eids-s`, that + the shim writes from the servable server set TS returns in `/auction` and + `/_ts/page-bids` responses. It mirrors server IDs for page scripts. The + server never opens, parses or trusts it, and the shim never copies it + into `ts-eids` or the `/auction` body. Defined in section 6.2. ## 1. Problem statement @@ -81,7 +96,9 @@ avoids the browser request. This spec builds that server-side path. In v1 it helps: - providers that need no hashed email, such as ID5 -- providers whose traffic TS already proxies, such as Lockr +- providers whose traffic TS already proxies, such as Lockr, for removals + and opt-outs. A positive capture on a proxied API call is not persisted + (section 2.3) - generic pull partners - any owning provider that can refresh an ID a browser module obtained earlier, so the ID survives when the module later fails in that browser @@ -160,12 +177,18 @@ can be written, and each extra write is another CAS race between tabs. - **G4.** At most one browser-driven KV write per page load, and a write that fails is not retried. This is strict. The only page-load exceptions are a consent withdrawal's tombstone (section 5.1), which also covers a - capture provider's global revoke (section 8), and a removal batch from a - real browser that the removal cookies cannot hold (section 6.2). Writers outside page loads - keep their own writes: batch sync, the admin endpoints and the deferred - deletion framework (sections 2.2, 5.5). -- **G5.** Provider-collected IDs are stored in both a first-party cookie - (`ts-eids-s`) and KV, and the cookie is read first. + capture provider's global revoke (section 8), and the two removal writes: + the capture removal write a proxied revocation from a real browser makes + (section 5.3), and the removal-only retry of a failed flush (section 5.2, + phase 4). All are rare and removal-only. Writers outside page loads keep + their own + writes: batch sync, the admin endpoints and the deferred deletion + framework (sections 2.2, 5.5). +- **G5.** Provider-collected IDs are stored in KV and mirrored in a + first-party cookie (`ts-eids-s`) that page scripts read. KV is the only + server-side source of provider IDs: eligibility, the auction merge, + `/identify` and servability read KV, never `ts-eids-s`. Section 15 asks + whether this meets the product rule that the cookie is read first. - **G6.** Each source has at most one owning provider, configured by the operator. A valid ID the owner wrote wins in the auction over the browser's. @@ -176,22 +199,37 @@ can be written, and each extra write is another CAS race between tabs. behaviour that does change. - **G9.** The trait can model a provider that needs a HEM, even though v1 has no way to obtain one. -- **G10.** No browser EID that reaches KV today is lost. Everything today's - `/auction` ingest writes still reaches KV, through the next writer request. - What today's `/_ts/page-bids` ingest writes comes from `ts-eids`, which the - next navigation ingests; only an SPA session longer than that cookie's - one-day lifetime can lose such an ID. A body-only ID beyond the `ts-eids` - trim is lost only when KV-gap entries alone exceed the `ts-eids-s` cap - and the browser sends no further `/auction` before the next navigation. +- **G10.** No browser EID that reaches KV today is lost, within one bound. + Browser ingest runs only on the writer request, from `ts-eids`. The shim + trims `ts-eids` keeping registered sources first (section 5.3), so the + next writer request ingests every registered source today's `/auction` + ingest writes from the body. A body-only registered ID is lost only when + the registered sources alone exceed the 3,072-byte trim. What today's + `/_ts/page-bids` ingest writes comes from `ts-eids` too; only an SPA + session longer than that cookie's one-day lifetime can lose such an ID. - **G11.** Writing a provider is easy, and extending the trait does not break one. A simple provider is one struct, one descriptor and two short methods (section 4.2). -- **G12.** Removals win. An ID removed from KV (opt-out, invalidation, - withdrawal, deletion) stops reaching bidders within a bounded time, even - from the cookie. A removal staged by a stager stops that client's - bidstream at once and reaches KV at the client's next writer navigation; a - client that never makes one, such as a script that fails the real-browser - check (section 4.7), keeps it in `ts-eids-r` until it does. +- **G12.** Removals win. Every removal (opt-out, invalidation, withdrawal, + deletion) is written to KV on the request that carries it, and TS serves + only from KV, so the ID stops reaching bidders through TS at once. A + removed provider ID also stops reaching them as a browser value: the + removing provider's `ps.iv`, `ps.ov` and `ps.o` filter browser candidates + (section 7.1). The `ts-eids-s` mirror drops it at the next `/auction` or + `/_ts/page-bids` response. The one bound: a removal write that fails all + its attempts is lost (section 2.3). +- **G13.** A provider can return complete EIDs, with the same fidelity the + browser's `/auction` body has: several EID entries per source, several + UIDs per entry, UID-level `atype` and `ext`, and the EID-level fields. TS + stores the entry set whole in KV, mirrors it whole in `ts-eids-s` when it + fits, and sends it to bidders whole (sections 4.5, 6.1, 6.2, 7.1). This is + additive in behaviour: G13 itself changes none of the paths that predate + this spec. Browser ingest, `ts-eids`, batch sync, generic pull and + `/identify` keep the behaviour the rest of this spec gives them (sections + 5, 6.2, 7.2, 10.4), and every record they write is one entry with one + UID and no EID-level fields. The storage shape changes for every record + (schema version 2, section 6.1); what bidders receive for these records + does not. ### 2.2 Non-goals @@ -210,6 +248,58 @@ can be written, and each extra write is another CAS race between tabs. - **Batch sync and admin writes.** They keep their own writes. Section 7.2 changes only how batch sync treats an owned source. - **Removing `ts-eids` or `x-ts-eids`.** Both stay. +- **Storing the browser's complete EID set.** Browser ingest keeps today's + rules: registered sources only, one UID per source picked as + `collect_eid_updates` picks it today (`ec/prebid_eids.rs:146-151`: for a + repeated source, the first valid UID of its last entry, with the sharedId + cookie applied last), no EID-level fields, fill-missing + (`ec/kv.rs:145-155`). + The browser's complete set still reaches bidders from the `/auction` body, + as the EID fidelity work defines. G13 applies only to IDs a provider + writes. + +- **Unregistered sources for providers.** A provider still claims only + sources with an `[[ec.partners]]` entry (section 10.2). Full fidelity + changes what a provider may return for a source, not which sources. + +### 2.3 Accepted tradeoffs + +Revisions 14 and 15 make `ts-eids-s` client-only, keep every positive ID +off stagers and drop every server-owned identity cookie. The design accepts +these costs for it: + +- **Providers run only on writer navigations.** `/auction` and + `/_ts/page-bids` call no provider; they serve what KV holds. On a site + whose auctions all go through `/auction`, hold-mode providers run after + send on the navigation, and their answers reach the next auction from KV + (section 4.7, step 7). +- **Positive captures are not used in v1.** Capture is revoke-only: a + proxied call can remove IDs, opt a provider out and clear that opt-out, + and nothing else. The browser path (Prebid `ortb2` into `ts-eids`) still + carries to KV the one UID browser ingest picks for each registered source + (section 2.2). Full-fidelity sets from capture wait for open question 4 + (sections 4.7, + 12.2, 15). +- **A failed removal write can be lost.** A capture removal write is tried + before send with the withdrawal path's CAS attempts and once more after + send (section 5.3). A writer whose flush fails retries its removals alone + with the same CAS attempts (section 5.2, phase 4). When every attempt + fails, the removal is logged at error and counted in `removal_lost`, and + nothing keeps it. Removals are rare, and a vendor's own SDK holds a + revocation too. +- **A failed flush has no backup.** Browser ingest is re-derived from + `ts-eids`, and provider results are resolved again by a later writer + (section 5.2, phase 4). +- **Every returning writer navigation reads KV once.** No navigation is + answered from a cookie with zero reads (section 5.2). +- **Server-only IDs are readable by page scripts.** UID2 advertising tokens + and LiveRamp envelopes are in `ts-eids-s`, which is not `HttpOnly`. + `x-ts-eids` already exposes them (section 9.3). +- **No provider runs on a new visitor's first navigation.** A visitor who + leaves after one page never gets a provider ID. Section 15 asks whether + hold-mode providers should feed the create. +- **Rollback past rollout step 1 is unsupported.** Step 1 writes KV schema + version 2, which older builds reject (sections 6.1, 14). ## 3. Background @@ -238,6 +328,9 @@ send. The result is consent-gated and reported in `x-ts-eids` Browser ingest is fill-missing: a different stored value is never replaced, because a browser value carries no freshness signal (`ec/kv.rs:146-154`). +A KV record holds one UID per source and no EID-level fields +(`ec/kv_types.rs:102`), and a KV value goes out as one entry with the +partner's `openrtb_atype` (`ec/eids.rs:64-76`). ### 3.3 The existing provider pattern @@ -255,7 +348,8 @@ follows the same split. (`ec/mod.rs:407-455`). Finalize refuses a generated cookie without a backing row (`ec/finalize.rs:145-149`). This spec keeps that. - **After-send work starts after the last byte.** `send_edgezero_response` - streams the whole body before returning (`adapter-fastly/src/main.rs:699-702`), + streams the whole body before returning + (`adapter-fastly/src/main.rs:699-702`), and pull sync runs after that (`main.rs:510-517`). The response has been moved into the send by then, so after-send code cannot see its status unless it is captured first (`main.rs:512-517`). @@ -281,7 +375,8 @@ follows the same split. publisher fallback may rotate (`adapter-fastly/src/app.rs:885-890`). - **Today's navigation preload finds orphans.** Every returning GET navigation preloads the row unless a valid pull-complete marker lets it - skip (`publisher.rs:3110-3117`, `:4819-4831`). + skip (`publisher.rs:3110-3117`, `:4819-4831`). This spec removes the + marker (section 10.4), so every returning navigation preloads. - **Withdrawal tombstones existing rows only,** with a CAS write of up to five attempts and then a completion-marker write, per withdrawn ID (`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:490`, `:509-525`). @@ -298,10 +393,12 @@ follows the same split. everyone behind one address shares one budget. - **The shim's `ts-eids` is small.** It is trimmed to 3,072 encoded bytes, lives one day and has no EID-level `ext` (EID fidelity spec §4.3). The - server accepts up to 8 KB (`ec/prebid_eids.rs:24`). + server accepts up to 8 KB (`ec/prebid_eids.rs:24`). This spec changes + only the trim order: registered sources are kept first (section 5.3). - **Prebid Server receives the browser's cookies.** The Prebid transport copies the inbound `Cookie` header into every Prebid Server call and strips - only consent cookies, only in `OpenrtbOnly` mode (`integrations/prebid.rs:2091-2108`, + only consent cookies, only in `OpenrtbOnly` mode + (`integrations/prebid.rs:2091-2108`, `consent_config.rs:137-150`). The publisher proxy forwards the `Cookie` header to the origin unchanged (`publisher.rs:3124-3147`). @@ -312,20 +409,18 @@ follows the same split. ```mermaid flowchart TD subgraph Writer["Writer request: a top-level navigation"] - A[Read ts-ec, consent, ts-eids, ts-eids-s] --> B{Cookies answer
everything?} - B -- no --> C[KV snapshot read] - B -- yes --> D + A[Read ts-ec, consent, ts-eids] --> C[Returning EC:
KV snapshot read] C --> D[Pick eligible providers] D --> E[Hold-mode providers, if the
navigation runs an auction] - E --> F[Finalize: queue changes,
set cookies only if needed] + E --> F[Finalize: queue changes] F --> G[Send response] - G --> H[Returning EC: re-read live row] + G --> H[After-send provider due:
re-read live row] H --> I[After-send providers] I --> J[Apply IdentityMutation:
one conditional write] end subgraph Stagers["Stagers: /auction, /_ts/page-bids, integration proxy"] - K[Read cookies, and KV where
the route reads it today] --> L[Hold-mode providers,
passive capture] - L --> M[Save results and the KV gap
into ts-eids-s; never write KV] + K[Read KV where the
route reads it today] --> L[No providers;
revoke-only capture] + L --> M[Mirror payload in the JSON response;
a captured revocation writes KV] end ``` @@ -337,25 +432,35 @@ The design has four parts: 3. **`IdentityMutation`** (section 5.4). A request-scoped accumulator. Every KV change in the writer request is added to it, and it is applied with one write. -4. **`ts-eids-s` cookie** (section 6.2). The cookie-first copy of - provider-collected IDs and the place stagers save changes. +4. **The mirror** (section 6.2). `ts-eids-s` is a client-only cookie the + shim writes from the servable server set, for page scripts. The server + sets no identity cookie of its own beyond `ts-ec`. New code lives in `crates/trusted-server-core/src/ec/identity/`: -| File | Contents | -| ----------------- | ---------------------------------------------------------------------- | -| `mod.rs` | Module docs, re-exports | -| `provider.rs` | The `IdentityProvider` and `IdentityProviderConfig` traits | -| `descriptor.rs` | `ProviderDescriptor`, `Resolves`, `Captures`, `Input`, `ConsentPolicy` | -| `input.rs` | `ResolveInput`, `CaptureInput`, `StoredId`, `HashedEmail`, `UnixTime` | -| `exchange.rs` | `ProviderRequest`, `ProviderResponse` | -| `resolution.rs` | `Resolution`, `ResolvedId`, `IdentityProviderError` | -| `coordinator.rs` | Eligibility, dispatch, circuit breaker, result-to-change map | -| `mutation.rs` | `IdentityMutation` and its merge into a live `KvEntry` | -| `cookie.rs` | `ts-eids-s` and `ts-eids-r`: seal, open, encode, decode, size caps | -| `generic_pull.rs` | `GenericPullProvider`, today's pull sync on the trait | -| `config.rs` | `[ec.identity]` settings and ownership validation | -| `test_support.rs` | `ProviderHarness`, a fake HTTP client and a write-counting KV store | +| File | Contents | +| ----------------- | ------------------------------------------------------------------------------- | +| `mod.rs` | Module docs, re-exports | +| `ids.rs` | `IdentityProviderId`, `SourceDomain`, `UnixTime` | +| `provider.rs` | The `IdentityProvider` and `IdentityProviderConfig` traits | +| `descriptor.rs` | `ProviderDescriptor`, `Resolves`, `Captures`, `Input`, `ConsentPolicy` | +| `input.rs` | `ResolveInput`, `CaptureInput`, `StoredId`, `HashedEmail` | +| `exchange.rs` | `ProviderRequest`, `ProviderResponse` | +| `resolution.rs` | `Resolution`, `ResolvedEid`, `ResolvedUid`, `IdentityProviderError`, entry sets | +| `policy.rs` | Consent policy evaluation and the consent fingerprint | +| `revision.rs` | Revisions and the UID hashes of `ps.iv` and `ps.ov` | +| `catalog.rs` | The configured providers, their narrowed sources and source ownership | +| `servable.rs` | The servability check of section 7.1 | +| `coordinator.rs` | Eligibility, dispatch, circuit breaker, result-to-change map | +| `mutation.rs` | `IdentityMutation` and its merge into a live `KvEntry` | +| `mirror.rs` | The `ts-eids-s` mirror payload and its size cap | +| `request.rs` | The writer decision and the request's identity state | +| `removal.rs` | The capture removal write of section 5.3 | +| `runtime.rs` | `IdentityRuntime`: the catalog and keys an adapter builds once | +| `flush.rs` | Phase 4, the single write after send | +| `generic_pull.rs` | `GenericPullProvider`, today's pull sync on the trait | +| `config.rs` | `[ec.identity]` settings and ownership validation | +| `test_support.rs` | `ProviderHarness`, a fake HTTP client and a write-counting KV store | ### 4.2 The trait @@ -377,10 +482,8 @@ The trait follows five rules: - **Pure.** `request`, `response` and `capture` have no side effects and no hidden inputs. Randomness comes only from `ResolveInput::random_bytes`, and anything `response` needs from `request` travels in the call context - (section 4.5). The coordinator may call `request` twice in one request, - first as a dry run (section 5.2). `input.is_dry_run()` lets a provider skip - expensive work in the dry run. A provider with private state gets no dry - run. + (section 4.5). The coordinator calls `request` at most once per + dispatch. - **Safe defaults.** The default consent policy is the strictest one, a provider can read only what it declared, and a capability declared but not implemented fails loudly in tests (section 13). @@ -504,14 +607,19 @@ the descriptor. Providers never read them. **Conventions for the listings below.** Every builder method and getter is `#[must_use]`. `Resolves`, `Captures` and `EidProvenance` implement `Default`. `Input`, `Access`, `ResolveKind` and `UsOptOut` derive -`Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display`. +`Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, +derive_more::Display`. `When` derives `Debug, Clone, Copy, PartialEq, Eq` only, because `Duration` -has no `Display`. `ProviderRequest`, `StoredId` and `HashedEmail` derive -`Debug, Clone`; `HashedEmail`'s `Debug` prints a redacted marker. +has no `Display`. `ProviderRequest` derives `Debug, Clone`. `StoredId` and +`HashedEmail` derive `Clone` and have redacting `Debug` implementations +(section 9.3). `StateChange` derives `Debug, PartialEq, Eq`. `ProviderDescriptor`, `Resolves`, `Captures`, `ConsentPolicy` and `EidProvenance` derive -`Debug, Clone, PartialEq, Eq`, so tests can compare descriptors. `UnixTime` derives -`Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord`. `IdentityProviderId` +`Debug, Clone, PartialEq, Eq`, so tests can compare descriptors. `UnixTime` +derives +`Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord`. `ResolvedEid` and +`ResolvedUid` derive `Clone` and have redacting `Debug` implementations +(section 9.3). `IdentityProviderId` and `SourceDomain` derive `Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display` and implement `FromStr` (with `type Err = Report`), `Serialize` and `Deserialize`. @@ -566,16 +674,21 @@ impl ProviderDescriptorBuilder { /// Marks the IDs of one declared source as expiring, for a provider /// whose sources differ. pub fn with_expiring_source(self, source: &SourceDomain) -> Self; - /// EID-level fields for a declared source. The coordinator adds them to - /// every server-side value of that source this provider wrote. No KV - /// field stores them. + /// Default EID-level fields for a declared source. At output, the + /// coordinator applies them to a stored entry of that source, written + /// by this provider, only when the entry sets none of `inserter`, + /// `matcher` and `mm`. An entry that sets any of them is the vendor's + /// complete statement, so vendor and config provenance never mix in one + /// entry. The defaults are not stored, so a config change applies at + /// once. pub fn with_provenance(self, source: &SourceDomain, provenance: EidProvenance) -> Self; /// # Errors /// /// Returns `Config` when the descriptor has no source, no capability, a /// resolver with no endpoint or host, an endpoint that is not an `https` - /// URL, a host that is not a bare hostname, or provenance or an - /// expiring mark for a source it does not declare. + /// URL, a host that is not a bare hostname, provenance or an expiring + /// mark for a source it does not declare, or a provenance `inserter` or + /// `matcher` that is blank or over 255 bytes, or an `mm` below 0. pub fn build(self) -> Result>; } @@ -594,8 +707,8 @@ impl Resolves { pub fn with_wants(self, inputs: impl IntoIterator) -> Self; /// The kinds this provider handles. Default: `Initial` and `Refresh`. pub fn with_kinds(self, kinds: impl IntoIterator) -> Self; - /// This provider keeps private state, so it runs only on the writer - /// request, where state can be stored (section 5.3). + /// This provider keeps private state in KV (`ps.st`, section 6.1). It + /// can then read [`ResolveInput::state`] and return a state change. pub fn with_private_state(self) -> Self; } @@ -605,16 +718,13 @@ impl Captures { pub fn with_path(self, method: Method, path: &str) -> Self; /// Offers proxied responses whose path starts with `prefix`. Repeatable. pub fn with_path_prefix(self, prefix: &str) -> Self; - /// JSON pointers to consent strings in the proxied request's JSON body - /// (TCF, GPP or US Privacy, detected by format). The coordinator - /// extracts and decodes only these fields, and gates capture by the - /// stricter of the cookie consent and the body consent. The provider - /// never sees the body. - pub fn with_request_consent(self, pointers: impl IntoIterator) -> Self; } -/// EID-level fields (OpenRTB 2.6 §3.2.27) that bidders read, such as who -/// inserted an ID and how it was matched. +/// Default EID-level fields (OpenRTB 2.6 §3.2.27) that bidders read, such as +/// who inserted an ID and how it was matched. A provider whose vendor +/// returns these per response sets them on each `ResolvedEid` instead +/// (section 4.5). An unset field stays unset: OpenRTB 2.6 reads a missing +/// `matcher` as `source`, so a default never needs a blank value. impl EidProvenance { pub fn new() -> Self; pub fn with_inserter(self, inserter: &str) -> Self; @@ -688,7 +798,8 @@ accessor with the same visibility. Only `generic_pull.rs` uses them. Every capture path and prefix must lie under the registering integration's own proxy routes (`IntegrationProxy::routes`, `integrations/registry.rs:316`). -`IntegrationRegistration::build()` cannot fail (`integrations/registry.rs:720-722`), +`IntegrationRegistration::build()` cannot fail +(`integrations/registry.rs:720-722`), so the check runs in `IntegrationRegistry::with_plan` (`integrations/registry.rs:842-865`), which already returns an error and merges every registration. The error names the integration. @@ -728,7 +839,8 @@ Accessors fail in two ways: - **`UndeclaredInput { access }`**: the descriptor never declared the input, the endpoint or private state. That is a provider bug. `.ok()` would hide it, so every accessor also records an undeclared read on the - `ResolveInput`. After `request` or `response` returns, the coordinator logs any recorded + `ResolveInput`. After `request` or `response` returns, the coordinator + logs any recorded read as `Config`, and the harness fails the test (section 13), whatever the provider did with the error. - **`InputUnavailable { input }`**: the input was declared but is missing @@ -758,11 +870,6 @@ impl ResolveInput<'_> { /// starts fresh (section 8). Without `with_private_state()` it is /// `UndeclaredInput`. pub fn state(&self) -> Result, Report>; - /// True during the dry run of section 5.2. The request is discarded, so - /// a provider may skip expensive work and return `Some` early. Always - /// false for a provider declared `with_private_state()`, which gets no - /// dry run. - pub fn is_dry_run(&self) -> bool; /// The endpoint declared with `Resolves::with_endpoint`. pub fn endpoint(&self) -> Result<&Url, Report>; pub fn consent(&self) -> &ProviderConsent; @@ -787,7 +894,7 @@ impl CaptureInput<'_> { pub fn primary_source(&self) -> &SourceDomain; pub fn source_named(&self, name: &str) -> Option<&SourceDomain>; pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId>; - /// The stricter of the cookie consent and the request-body consent. + /// The request's consent, from its cookies and headers. pub fn consent(&self) -> &ProviderConsent; pub fn publisher_domain(&self) -> &str; pub fn now(&self) -> UnixTime; @@ -803,12 +910,22 @@ pub enum ResolveKind { Refresh, } +/// A source's stored record: its entry set and the record's metadata. impl StoredId { + /// Every stored entry for the source, in stored order, with its UIDs and + /// EID-level fields. A record that browser ingest, generic pull, batch + /// sync or a schema version 1 row wrote is one entry with one UID. + pub fn eids(&self) -> &[Eid]; + /// The first UID of the first entry. What a single-UID refresh, such as + /// LiveRamp's, sends back. pub fn uid(&self) -> &str; + /// The `atype` of that first UID. pub fn atype(&self) -> Option; - /// UID-level ext, as bidders read it in `uids[].ext`. + /// The UID-level ext of that first UID, as bidders read it in + /// `uids[].ext`. pub fn ext(&self) -> Option<&serde_json::Value>; - /// The provider that wrote it. `None` for browser ingest and legacy rows. + /// The provider that wrote it. `None` for browser ingest and version 1 + /// records. pub fn writer(&self) -> Option<&IdentityProviderId>; pub fn acquired_at(&self) -> Option; pub fn expires_at(&self) -> Option; @@ -861,13 +978,12 @@ page URL into `EcContext`, because after-send code no longer has the request - **`ClientIp`**: `services.client_info().client_ip`, the platform's address (as `publisher.rs:5574-5578` does), never a request header and never the normalized EC IP. -- **`PageUrl`**: the navigation URL; on `/auction`, the Referer as the - auction sanitizer reduces it (`auction/formats.rs:106-155`); on - `/_ts/page-bids`, the request origin plus the normalized `path` parameter - (`publisher.rs:6978-6989`). Always without query string or fragment. A - path can still carry identifiers on some sites, so `PageUrl` is a wanted - input for most providers, never needed. -- **`Region`**: from the request's `GeoInfo` (`adapter-fastly/src/app.rs:423-432`). +- **`PageUrl`**: the navigation URL, the only request providers run on + (section 4.7, step 7). Always without query string or fragment. A path + can still carry identifiers on some sites, so `PageUrl` is a wanted input + for most providers, never needed. +- **`Region`**: from the request's `GeoInfo` + (`adapter-fastly/src/app.rs:423-432`). `GeoInfo.region` is a free-form `Option` (`platform/types.rs:32`), so the coordinator normalizes it to the ISO 3166-2 subdivision and upper case. City, coordinates, metro and ASN are never passed. @@ -938,11 +1054,15 @@ impl ProviderResponse { } impl Resolution { - /// One or more IDs, at most one per source: KV holds one UID per - /// source. The coordinator keeps the first ID for a source, discards the - /// rest with a log, and the harness fails the test. An empty set becomes - /// `no_match`, and `is_no_match()` then returns true. - pub fn found(ids: impl IntoIterator) -> Self; + /// One or more EID entries. A source may appear in several entries, in + /// the order bidders should see them. The entries of one source form its + /// new entry set. For an owned source it replaces the stored set whole + /// (section 6.1); for an unowned source it only fills an absent or + /// expired record (`FillMissing`, section 7.2) and is never replaced by + /// its writer. A source the result does not name is left as it is. An + /// empty result becomes `no_match`, and `is_no_match()` then returns + /// true. + pub fn found(eids: impl IntoIterator) -> Self; /// The provider recognized no one, or a capture held nothing. pub fn no_match() -> Self; /// The stored ID for `source` is no longer valid. @@ -968,7 +1088,7 @@ impl Resolution { pub fn replacing_state(self, state: &impl Serialize) -> Self; pub fn clearing_state(self) -> Self; - pub fn ids(&self) -> &[ResolvedId]; + pub fn eids(&self) -> &[ResolvedEid]; pub fn invalidated(&self) -> &[SourceDomain]; pub fn is_no_match(&self) -> bool; pub fn is_opted_out(&self) -> bool; @@ -983,12 +1103,23 @@ pub enum StateChange<'a> { Clear, } -impl ResolvedId { - pub fn new(source: &SourceDomain, uid: impl Into) -> Self; - pub fn with_atype(self, atype: i32) -> Self; - /// UID-level ext. Over 256 serialized bytes it is dropped and the ID - /// kept. +/// One OpenRTB EID entry a provider returns: a source, its UIDs and its +/// EID-level fields. +impl ResolvedEid { + /// An entry with one UID, the common case. + pub fn new(source: &SourceDomain, uid: ResolvedUid) -> Self; + /// Adds another UID to this entry. Repeatable. + pub fn with_uid(self, uid: ResolvedUid) -> Self; + /// EID-level fields. An entry that sets any of `inserter`, `matcher` + /// and `mm` takes no `with_provenance` default (section 4.3). + pub fn with_inserter(self, inserter: &str) -> Self; + pub fn with_matcher(self, matcher: &str) -> Self; + pub fn with_mm(self, mm: i32) -> Self; + /// EID-level ext. Over 2,048 bytes as compact JSON it is dropped and + /// the entry kept. pub fn with_ext(self, ext: serde_json::Value) -> Self; + /// Timing applies to the source's whole entry set. When several entries + /// of one source set it, the earliest time wins (below). pub fn with_expires_in(self, delay: Duration) -> Self; pub fn with_expires_at(self, at: UnixTime) -> Self; pub fn with_refresh_in(self, delay: Duration) -> Self; @@ -1000,14 +1131,29 @@ impl ResolvedId { pub fn with_refresh_until(self, at: UnixTime) -> Self; pub fn source(&self) -> &SourceDomain; - pub fn uid(&self) -> &str; - pub fn atype(&self) -> Option; + pub fn uids(&self) -> &[ResolvedUid]; + pub fn inserter(&self) -> Option<&str>; + pub fn matcher(&self) -> Option<&str>; + pub fn mm(&self) -> Option; pub fn ext(&self) -> Option<&serde_json::Value>; pub fn expiry(&self) -> Option; pub fn refresh(&self) -> Option; pub fn refresh_until(&self) -> Option; } +/// One UID within a [`ResolvedEid`]. +impl ResolvedUid { + pub fn new(id: impl Into) -> Self; + pub fn with_atype(self, atype: i32) -> Self; + /// UID-level ext. Over 2,048 bytes as compact JSON it is dropped and the + /// UID kept. + pub fn with_ext(self, ext: serde_json::Value) -> Self; + + pub fn id(&self) -> &str; + pub fn atype(&self) -> Option; + pub fn ext(&self) -> Option<&serde_json::Value>; +} + /// A time as a provider gave it: relative to `now`, or absolute. pub enum When { In(Duration), @@ -1058,14 +1204,53 @@ Relative times (`with_expires_in`, `with_refresh_in`, `with_retry_in`) are resolved against the request's `now` by the coordinator, so providers rarely need the clock. -The coordinator discards a `ResolvedId` whose source the provider does not -claim (or that the operator narrowed away), or whose `uid` is empty or over -`MAX_UID_LENGTH` (`ec/kv_types.rs:35`). It logs each discard with the -provider ID and source only. An ID of an expiring source that carries no -expiry keeps the stored `exp` when its UID is unchanged, and is discarded -otherwise, because an owned expiring ID without `exp` is never served -(section 6.1). That fallback is for a vendor that omits an expiry at -runtime; the harness fails a fixture that does. +**From entries to an entry set.** The coordinator groups a result's entries +by source, keeping their order, and checks each group with the browser +parser's field rules (EID fidelity spec §4.4), then with per-source caps +the browser parser does not have: 8 entries, 32 UIDs across them, and +`MAX_ENTRY_SET_BYTES`. The limits are counted before the result joins the +mutation, so a stateful result whose set is discarded is handled like an +oversized state: its positive part is rejected whole (section 5.4) and its +removals still apply: + +- An entry whose source the provider does not claim (or that the operator + narrowed away) is discarded. +- A UID whose `id` is empty, blank or over `MAX_UID_LENGTH` + (`ec/kv_types.rs:35`) is dropped. An entry left with no UID is discarded. +- An `inserter` or `matcher` that is blank or over 255 bytes, a UID-level or + EID-level `ext` that is not an object or is over 2,048 bytes as compact + JSON, and an `mm` or `atype` below 0 are dropped as fields, and the entry + is kept. OpenRTB 2.6 reads a missing `matcher` as `source`, so dropping a + blank one keeps its meaning, unless the entry is left with no EID-level + field and a provenance default then applies (section 4.3). +- An entry equal to an earlier kept entry of the same source is dropped. +- A source keeps its first 8 entries and its first 32 UIDs across them. +- A source's entry set over `MAX_ENTRY_SET_BYTES` (4,096 bytes) is + discarded whole, never trimmed, because a trimmed set is not what the + vendor returned. The field caps do not guarantee the set cap: a set whose + legal fields exceed 4,096 bytes is discarded whole. The harness fails a + fixture that hits this. + +`MAX_ENTRY_SET_BYTES` is measured on the stored `eids` array of section 6.1: +entries without `source`, after the field rules above, without provenance +defaults, serialized by `serde_json::to_vec`. The coordinator and +`KvEntry::validate` call the same function, so a set one accepts the other +accepts. + +A discarded set leaves the stored set as it is, as if the result did not +name the source. A result all of whose sets are discarded is handled as +`Malformed` (section 8): `f` grows with its backoff, and its removals still +apply. + +Each discard and drop is logged with the provider ID, source and field +only. The set's `exp`, `ra` and `ru` are the earliest values its kept +entries set, so a set is never served or refreshed past any of its entries' +times. A dropped entry's times do not count. +A set of an expiring source that carries no expiry keeps the stored `exp` +when its entries are unchanged, and is discarded otherwise, because an +owned expiring ID without `exp` is never served (section 6.1). That +fallback is for a vendor that omits an expiry at runtime; the harness fails +a fixture that does. `replacing_state` with a value that does not serialize is handled like an oversized one: the positive part of the result is rejected whole, so a new ID @@ -1185,15 +1370,16 @@ impl IdentityProvider for ExampleIdProvider { return Ok(Resolution::no_match()); }; - let mut id = ResolvedId::new(input.primary_source(), uid).with_atype(1); + let mut uid = ResolvedUid::new(uid).with_atype(1); if let Some(link_type) = reply.link_type { - id = id.with_ext(json!({ "linkType": link_type })); + uid = uid.with_ext(json!({ "linkType": link_type })); } + let mut eid = ResolvedEid::new(input.primary_source(), uid); if let Some(max_age) = reply.max_age_sec { - id = id.with_refresh_in(Duration::from_secs(max_age)); + eid = eid.with_refresh_in(Duration::from_secs(max_age)); } - let resolution = Resolution::found([id]); + let resolution = Resolution::found([eid]); Ok(match reply.signature { Some(signature) => resolution.replacing_state(&ExampleIdState { signature }), None => resolution, @@ -1245,15 +1431,23 @@ impl IdentityProvider for ExampleCaptureProvider { }; Ok(Resolution::found(reply.tokens.into_iter().filter_map(|token| { let source = input.source_named(&token.source)?; - Some(ResolvedId::new(source, token.value).with_expires_at(UnixTime::from_secs(token.expires))) + Some( + ResolvedEid::new(source, ResolvedUid::new(token.value)) + .with_expires_at(UnixTime::from_secs(token.expires)), + ) }))) } } ``` Its descriptor declares -`.with_captures(Captures::new().with_path(Method::POST, "/integrations/examplecapture/api/token"))` -and no `resolves`, so it is never dispatched. +`.with_captures(Captures::new().with_path(Method::POST, +"/integrations/examplecapture/api/token"))` +and no `resolves`, so it is never dispatched. In v1 the coordinator uses a +capture's removals and opt-out, and uses its positive IDs only as the sign +that the vendor issued again (section 4.7). The IDs themselves are not +stored. The trait returns them anyway, so a later durable path needs no +trait change (section 15). ### 4.7 The coordinator @@ -1263,20 +1457,9 @@ rate limiter, runtime services) travel in one `IdentityDeps` struct, because `handle_auction` and `handle_publisher_request` already take the maximum of seven arguments (`auction/endpoints.rs:114-122`, `publisher.rs:4433-4441`). -**The eligibility view.** Eligibility reads one merged view of stored state, -on every route: - -- KV is the base when the request read it, else the cookie's last - confirmed copy. -- Cookie entries with `k: 0` override it when their `pv` matches the base - revision. Removals and opt-outs in the removals cookie `ts-eids-r` always - apply (section 6.2). -- `ps.n` is the later of the cookie's and KV's. `ps.iv` is the union. -- `ps.o`: a staged opt-out (in `ts-eids-r`, not yet in KV) always counts, - unless `ts-eids-s` stages a clear `ps.oc` later than it (section 8). A - mirrored opt-out (a copy in `ts-eids-s`) yields to KV when this request - read KV, so a clear in KV (section 8) is not undone by the browser's copy. -- Pending markers come from the cookie only. +**The eligibility view.** Eligibility reads the KV row the writer read in +phase 1, and nothing else. It never reads `ts-eids-s`. Every removal is +already in KV (section 5.3), so no cookie has to be applied on top. **Eligibility.** A provider that declares `resolves` is eligible for this request when all of these hold: @@ -1306,23 +1489,19 @@ request when all of these hold: the provider is skipped and the skip is logged at debug. 5. `ps.n` (next eligible time) has passed, and no sticky `ps.o` opt-out is set (section 8). -6. No unanswered pending marker `p` is set for the provider (section 6.2). -7. The provider's circuit breaker is closed (below). -8. The request mode allows it, subject to the restrictions on stateful - providers and first navigations below: - - Hold-mode providers run on any request that runs an auction: a - navigation with matched slots, `/auction` or `/_ts/page-bids`. - - After-send providers run only on the writer request for a returning - EC. A new visitor's navigation runs none, so its create stays the page - load's one write (section 5.2). Its after-send providers start at the - next navigation. - - A provider declared `with_private_state()` runs only on the writer - request. In hold mode it runs on writer navigations that run an - auction, and after send on writer navigations that do not, so a site - whose auctions all go through `/auction` still refreshes it. - - No provider runs on a new visitor's first navigation, in either mode. - Its row is created before routing, carrying only browser ingest. -9. The per-visitor burst cap is not exceeded. This check comes last, just +6. The provider's circuit breaker is closed (below). +7. The request is the writer request for a returning EC. Stagers run no + resolving provider, in either mode, and a new visitor's first + navigation runs none, so its create stays the page load's one write + (section 5.2). Within the writer: + - Hold-mode providers run before the auction when the navigation runs + one (matched slots). When it runs none, they run after send instead, + so a site whose auctions all go through `/auction` still stores their + answers, and its next `/auction` serves them from KV. Moved after send, + a hold-mode provider waits `after_send_timeout_ms`, not the hold + budget, and a miss is an ordinary timeout (section 8). + - After-send providers run after send, with `after_send_timeout_ms`. +8. The per-visitor burst cap is not exceeded. This check comes last, just before dispatch, because the Fastly limiter increments on every check under the limit (`adapter-fastly/src/rate_limiter.rs:54-66`). The cap is `max_calls_per_minute` on the key `identity::`, where @@ -1332,16 +1511,17 @@ request when all of these hold: Pacing per visitor comes from `ps.n` and the refresh rules in step 4, not from the limiter. The limiter only caps bursts, because its windows are 60 -seconds long. +seconds long. It also keeps two overlapping writer navigations from both +dispatching one provider. Every call runs on a writer, so every outcome, +`no_match` and failure backoff included, is stored in `ps` (section 8). **Pre-disclosure revalidation.** Before any byte goes to a provider for a -returning EC, the request must have read the live KV row and found it live -and consenting. Eligibility steps 4 to 6 are then recomputed from that live -row, so an opt-out or removal that the cookie has not caught up with still -stops the call. That read is the writer's phase 1 read for hold mode, its -phase 4 read for after-send mode (section 5.2), or the read `/auction` and -`/_ts/page-bids` already do. A new visitor's first navigation runs no -provider (step 8), so every provider call follows a confirmed row. +returning EC, the writer must have read the live KV row and found it live +and consenting, and computed eligibility steps 4 and 5 from it. That read +is the phase 1 read for a provider dispatched before send, and the phase 4 +re-read for one dispatched after send (section 5.2). A new visitor's first +navigation runs no provider (step 7), so every provider call follows a +confirmed row. **Dispatch.** For each eligible provider, the coordinator builds a `ResolveInput` holding only the declared inputs and calls `request`. It then @@ -1354,7 +1534,8 @@ checks the request before sending it: A request that fails a check is dropped and logged as `Config`. The coordinator sends it through `RuntimeServices::http_client().send_async`, up to `max_concurrency` at once, with a backend first-byte timeout of the -provider's timeout (for hold mode, `min(timeout_ms, remaining budget)`, +provider's timeout (`after_send_timeout_ms` after send; before the auction, +`min(timeout_ms, remaining budget)`, rounded to buckets as `auction/orchestrator.rs:453-473` does). It does not follow redirects. It reads at most `max_response_bytes`, checked against `Content-Length` first (`pull_sync.rs:412`) and against the buffered body @@ -1381,7 +1562,7 @@ Why 100 ms: - Missing the budget costs that one auction the ID. A call cut off by the budget is not a failure: it does not count toward `ps.f` or the breaker. It sets `ps.n = now + hold_retry_sec` (default 300 seconds), so the next - request does not wait on the same slow provider straight away. + navigation does not wait on the same slow provider straight away. The coordinator logs each provider's response time. An operator raises the budget only when a provider's measured p95 latency at the edge exceeds it. @@ -1392,13 +1573,11 @@ provider delays bid dispatch. In modes where the origin fetch waits for dispatch, it also delays the page's first byte by up to `hold_budget_ms`. A provider is due rarely (step 4), so few navigations pay this. -**Results and headers.** A hold-mode call either answers within its budget -or is cut off; no call outlives the router (section 3.4). An answer is saved -in that response's `ts-eids-s` with `k: 0`, on the writer request as on -stagers, and a writer also adds it to its mutation. So a later request reads -it from the cookie first, and a failed flush loses nothing. After-send -results exist only in phase 4, after headers are sent. The writer marks each -after-send provider with a pending marker before headers go out (section +**Results.** A hold-mode call either answers within its budget or is cut +off; no call outlives the router (section 3.4). An answer feeds that +navigation's auction and joins the writer's mutation. No answer goes into +a cookie. After-send results exist only in phase 4, after headers are +sent, and are dispatched and committed in the same phase 4 flush (section 5.2). **Circuit breaker.** The rate-limiter trait (`ec/rate_limiter.rs:16-47`) @@ -1415,63 +1594,48 @@ counters are kept per POP, so the breaker is too. `IntegrationProxy::handle` itself has no EC context (`integrations/registry.rs:319-324`). -Capture is off by default. `capture_enabled` (section 10.1) defaults to -`false` for every provider that declares `captures`. While it is false, the -coordinator still runs `capture`, discards the result's positive IDs and -logs at debug what it discarded, by provider ID and source only. The -result's removals and opt-outs still apply as below, because removals win -(G12): an ID captured while capture was on must stay removable after it is -turned off. An operator turns capture on per provider after the rollout -checklist (section 14). +Capture is revoke-only in v1. Every proxied API call is a stager (section +5.1), and a stager persists no positive ID, so a capture result does three +things only: + +- Its removals (`invalidate`, `also_invalidate`) and opt-out (`opted_out`) + are written to KV. +- A positive result clears the provider's opt-out, under the rules below. +- Its positive IDs are counted in `capture_unpersisted`, by provider ID and + source, with no value, so the operator can measure what a durable path + would add (section 15). They are not stored. For each provider with a capture rule (`with_path` or `with_path_prefix`) matching the request: -- **Before `handle`.** When the descriptor's `Captures` declares `with_request_consent` and the - request has a body, `handle_proxy` buffers it with the existing - `collect_body_bounded` (`integrations/mod.rs:182`, 256 KiB cap), extracts - only the named JSON pointers, and re-wraps the bytes with - `EdgeBody::from` before calling `handle`, so the integration forwards them - unchanged. Today the request moves into `handle` unread - (`integrations/registry.rs:1061`), and Lockr buffers its own copy inside - `handle_api_proxy` (`integrations/lockr.rs:208-210`), out of the - coordinator's reach. A body over the cap or not valid JSON allows no - positive IDs; removals still apply. -- **The consent to apply.** `consent::stricter_of(cookie, body)` is a new - function. It combines two consent contexts field by field: any opt-out, - missing purpose or missing vendor consent in either one wins. -- **After `handle`,** on 2xx responses only, the coordinator decodes - `Content-Encoding` under the byte caps, builds a `CaptureInput` and calls - `capture`. -- **It applies the result.** Removals and opt-outs apply whatever the - consent, because a revocation arrives after consent is denied. Positive - IDs apply only when `capture_enabled` is true, eligibility steps 1 to 3 - pass under the stricter consent, and the provider has no sticky `ps.o`, - unless this result clears it: a capture-only provider without a GVL ID - that returns IDs under consenting cookie and body consent stages a clear, - `ps.oc = now`, beside its IDs in `ts-eids-s` (section 8). -- **The real-browser gate.** Eligibility step 1 already keeps positive IDs - from a client that fails `looks_like_browser()`. Such a client's removals - and opt-outs go only into its own response's `ts-eids-r` cookie. They never - trigger the removal overflow write (section 6.2), and a global revoke - (section 8) is staged as a provider opt-out instead. The batch reaches KV - only if that client later makes a writer navigation, which needs a real - browser (section 5.1). So a script replaying a copied `ts-ec` cookie - through the proxy can neither enrich the row nor strip or tombstone it. - Consent withdrawal keeps its own exception (section 5.1). - -The result is staged on `EcContext`. `ec_finalize_response`, which already -runs for proxy responses (`adapter-fastly/src/app.rs:909`, -`main.rs:606-614`), writes positive IDs into `ts-eids-s` and removals and -opt-outs into `ts-eids-r`, including on the unusable-consent path (section -6.2). On a writer request the result also joins the mutation. +- **After `handle`,** on 2xx responses only, `handle_proxy` buffers the + response body up to `max_response_bytes` (a larger body is passed through + uncaptured), decodes a copy under the same cap, builds a `CaptureInput` + and calls `capture`. It re-wraps the original bytes with + `EdgeBody::from(bytes)`, so the browser receives the response unchanged. + The request body is never read. +- **Removals and opt-outs apply whatever the consent,** because a + revocation arrives after consent is denied. The capture removal write + stores them on this request (section 5.3). +- **A clear** needs a positive result under the request's consent: + eligibility steps 1 to 3 pass, and the provider has a sticky `ps.o` that + only a capture can clear (a capture-only provider without a GVL ID, + section 8). The same capture removal write stores it as `ps.oc = now`. +- **The real-browser gate.** A client that fails `looks_like_browser()` + makes no capture removal write and no clear. Its removals, opt-outs and + global revokes are dropped and logged with the provider ID and source + only. Earlier revisions staged them in that client's own cookie, where + they reached KV only if the same client later made a real-browser + navigation, which in practice never happens. So a script replaying a + copied `ts-ec` cookie through the proxy can neither strip nor tombstone a + row. Consent withdrawal keeps its own exception (section 5.1). Capture never touches `ps.n`, `ps.f` or the breaker: a proxied call that -held nothing is not a provider failure. Only an opt-out and its IDs -persist. Capture never changes provider state: a state change in its result -is ignored with a warning, on writers and stagers alike, and the harness's -conformance check fails a `capture` that returns one (section 13). This -holds whether or not the provider declares `with_private_state()`. +held nothing is not a provider failure. Capture never changes provider +state: a state change in its result is ignored with a warning, and the +harness's conformance check fails a `capture` that returns one (section +13). This holds whether or not the provider declares +`with_private_state()`. Lockr's API responses are already buffered on Fastly: a non-streaming send reads the body whole, up to 10 MiB (`adapter-fastly/src/platform.rs:413-429`). @@ -1506,21 +1670,26 @@ An integration-proxy writer never rotates an EC, as today The adapter captures the response status before `send_edgezero_response` consumes the response, and passes it to phase 4 (`main.rs:512-517`, `:618`). -A returning EC's writer flushes only when that status is 2xx or 304. For any -other status it drops its mutation and dispatches no after-send provider. It -saves the answers of hold-mode calls it already made in the response's -cookie, because those calls happen before the origin status is known -(`publisher.rs:4887-4894`, `:5151-5175`). A redirect is followed by another -navigation, which recomputes everything from the same cookies. - -A new visitor's create (section 5.2, phase 1, before routing) is written whatever the -status, as today, because the response sets the EC cookie. +A returning EC's writer flushes its whole mutation only when that status is +2xx or 304. For any other status it dispatches no after-send provider and +flushes everything in its mutation except the positive part of the hold-mode +calls it already made before the origin status was known +(`publisher.rs:4887-4894`, `:5151-5175`): their removals and opt-outs, and +the `ps.n` and `ps.f` of every outcome other than `found`, so a `no_match`, +a failure or a cut-off call still backs off. Their positive answers, with +their state and scheduling, are resolved again by a later writer. Browser +ingest is dropped and re-derived at the next navigation. A redirect is +followed by another +navigation, which recomputes everything from KV and the same cookies. + +A new visitor's create (section 5.2, phase 1, before routing) is written +whatever the status, as today, because the response sets the EC cookie. Every other request is a **stager**: `/auction`, `/_ts/page-bids`, integration proxy calls that are not navigations (including Lockr's API proxy), prefetches, iframes, clients that send no `Sec-Fetch-Dest`, and every -other route. Stagers read cookies and KV but never write KV. Section 5.3 -covers what they save instead. +other route. Stagers read cookies and KV, run no resolving provider, and +write KV only for a captured revocation (section 5.3). `/_ts/api/v1/identify` stops writing too. The once-per-EC `cluster_size` evaluation (`ec/identify.rs:107`, `ec/kv.rs:1484`) moves into the writer's @@ -1532,46 +1701,44 @@ script reports each change to `/_ts/page-bids` (`publisher.rs:6890-6894`). An SPA view is part of the page load that loaded the document, so `/_ts/page-bids` is a stager. Today it ingests `ts-eids` into KV (`publisher.rs:6925`). Under this design it stops. A visitor who stays in an -SPA has their staged changes written at their next full navigation. -Auctions keep using the staged entries from the cookie in the meantime. +SPA has their `ts-eids` ingested at their next full navigation. Auctions in +the meantime use KV and the browser's EIDs. **Exception: consent withdrawal.** A withdrawal writes its tombstone on -whichever request carries it, before the response is sent, exactly as today. -That includes stagers and clients the bot gate does not treat as real -browsers (`adapter-fastly/src/app.rs:446-452`). Revocation cannot wait for a +whichever request carries it, before the response is sent, as today. That +includes stagers and clients the bot gate does not treat as real browsers +(`adapter-fastly/src/app.rs:446-452`). Revocation cannot wait for a navigation that may never come. The exception covers the whole of today's withdrawal path: a CAS tombstone with up to five attempts, then its completion-marker write, for each withdrawn EC ID (one or two; -`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). The same +`ec/kv.rs:1349-1430`, `:1115-1124`; `finalize.rs:509-525`). New in this +spec: when those attempts fail before send, the path runs once more after +send, by which time Fastly's per-key limit has usually passed. The same exception covers a captured opt-out from a provider with `revoke_scope = "global"` (section 8), from a real browser only: it runs this -path on the request that carries it. It and the removal overflow write -(section 6.2) are the only page-load exceptions to G4. The tombstone is not -part of any `IdentityMutation`. A -writer whose phase 4 re-read finds the tombstone drops its mutation (section -5.4, step 1). +path on the request that carries it. It and the two removal writes (sections +5.2, phase 4, and 5.3) are the only page-load exceptions to G4. The +tombstone is not +part of any `IdentityMutation`. A writer whose phase 4 re-read finds the +tombstone drops its mutation (section 5.4, step 1). ### 5.2 Phases of the writer request **Phase 1, at the routing decision. No KV write.** -- Read `ts-ec`, consent, `ts-eids`, sharedId and `ts-eids-s`. Verify - `ts-eids-s` (section 6.2). +- Read `ts-ec`, consent, `ts-eids` and sharedId. `ts-eids-s` is never read. - Decide whether this is the writer request (section 5.1). - Capture the user agent, DNT and page URL into `EcContext` (section 4.4). -- Read the KV snapshot when section 6.3 says to. A missing row for a - returning EC starts today's orphan recovery: confirm the absence, then - rotate before send (`ec/finalize.rs:371-400`). The rotated EC's create - carries this request's browser ingest, as today (`ec/finalize.rs:323`), - and its removals and sticky `ps.o`, `ps.oc` and `ps.iv`, but never - positive staged IDs. It is the page load's one write (section 5.6): a - request that rotates drops the rest of its mutation and its pending - markers, and skips phase 4. When the create is `Written`, its response - deletes the `ts-eids-r` cookies, whose contents the create carried, and - sets a new `ts-eids-s` from the created row. Otherwise both are kept. If the confirm - read finds the row after all, no rotation happens and that read is the - phase 1 snapshot (today's code writes at once there, `ec/finalize.rs:383-386`; - here the change joins the mutation). +- For a returning EC, read the KV snapshot. Every returning writer + navigation reads it once (section 6.3). A missing row starts today's + orphan recovery: confirm the absence, then rotate before send + (`ec/finalize.rs:371-400`). The rotated EC's create carries this + request's browser ingest, as today (`ec/finalize.rs:323`). It is the page + load's one write (section 5.5): a request that rotates drops the rest of + its mutation and skips phase 4. If the confirm read finds the row after + all, no rotation happens and that read is the phase 1 snapshot (today's + code writes at once there, `ec/finalize.rs:383-386`; here the change joins + the mutation). - For a new visitor, `generate_if_needed` creates the row before routing, as today (`ec/mod.rs:407-455`), so no unconfirmed ID is ever used. Today that create is an empty row and browser ingest is a second write in finalize @@ -1581,213 +1748,151 @@ writer whose phase 4 re-read finds the tombstone drops its mutation (section **Phase 2, routing.** -- The coordinator computes eligible providers from the merged view. +- The coordinator computes eligible providers from the phase 1 snapshot + (section 4.7). - If the navigation runs an auction, due hold-mode providers are dispatched - before it, after eligibility is recomputed from the phase 1 read, and - awaited within the hold budget. Their answers feed the auction. -- For a returning EC, the coordinator calls `request` for each eligible - after-send provider as a dry run, with `input.is_dry_run()` true. `request` - is pure and does no I/O. A provider that returns `Some` gets a pending - marker in phase 3. One that returns `None` is skipped (section 8). The real - request is rebuilt in phase 4 from the live row. -- A provider declared `with_private_state()` gets no dry run, because its - state is only in KV. When it is due on the cookie view it gets a marker, - and phase 4 rebuilds its request with the state from the live row. A - rebuilt `None` sets `ps.n` (section 8). This saves a phase 1 read on each - of its refreshes. -- Any other dry run sees what KV holds only when phase 1 read it. Section - 6.3 makes phase 1 read KV whenever an after-send provider is eligible on - the cookie view and has a candidate source that is unowned or only a - stub. + before it and awaited within the hold budget. Their answers feed the + auction. If it runs none, due hold-mode providers join the after-send + providers (section 4.7, step 7). +- A provider due on the phase 1 snapshot in after-send mode is due for + dispatch in phase 4, which rebuilds its request from the live row. There + is no dry run: phase 4 calls `request` once. A `None` then sets `ps.n` + (section 8). **Phase 3, finalize, before send.** - **New visitor.** Nothing to write: the row was created before routing (phase 1). `ts-ec` is set because the create was `Written`, as today (`ec/finalize.rs:145-149`). Phase 4 does nothing for a new visitor, and no - provider runs on its first navigation (section 4.7, step 8). + provider runs on its first navigation (section 4.7, step 7). - **Returning EC.** - Browser ingest from `ts-eids` and sharedId adds `FillMissing` changes to the mutation, instead of calling `sync_eid_cookie_updates` (`ec/finalize.rs:183`). It covers owned sources too, which seeds them for their owner (section 7.2). - - Staged entries from `ts-eids-s` pass the flush checks (section 5.3) and - are added to the mutation. - - Hold-mode answers and this request's captures join the mutation. + - Hold-mode answers join the mutation. A capture on an integration-proxy + writer navigation goes through the capture removal write, as on a stager + (section 4.7), and never joins the mutation. - **Consent withdrawal** runs today's synchronous path (section 5.1). -- **Cookies.** Removals and opt-outs from this request go into a new - `__Host-ts-eids-r-` cookie (section 6.2). `ts-eids-s` is set on the HTML response only - when needed: - - for a new EC, from its created row - - with the results of hold-mode providers that answered, saved with - `k: 0`, except a provider with private state: its answer stays in the - mutation only, because its ID and its state must land together - - with a pending marker for each after-send provider whose dry run - returned `Some`, only when the response status is 2xx or 304 (finalize - already holds the response) - - when the phase 1 read changed the cookie under the section 6.3 merge - - when the cookie was missing, failed to open, opened under a previous - key (section 6.2) or was past its revalidation age +- **Cookies.** This spec sets no cookie on the writer's response. A + returning visitor's HTML gets no new `Set-Cookie` from it and stays + cacheable. **Phase 4, after send. Returning ECs only. The only write.** -1. If the mutation is empty and no after-send provider was marked, stop. No +1. If the mutation is empty and no after-send provider is due, stop. No read, no write. -2. If phase 1 read the row successfully and no after-send provider is - marked, do not read again (a failed phase 1 read counts as no read, and - phase 4 re-reads): apply the mutation to the phase 1 snapshot and, if anything - changes, write with that generation. A concurrent change since then - fails the CAS, as any conflict does. Otherwise re-read the live row. - Either way, the snapshot or the live row decides: +2. If phase 1 read the row successfully and no after-send provider is due, + do not read again (a failed phase 1 read counts as no read, and phase 4 + re-reads): apply the mutation to the phase 1 snapshot and, if anything + changes, write with that generation. A concurrent change since then, + such as a capture removal write by another request of the page load + (section 5.3), fails the CAS, as any conflict does. Otherwise re-read + the live row. Either way, the snapshot or the live row decides: - A tombstone, or `consent.ok = false`: drop the mutation and stop. - - Missing: drop the mutation and stop. The next request that reads KV - finds the orphan (section 6.3). This covers an integration-proxy - writer, which never rotates. - - Failed: drop the mutation and stop. Staged entries stay in the cookie. -3. Recompute eligibility steps 4 to 6 from the live row, rebuild each marked + - Missing: drop the mutation and stop. The next writer navigation's + phase 1 read finds the orphan (section 6.3). This covers an + integration-proxy writer, which never rotates. + - Failed: drop the mutation and stop. +3. Recompute eligibility steps 4 and 5 from the live row, build each due provider's request, dispatch the after-send providers and turn their results into changes. This happens after the re-read, so no EC ID is disclosed before consent is confirmed (`ec/pull_sync.rs:100-110` does the same today). 4. Apply the mutation to the live entry (section 5.4). If anything changed, - write it once with `IfGenerationMatch`. + write it once with `IfGenerationMatch`. The after-send results commit in + this write; nothing marks them as pending in between. 5. If the write fails, on a precondition failure, Fastly's per-key rate - limit or any other error, log and stop. It is not retried. Fastly limits - writes to one key to about one per second, and the limit can be stricter, - so an immediate retry after a conflict would usually fail too. Nothing is - lost for good: + limit or any other error, log. The flush is not retried as a whole. + Fastly limits writes to one key to about one per second, and the limit + can be stricter, so an immediate retry after a conflict would usually + fail too. When the mutation holds removals, opt-outs or clears, they are + then written alone, with the CAS loop of the capture removal write + (section 5.3): a re-read, the removal rules of section 5.4 and up to five + attempts. When those fail too, they are lost and counted in + `removal_lost`. Nothing else backs the flush, so: - browser ingest is re-derived from `ts-eids` by the next navigation - - staged entries and stateless hold-mode answers are still `k: 0` in the - cookie - - removals and opt-outs stay in `ts-eids-r` until a KV read shows them - applied - - an after-send provider's result is lost, an `invalidate` or `opted_out` - included. The ID it rejected stays served, as it was before the call, - until the provider is called again once its pending marker passes its - deadline and repeats the removal. Nothing after send can save it, - because the KV write is what failed - - a stateful hold-mode provider's answer, state change and `ps.n`/`ps.f` - are lost with the mutation. Its answer never went into the cookie - (phase 3), so no ID can flush later without its state, and the next - writer calls it again with the state KV still holds. If the vendor has - rotated that state (a used UID2 refresh token, for example), the call - fails, the stored ID ages out, and the browser or a later initial call - re-seeds it - -**Cost per page load.** Writes are at most one, and never retried. Reads, + - every positive provider result is lost: hold-mode answers and + after-send results, with their state changes and `ps.n`/`ps.f`. Each + provider is still due, and the next writer calls it again, paced by the + burst cap. Their removals survive through the retry above, unless it + fails too + - a stateful provider is called again with the state KV still holds. If + the vendor has rotated that state (a used UID2 refresh token, for + example), the call fails, the stored ID ages out, and the browser or a + later initial call re-seeds it + +**Cost per page load.** Writes are at most one, plus the removal-only retry +when a flush that holds removals fails. Reads, compared with today: -| Navigation | Today | This design | -| --------------------------------------------------------------- | ---------------------- | ---------------------------------------------------------------------- | -| Runs an auction | 1 (preload) | 1 (phase 1). Phase 4 reuses it unless an after-send provider is marked | -| No auction, registered source in `ts-eids` | 1 (ingest compare) | 1 (phase 4) | -| No auction, nothing due, staged or pending, valid recent cookie | 0 or 1 | 0 | -| An after-send provider dispatches | 2 (preload, pull sync) | 1 or 2: the phase 4 re-read, plus phase 1 when section 6.3 requires it | - -Today's preload never skips a navigation whose auction needs the row -(`publisher.rs:3110-3116`, `:4825-4828`), and the cookie does not cache -unowned values, so such a navigation reads KV once, as today. The 6.3 -trigger list states the rule. -Each `/auction` and `/_ts/page-bids` keeps its one existing read. Orphan -recovery adds the reads it adds today, in the rare case it runs. +| Navigation | Today | This design | +| --------------------------------- | ---------------------- | ------------------------------------------------------------------- | +| Runs an auction | 1 (preload) | 1 (phase 1). Phase 4 reuses it unless an after-send provider is due | +| No auction, no provider due | 0 or 1 | 1 (phase 1). Phase 4 reuses it | +| An after-send provider dispatches | 2 (preload, pull sync) | 2: phase 1 and the phase 4 re-read | -A digest of the last-compared browser set in `ts-eids-s` could remove the -read on navigations with no auction. It is left out until measurements show -the read matters. +Every returning writer navigation reads KV once in phase 1. That read finds +orphans, answers eligibility and filters browser candidates, so no cookie +copy of the row is needed. Each `/auction` and `/_ts/page-bids` keeps its +one existing read. Orphan recovery adds the reads it adds today, in the rare +case it runs. ### 5.3 Stagers -A stager never writes KV. It saves positive changes into `ts-eids-s` and -removals and opt-outs into `ts-eids-r`, on its own response (section 6.2). -`ts-eids-s` is set in the returning-visitor branch of `ec_finalize_response` -(`ec/finalize.rs:81-127`), replacing today's ingest write at -`finalize.rs:110-112`. That branch already has the request's client EIDs -(`ec/mod.rs:568-578`) and its KV snapshot. `ts-eids-r` is also written from -`finalize_unusable_consent` (`ec/finalize.rs:457`), because a revocation can -arrive while consent is unusable but not explicitly withdrawn, and that path -returns before the returning-visitor branch (`finalize.rs:62-78`). A stager -saves: - -- **The KV gap, on `/auction`.** `/auction` holds the full body set - (`auction/endpoints.rs:298-303`) and the KV row it already reads - (`endpoints.rs:307-311`). For each registered source in the consent-gated - body set that the KV row lacks or holds only expired, it saves - `{s, u, b: 1, k: 0}`. It picks the - UID with today's ingest rules (`collect_eid_updates`, - `ec/prebid_eids.rs:151`). It saves only absent or expired sources, - because ingest is fill-missing (sections 3.2 and 5.4), and skips a UID that matches `ps.iv` (section - 7.2). It costs no extra KV read. - - When the client set came from the `ts-eids` cookie rather than a body, - nothing is saved: the next navigation ingests that cookie anyway. - - When the KV read failed, found the row missing or found a tombstone, - nothing is saved. - - `/auction` returns early on consent denial, before it reads KV - (`endpoints.rs:231-275`). Then nothing is saved. -- **Hold-mode results** on `/auction` and `/_ts/page-bids`, from providers - without state: IDs as entries with `pv` (the revision they replace) in - `ts-eids-s`, `ps` changes marked `sg: 1`, and invalidations and opt-outs - in `ts-eids-r`. `/_ts/page-bids` reads KV only when its auction - runs (`publisher.rs:7077-7090`), so it runs hold mode only then. -- **Passive capture results** on integration proxy responses (Lockr), the - same way. -- **Cookie upkeep from the KV read** that `/auction` and `/_ts/page-bids` - already do (section 6.3). These responses are JSON and already private, so - cookie upkeep costs no HTML cacheability. - -Providers with state run only on the writer request (section 4.7, step 8), -because state lives in KV only and a stager cannot write it. A stager that -sees one of them due leaves it to the next navigation. - -**No lost EIDs (G10).** If the KV gap does not fit in the cookie's size cap, -the next `/auction` finds the same gap, since KV still lacks it, and saves it -again. An ID is delayed by a page view, never dropped, as long as the browser -keeps sending it. Today's `/auction` ingest writes the same fill-missing set -(`ec/kv.rs:146-154`), so everything it writes still reaches KV. - -**Concurrent stagers.** Parallel `/auction` or `/_ts/page-bids` calls each -rewrite the whole cookie from their own copy, and the last `Set-Cookie` -wins. A saved entry can be lost this way: - -- A lost KV-gap entry is saved again by the next `/auction`. -- A lost hold-mode result costs one later provider call. The per-visitor - burst cap, one call per minute by default, makes near-simultaneous - duplicate calls rare. The limiter is best-effort under concurrency - (`ec/rate_limiter.rs:25-27`), so they are not impossible. -- Removals and opt-outs are never lost this way. Each batch lives in its own - `ts-eids-r` cookie, which no other response rewrites and only a writer - whose KV read shows it applied deletes (section 6.2), so a concurrent - rewrite of `ts-eids-s` or a second removal cannot drop them. A capture cannot be re-derived, so this matters most for - Lockr's `revoke-consent`. - -The loss of positive entries is accepted. - -**Flush checks for staged entries.** In phase 3, the writer adds a staged -entry to its mutation only when it passes all of these: - -These checks apply to positive entries. Removals and staged opt-outs keep -only the binding and revision checks (section 5.6). - -- The cookie opens with this EC's ID as associated data (section 6.2). -- The entry is fresh: its staged-at time `sa` is no more than - `staged_max_age_sec` ago. Older entries are dropped from the cookie on its - next rewrite. -- Its writer `w` is an enabled provider that currently claims the source and, - for an owned source, owns it. -- It passes `gate_eid_updates_by_consent` (`ec/finalize.rs:268`) and the - writing provider's consent policy, both against this request's consent, - which may have changed since staging. -- A provider entry is dropped when `ps.o` marks that provider opted out, - unless the cookie also stages a clear `ps.oc` later than that opt-out's - `at`. A replayed old cookie therefore cannot re-insert an opted-out ID: its - clear predates the opt-out. -- A provider entry becomes an `OwnerWrite` with its `pv` as the expected - revision when its writer currently owns the source, and a `FillMissing` - otherwise, so an unowned source keeps the fill-missing contract. The cookie - may hold `k: 0` entries for unowned sources; it caches only owned sources - once they are in KV. A removal from `ts-eids-r` becomes an `OwnerRemove`, - or a `WriterRemove` when it carries no revision. A browser entry becomes a - `FillMissing`. A staged `ps` change merges as described in - section 6.2. +A stager runs no resolving provider and never persists a positive ID. It +reads KV only where its route already does: `/auction` reads the row it +reads today (`auction/endpoints.rs:307-311`), and `/_ts/page-bids` reads it +only when its auction runs (`publisher.rs:7077-7090`). Its auction serves +the server set from that read (section 7.1). It no longer runs today's +ingest write in the returning-visitor branch of `ec_finalize_response` +(`ec/finalize.rs:110-112`). + +Earlier revisions let stagers run stateless hold-mode providers in memory. +With no `ps.n` or `ps.f` stored on a stager, a provider that answered +`no_match` or a 4xx was called again once a minute on every `/auction`, for +the whole of an SPA session. Revision 15 removes that. + +**The capture removal write.** A revocation captured from a real browser +(section 4.7) is written to KV on the proxied request that carries it. It is one of the page-load exceptions to G4 (section 5.1). The write: + +- reads the row and applies only the capture's removals (with the `ps.iv` or `ps.ov` hashes they derive), its opt-out and its clear, under the rules of + section 5.4 +- writes with a CAS and the withdrawal path's five attempts, re-reading and + trying again only when the generation no longer matches. Any other + failure, Fastly's per-key limit included, ends the attempt at once +- runs before the proxied response is sent and, when that attempt fails, + once more after send +- writes nothing when the read shows a tombstone, a missing row or the + removal already applied +- when both attempts fail, logs at error with the provider ID and source + only, and increments `removal_lost`. Nothing keeps the removal + (section 2.3) + +It runs whatever the request's consent, including when consent is unusable +but not withdrawn, because a revocation can arrive then. A writer +navigation of the same page load that flushes with its phase 1 generation +after this write fails its CAS and is not retried (section 5.2, phase 4); +its browser ingest is re-derived at the next navigation. + +`/auction` and `/_ts/page-bids` return the mirror payload of section 6.2 in +their JSON response, from the KV row they read. The shim writes +`ts-eids-s` from it. These responses are already private, so the mirror +costs no HTML cacheability. + +**The `ts-eids` trim keeps registered sources first.** Today `/auction` +ingests registered sources from its body, which carries more than the +3,072-byte `ts-eids` cookie. Under this design only the writer ingests, and +only from `ts-eids`. So the shim, when it trims `ts-eids` (EID fidelity +spec §4.3), drops unregistered sources before any registered one. It learns +the registered source domains from the configuration TS injects with its +script. Within each group the EID fidelity trim order is unchanged. Every +registered source today's `/auction` ingest writes then reaches the next +writer's ingest, unless the registered sources alone exceed the trim (G10). + +**Concurrent stagers.** Stagers set no server cookie, so concurrent +responses cannot overwrite each other's state. Each can return a mirror +payload; the last one the shim writes wins, and all of them come from KV. ### 5.4 `IdentityMutation` @@ -1800,43 +1905,56 @@ pub(crate) struct IdentityMutation { } /// The positive part of one stateful provider's result, applied whole or not -/// at all. Its removals, opt-out and `iv` hashes are not part of it: they go -/// into `partner_ids` and `provider_state` and apply on their own. +/// at all. Its removals and opt-out are not part of it: they go into +/// `partner_ids` and `provider_state` and apply on their own. pub(crate) struct ProviderResult { provider: IdentityProviderId, /// `OwnerWrite` and `FillMissing` changes for the sources it returned. ids: Vec, /// The state replacement or clear, and the new `n` and `f`. state: ProviderStateChange, - /// Keyed hash of the `ps.st` the call read. `None`: it read no state. + /// Hash of the `ps.st` the call read. `None`: it read no state. expected_state: Option, } pub(crate) enum PartnerIdChange { - /// From browser ingest, a KV-gap entry or an unowned provider. + /// From browser ingest or an unowned provider. /// Applies only if the source is absent or its record is expired. FillMissing { source: SourceDomain, record: StoredRecord }, /// From the owning provider. Applies if the live record's revision equals /// `expected`, or if `expected` is `None` and the source has no valid ID - /// that this provider wrote. A browser-seeded or legacy value never - /// blocks the owner. + /// that this provider wrote. A browser-seeded value never blocks the + /// owner. OwnerWrite { source: SourceDomain, record: StoredRecord, expected: Option }, /// From `Resolution::invalidate` or `also_invalidate`. Applies only if - /// the revision matches. - OwnerRemove { source: SourceDomain, expected: Revision }, - /// From `Resolution::opted_out`, or a staged removal without a revision. - /// Removes the source's record if this provider wrote it, at any - /// revision. + /// the revision matches. Applying it appends the hash of every UID of the + /// record it removes to `provider`'s `ps.iv` (section 7.2). `provider` is + /// the provider that invalidated, which for an unowned source is not an + /// owner: its `ps.iv` still filters the browser's candidates (section 7.1). + OwnerRemove { source: SourceDomain, provider: IdentityProviderId, expected: Revision }, + /// From `Resolution::opted_out`. Removes the source's record if this + /// provider wrote it, at any revision. For a capture-only provider it + /// also removes a record no provider wrote (browser ingest, schema + /// version 1 rows), because such a provider never writes one itself + /// (section 8). WriterRemove { source: SourceDomain, writer: IdentityProviderId }, } +/// A `StoredRecord` is a source's whole record: its entry set (section 6.1), +/// `w`, the timing fields and `rv`. Every change below writes or removes a +/// source's set whole. No change edits one entry or one UID, so one `rv` +/// per source covers every entry in it. +/// /// A record's opaque revision: its KV `rv`, 64 random bits from the /// platform generator, drawn fresh on every write of the record, including a /// refresh that keeps the UID. Neither the UID nor `acquired_at` can serve: -/// two refreshes of an unchanged UID in one second share both. A legacy -/// record without `rv` uses the keyed hash of its `(uid, at)` pair until its -/// next write. The cookie's `pv` stores the revision a staged entry replaces. -pub(crate) struct Revision(RecordRevision); +/// two refreshes of an unchanged UID in one second share both. A record +/// without `rv`, which only a schema version 1 row holds, has the revision +/// `Unset` until its next write gives it one. +pub(crate) enum Revision { + Unset, + Set(RecordRevision), +} ``` A new visitor's create is not a mutation: `generate_if_needed` builds the @@ -1847,21 +1965,31 @@ Applying the mutation to a live entry, in order: 1. If the live entry is a tombstone, drop everything. 2. Apply removals first: every `OwnerRemove` and `WriterRemove`. Applying a new `ps.o` for a provider also removes every live record that provider - wrote, owned or unowned, at any revision. Removals are never skipped. + wrote, owned or unowned, at any revision, and for a capture-only + provider every record of its sources that no provider wrote. Removals + are never skipped. Every invalidation that applies appends to the invalidating + provider's `ps.iv`, and every opt-out removal appends to the provider's `ps.ov`, the + hash of every UID of the live record it removes, so the hashes always + come from the record KV held. One that no longer + applies adds none: the record it named is gone or was replaced. An opt-out applies when its `at` is after the live `ps.o.at` and at or after the live `ps.oc`, so an opt-out and a clear in the same second - leave the provider opted out. Then apply each staged clear `ps.oc` that is - strictly later - than the provider's opt-out `at`, live or staged, and store it as the - live `ps.oc`, so the next step sees the provider as not opted out and a - later replay of the old opt-out is stale. A clear that is not later is - dropped. + leave the provider opted out. Then apply a clear (section 4.7) when it + is strictly later than the provider's live opt-out `at`, and store it as + the live `ps.oc`, so the next step sees the provider as not opted out. A + clear that applies also empties the provider's `ps.ov`, so the IDs its + opt-out removed can come back through the browser path. `ps.iv` is never + emptied: an invalidation is about the ID, not about consent. + A clear that is not later is dropped. A clear with no live opt-out to + clear is dropped too and writes nothing. 3. Apply the positive changes by their own rules. Every change written by a provider the live `ps.o` marks opted out is skipped, `OwnerWrite` and - `FillMissing` alike, and so is a staged entry whose `sa` is at or before - that provider's latest opt-out `at`, even after a later clear. A - `FillMissing` whose UID matches the live `ps.iv` is skipped. Changes that - no longer apply are dropped silently: a newer write won. + `FillMissing` alike. A `FillMissing` any of whose UIDs matches the live + `ps.iv` or `ps.ov` of a provider that claims the source is skipped, owned + or + unowned. A browser-ingest `FillMissing` for a source of an opted-out + capture-only provider is skipped too. + Changes that no longer apply are dropped silently: a newer write won. Each `ProviderResult` applies atomically. Its IDs, its state change and its `n` and `f` apply together, and only when every ID change in it would apply on its own and the live `ps.st` still hashes to @@ -1869,8 +1997,14 @@ Applying the mutation to a live entry, in order: phase 4 re-read finds that another writer committed a newer ID or state since its call read the row therefore stores neither its ID nor its state, and no row pairs one call's ID with another call's state. -4. Apply `provider_state` changes (`n`, `f`, `o`, `iv`, `st`) and - `cluster_size`. Drop the `ps` entries of providers no longer configured. +4. Apply `provider_state` changes (`n`, `f`, `o`, `iv`, `ov`, `st`) and + `cluster_size`. Truncate each `ps.iv` and `ps.ov` from the front to + `MAX_INVALIDATED_UIDS` (section 6.1), so step 5 never fails on them. For + a provider no longer configured, drop every `ps` field except `o`, `oc` + and `ov`, so a rollback and a roll-forward never forget an opt-out + (section + 14). Such an entry still counts toward the 32-provider cap, and is the + first dropped when the cap is exceeded. Drop expired records, except **refresh anchors**: an expired record of an owned source whose owner still holds `ps.st` and whose `ru` has not passed. An anchor is never served (section 7.1), but it keeps the @@ -1880,15 +2014,18 @@ Applying the mutation to a live entry, in order: included) or missing, or when the owner is no longer configured. A record without `ru` is dropped at expiry, as before. After a drop the owner sees no record and selects `Initial`. + 5. Validate the result with `KvEntry::validate`. If it fails, keep only the removals and opt-outs and validate again, so a full row can always shrink. If that also fails, drop the mutation and log. An invalid row is never written. 6. If nothing changed, skip the write. +The capture removal write (section 5.3) applies a mutation that holds only +removals, an opt-out and a clear, through steps 1, 2, 4 (the `o`, `oc`, `iv` and `ov` fields only), 5 and 6. + Because the rules are checked against the live entry, a later writer can -re-apply the same changes, re-derived or still staged, without losing a -concurrent tab's newer data. +re-apply re-derived changes without losing a concurrent tab's newer data. ### 5.5 Orphans, collisions and deletion @@ -1901,11 +2038,11 @@ safety properties, which a deferred create would lose: - **Collisions are caught.** An `AlreadyExists` on the first create re-mints the ID before any cookie is set, so no browser adopts another visitor's row. -- **Orphans are found.** A row that goes missing later is found by a KV read - and recovered by today's rotation, before send (section 5.2, phase 1). - Section 6.3 guarantees such a read at least every `kv_revalidate_sec`, and - any stager that sees the row missing replaces `ts-eids-s` with a minimal - `m: 1` cookie to force one at the next navigation. +- **Orphans are found.** A row that goes missing later is found by the next + writer navigation's phase 1 read, which every returning writer navigation + makes, and recovered by today's rotation, before send (section 5.2, + phase 1). A stager that sees the row missing stores nothing and tells the + shim to clear `ts-eids-s`. **Deletion.** The IAB data-deletion framework, deferred today, would hard-delete rows (`ec/kv.rs:1551-1563`). When that framework lands, it writes @@ -1913,69 +2050,32 @@ a tombstone with `TOMBSTONE_TTL` (24 hours, `ec/kv.rs:38-39`) instead, using its own unconditional writer and today's completion marker. That replaces one delete with one write, the TTL removes the tombstone with no further write, and every TS request that reads the row in those 24 hours treats the -identity as withdrawn. A browser that visits within those 24 hours -revalidates (section 6.3) and expires its cookie. One that returns later -finds the row missing, and section 5.6 makes a missing row expire the cookie -and rotate without carrying any positive IDs, so a deletion is never undone -through the cookie. - -### 5.6 Rules that keep removals winning and markers answered - -Round 4 found paths where a cookie could undo a removal or a marker could -loop. These rules close them, and take precedence over any earlier sentence -that seems to allow otherwise: - -- **Removals survive every filter.** Removals and staged `ps.o` and `ps.iv` - live in `ts-eids-r` (section 6.2). They are exempt from the consent - re-gate, the writer-still-owns check and `staged_max_age_sec` in the flush - checks (section 5.3), and keep only the EC binding and, when they carry - one, the revision check. Withdrawing consent, disabling a provider or a - week in an SPA can delay a positive ID, never a removal. The one gate a - removal does face is the real-browser gate on capture (section 4.7): a - capture removal from a client that fails `looks_like_browser()` stays in - that client's own `ts-eids-r` and reaches KV only through that client's - own writer navigation, never through the removal overflow write. -- **A config change keeps removals.** When the `cf` of `ts-eids-s` does not - match, its positive entries are discarded and it is rebuilt from KV. - Pending markers are kept. `ts-eids-r` carries no `cf`, so its removals - apply against the current config unchanged. -- **A missing row empties the cookie.** Any KV read that finds the row +identity as withdrawn. A browser that visits within those 24 hours reads +the tombstone at its next writer navigation or `/auction`, and the latter +tells the shim to clear `ts-eids-s`. One that returns later finds the row +missing and rotates. The rotation carries no positive ID, because the +server keeps none outside KV, so a deletion is never undone through a +cookie. + +### 5.6 Rules that keep removals winning + +- **Removals reach KV on the request that carries them.** A provider's + removals, opt-outs and clears are written by the writer's own flush, or + by the capture removal write for a proxied call (section 5.3). They + apply whatever the consent and whether or not their provider is still + enabled, and no cookie or config change can strand one. The one gate a + removal faces is the real-browser gate on capture (section 4.7). +- **A missing row carries no positive ID.** Any KV read that finds the row missing (outside a recovery in the same request) treats it like a - tombstone for the server set: no positive entries or markers survive, and - `ts-eids-s` is replaced by a minimal cookie holding only `m: 1`, so the - next navigation reads KV and recovers. `ts-eids-r` is kept. `/auction` - saves no KV gap when the row is missing. Orphan rotation carries its - removals and sticky `ps.o`, `ps.oc` and `ps.iv` into the new row, never - positive staged IDs. -- **Markers are answered without busy writes.** A KV read answers a marker - when the live row makes that provider ineligible (eligibility steps 4 to - 6), as well as by the rules of section 6.2. Phase 4 adds - `ps.n = max(ps.n, now + 60 s)` only for a marked provider it did not dispatch for a - reason the live row does not show, such as the burst cap or the breaker. A - rebuilt `request` that returns `None` already sets `ps.n` (section 8). If - the flush fails, the marker expires at its deadline as before. -- **Unowned dry runs need KV.** A dry run for a provider whose sources are - unowned returns `Some` only when the phase 1 read showed the source missing - and the pull-complete marker is absent. Otherwise the cookie cannot know, - and no marker is set. -- **Staged entries trigger a reconciling read.** A cookie holding `k: 0` - entries makes the writer read KV in phase 1. That read flips flushed - entries to `k: 1` on this response (phase 3's "the phase 1 read changed the - cookie" rule). Phase 4 reuses that generation only when no after-send - provider is marked. Otherwise it re-reads the live row before any - dispatch, so a withdrawal that lands after send is always seen. -- **Withdrawn consent never refills the cookie.** A stager whose decoded - consent denies EID persistence never writes positive entries into - `ts-eids-s`, so a concurrent `/auction` that started before a withdrawal - cannot bring IDs back. It may still set removals in a new `ts-eids-r` - cookie. -- **The cookie is rewritten only when it changes.** A request that read KV - re-emits `ts-eids-s` only when its entries, markers or `ps` changed, when - the inbound `kc` is older than half of `kv_revalidate_sec`, or when it - opened under a previous key (section 6.2), so it moves to the current key. - Phase 3 - stamps `kc` only when this request read KV. This keeps `Set-Cookie` off - most responses and narrows the window for concurrent overwrites. + tombstone for the server set: an `/auction` or `/_ts/page-bids` response + tells the shim to clear `ts-eids-s`. Nothing carries `ps` across an + orphan rotation, so after one a UID a provider rejected can be re-seeded + once, and an opted-out provider can be called again until it reports the + opt-out again. +- **Withdrawn consent never refills the mirror.** A stager whose decoded + consent denies EID persistence returns a clear for `ts-eids-s`, never a + value, so a concurrent `/auction` that started before a withdrawal cannot + bring IDs back. - **Cluster size rides on a write.** `/_ts/api/v1/identify` is a cross-site, bearer-token endpoint (`ec/identify.rs:23-60`), so it cannot rely on TS cookies. When it reads a row without `cluster_size`, it @@ -1985,12 +2085,6 @@ that seems to allow otherwise: list (`ec/kv.rs:1460-1469`) never delays a page. That is one list and at most one write per EC, once, as today (`ec/kv.rs:1497-1529`); the write is the page load's one write. -- **The size cap always holds.** If dropping every droppable entry still - leaves `ts-eids-s` over its cap (section 6.2), it is replaced by a minimal - cookie with `kc = 0` and the markers only, which forces a read at the next - navigation. A source that is stubbed on every rewrite is left as a stub, - and the auction reads it from KV only when the navigation's auction needs - that source. - **EC minting on non-writer navigations.** Today a real browser gets an EC on any navigation `is_navigation_request` accepts, including prefetch and the `Accept: text/html` fallback (`adapter-fastly/src/app.rs:846-857`). @@ -1998,21 +2092,41 @@ that seems to allow otherwise: visitor's writer does, and that create is its page load's one write. It runs no providers and writes nothing else. - **Proxy capture plumbing.** `ProxyDispatchInput` gains `is_real_browser`, - the writer decision and `IdentityDeps`. To capture, `handle_proxy` buffers - the response body up to `max_response_bytes` (a larger body is passed - through uncaptured), decodes a copy for `capture`, and re-wraps the - original bytes with `EdgeBody::from(bytes)`, so the browser receives the - response unchanged. + the writer decision and `IdentityDeps`. Section 4.7 covers how + `handle_proxy` buffers and decodes a response for `capture`. ## 6. Storage ### 6.1 KV record -`KvPartnerId` (`ec/kv_types.rs:102`) gains optional fields. `KvEntry` gains -`ps`. All new fields use `#[serde(default, skip_serializing_if = ...)]`, so -`SCHEMA_VERSION` stays 1 and old rows read unchanged. KV item metadata, which -Fastly caps at 2,048 bytes, holds no IDs (`ec/kv_types.rs:199-219`) and is -unaffected. +`SCHEMA_VERSION` becomes 2 (`ec/kv_types.rs:17`). `KvPartnerId` +(`ec/kv_types.rs:102`) is replaced by one record shape, and `KvEntry` +gains `ps`. KV item metadata, which Fastly caps at 2,048 bytes, holds no IDs +(`ec/kv_types.rs:199-219`) and is unaffected. + +**One record shape.** Every record stores its source's entry set in `eids`: +an array of OpenRTB EID entries without `source` (the map key holds it), +each `{uids: [{id, atype?, ext?}], inserter?, matcher?, mm?, ext?}`. A +record that browser ingest, generic pull or batch sync writes is one entry +with one UID and no EID-level fields; only a provider result writes more. +The stored entry types are dedicated (`KvEid`, `KvUid`), carry no `source` +and ignore unknown fields. They are never `openrtb::Eid`, which denies +unknown fields and requires `source` (`openrtb.rs:72`), so a field a later +build adds never makes an older build fail to read the row. + +**Reading schema version 1.** Every row on `main` is version 1, whose +records hold one `uid`. The reader accepts both versions. It reads a +version 1 record as one entry holding that `uid`, with no `w`, no `rv` and +no timing beyond `at`, and the next write of the row stores it as version 2. +Today's reader rejects any version but its own (`ec/kv_types.rs:376-381`), +so a build older than rollout step 1 cannot read a row step 1 has written +(section 14). + +`KvEntry::validate` rejects, when a row is written, a record whose `eids` is +empty or whose set breaks the limits of section 4.5. It runs on every read +too (`ec/kv.rs:366`), so on read a bad record is dropped from the row and +the violation is logged with the source only. One bad record never makes +the row unreadable. Only `serialize_entry` rejects. `KvPartnerId` and `KvEntry` are public structs without `Default`, and about 30 `KvPartnerId { .. }` literals in 8 files would stop compiling (for example @@ -2023,18 +2137,46 @@ literals move to them. `PartnerIdUpdate` (`ec/kv.rs:75-90`) gains ```json { + "v": 2, "ids": { "id5-sync.com": { - "uid": "ID5*example", "w": "id5", "rv": "9c1d04b7e6a83f2a", "at": 1760000000, "exp": 1767776000, "ra": 1760086400, - "atype": 1, - "ext": { "linkType": 2 } + "eids": [ + { + "uids": [ + { "id": "ID5*example", "atype": 1, "ext": { "linkType": 2 } } + ] + } + ] + }, + "example-ids.example.com": { + "w": "exampleid", + "rv": "4be71f0c2d9a6e35", + "at": 1760000000, + "eids": [ + { + "uids": [ + { "id": "example-a", "atype": 1 }, + { "id": "example-b", "atype": 3, "ext": { "stype": "ppuid" } } + ], + "inserter": "example.com", + "mm": 3 + }, + { + "uids": [{ "id": "example-c", "atype": 1 }], + "matcher": "partner.example.com" + } + ] }, - "sharedid.org": { "uid": "example-shared-id", "at": 1759990000 } + "sharedid.org": { + "rv": "07a3c9e15b2d48f6", + "at": 1759990000, + "eids": [{ "uids": [{ "id": "example-shared-id" }] }] + } }, "ps": { "id5": { "n": 1760003600, "f": 1, "st": "example-opaque-state" } @@ -2042,354 +2184,168 @@ literals move to them. `PartnerIdUpdate` (`ec/kv.rs:75-90`) gains } ``` -| Field | Meaning | -| ------- | ------------------------------------------------------------------------------------------------------------------- | -| `w` | Writer: a provider ID. Absent for browser ingest and legacy rows | -| `rv` | The record's revision: 16 hex digits, drawn at random on every write of the record (section 5.4) | -| `at` | Unix seconds when the ID was acquired or last refreshed | -| `exp` | Unix seconds after which the ID must not be sent | -| `ra` | Unix seconds after which the ID is due for refresh | -| `ru` | Unix seconds after which the ID can no longer be refreshed. Keeps an expired record as a refresh anchor until then | -| `atype` | Provider-supplied atype. Absent: use `[[ec.partners]].openrtb_atype` | -| `ext` | UID-level ext, at most 256 serialized bytes | -| `ps.n` | Unix seconds before which the provider must not be called | -| `ps.f` | Consecutive failures, for backoff | -| `ps.o` | Sticky opt-out: `{at, fp}`, when it was reported and the consent fingerprint | -| `ps.oc` | When the provider's opt-out was last cleared. An opt-out reported before it is stale and never re-applied | -| `ps.iv` | Keyed hashes of UIDs the owner invalidated, at most 4, so they are not re-seeded | -| `ps.st` | The provider's private state, as embedded JSON (a raw value, not an escaped string) of at most 1,024 bytes. KV only | - -`KvEntry` validation (`ec/kv_types.rs:375`) adds: `ext` at most 256 bytes, -`st` at most 1,024 bytes of embedded JSON, `ps` at most 32 providers. -Mutations prune the `ps` entries of providers no longer configured, and more +| Field | Meaning | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `w` | Writer: a provider ID. Absent for browser ingest and version 1 records | +| `rv` | The record's revision: 16 hex digits, drawn at random on every write of the record (section 5.4) | +| `at` | Unix seconds when the ID was acquired or last refreshed | +| `exp` | Unix seconds at or after which the ID must not be sent | +| `ra` | Unix seconds after which the ID is due for refresh | +| `ru` | Unix seconds after which the ID can no longer be refreshed. Keeps an expired record as a refresh anchor until then | +| `eids` | The source's entry set, in order, with UID-level and EID-level fields (section 4.5). A missing `atype` uses `[[ec.partners]].openrtb_atype` | +| `ps.n` | Unix seconds before which the provider must not be called | +| `ps.f` | Consecutive failures, for backoff | +| `ps.o` | Sticky opt-out: `{at, fp}`, when it was reported and the consent bits the provider's policy read then (section 8) | +| `ps.oc` | When the provider's opt-out was last cleared. An opt-out reported before it is stale and never re-applied | +| `ps.iv` | Hashes of UIDs the owner invalidated, oldest first, at most 128, so they are not re-seeded (section 7.2) | +| `ps.ov` | Hashes of UIDs the provider's opt-out removed, oldest first, at most 128. Emptied when the opt-out clears (section 8) | +| `ps.st` | The provider's private state, as embedded JSON (a raw value, not an escaped string) of at most 1,024 bytes. KV only | + +`KvEntry` validation (`ec/kv_types.rs:375`) adds: the entry-set limits of +section 4.5 (UID-level and EID-level `ext` at most 2,048 bytes, 8 entries +and 32 UIDs per source, a set at most 4,096 bytes), `st` at most 1,024 +bytes of embedded JSON, `ps` at most 32 providers and `ps.iv` and `ps.ov` each at most `MAX_INVALIDATED_UIDS` (128) hashes. Mutations prune the `ps` entries +of providers no longer configured to their opt-out (section 5.4), and more than 32 enabled providers is a config error (section 10.2), so the cap is -never reached in practice. With every cap reached, -eight owned sources come to about 7 KB, plus 1 KB for each provider with -state. A row normally carries one or two stateful providers. That is above the ~5 KB estimate in -`ec/kv_types.rs:40`, which is a documented estimate rather than an enforced -limit. Fastly's value limit is far larger. +never reached in practice. + +`ps.iv` and `ps.ov` are ordered lists, oldest first, with the same rules. +Each hash is the first 16 hex +digits of the unkeyed SHA-256 of the UID. A new hash is appended, and a +hash already present moves to the end. Mutations truncate it from the front +before validation (section 5.4). 128 hashes hold four whole 32-UID sets, so +one removal never evicts the previous one, and an `also_invalidate` of +several sources keeps all their UIDs. The hash is unkeyed because the same +row stores every live UID in plain text, so a key would protect nothing, +and an unkeyed hash survives an `ec.passphrase` rotation (section 9.3). + +Size. A one-UID record is about 120 bytes: today's `uid`, the `eids` +wrapper and the metadata. A record at every cap is at most +`MAX_ENTRY_SET_BYTES` (4,096 bytes) plus its metadata, about 4.2 KB. With +every cap reached, eight owned sources come to about 34 KB, plus 1 KB for +each provider with state, and `ps.iv` and `ps.ov` add up to 2.4 KB each per +provider. A row +that carries two full sets and two stateful providers at every cap is about +15 KB, and typical sets are far smaller. That is above the ~5 KB estimate +in `ec/kv_types.rs:40`, which is a documented estimate rather than an +enforced limit, and the estimate is updated. Fastly's value limit is far +larger. A row that only browser ingest and pull sync write grows by about +60 bytes per source. The admin EC lookup returns the stored JSON verbatim (`ec/admin.rs:394-397`). It redacts `ps.st`. -**Rollback rule.** The current reader drops unknown fields when it writes the -row again. If TS is rolled back after providers have written metadata, the -next write strips `w`, `exp` and the rest. So, for an owned source whose -provider marks the source as expiring, only records with `w` set to the owner and an +**Serving owned expiring sources.** For an owned source whose provider +marks it as expiring, only records with `w` set to the owner and an unexpired `exp` are sent to bidders from the server. Any other record for -that source (browser-seeded, or stripped by an older writer) is used only as -the input to the owner's refresh. This fails closed: an ID is re-fetched, -never sent past its real expiry. - -### 6.2 The `ts-eids-s` and `ts-eids-r` cookies - -Two server-owned cookies: - -- **`ts-eids-s`** (`__Host-ts-eids-s`) caches provider-collected IDs for owned - sources, holds positive entries stagers saved, and carries pending markers - and mirrored `ps`. -- **`ts-eids-r`** is a family of cookies, normally at most four, - `__Host-ts-eids-r-`, that hold removals and opt-outs not yet in KV. - Each response that produces a removal or opt-out sets a new one, with a - fresh random 8-hex-digit ``, holding only its own batch. No response - rewrites another's cookie, so neither a concurrent `/auction` rewriting - `ts-eids-s` nor another removal in flight can lose a batch. The one - rewrite is the writer's re-seal after a key rotation (below), which keeps - the same name and the same batch. Stagers never - delete one. They are deleted, by name, in three cases only: by the writer - navigation, after its phase 1 read shows a cookie's entries all applied - or the cookie fails to open; by any request whose KV read confirms a - tombstone; and by an orphan rotation whose create was `Written`. That keeps a capture's revocation, which - nothing can re-derive, safe (section 5.3). - -**Attributes.** Both use `Path=/; Secure; SameSite=Lax; HttpOnly`, with -`Max-Age=31536000`. The `ts-eids-r` cookies get the same lifetime so that a -batch never expires before a writer applies it; they are deleted once -applied, and their count and size caps bound them. The -`__Host-` prefix makes them host-only, with no `Domain`. They are caches of -KV, so per-host copies are fine. The repo's other server-set cookie, the -pull-complete marker, is host-only for the same reason: a `Domain` attribute -would let sibling hosts that share the passphrase replay it -(`pull_sync_marker.rs:198-201`). Host-only also stops a sibling subdomain from -reading the IDs or planting a competing cookie. A request carrying two -cookies of the same name is treated as carrying none. - -**Sealed, not just signed.** These cookies are stripped before the origin -and Prebid Server (section 6.2, outbound requests), but they still travel in -every request from the browser, through any intermediary that logs headers. -A signed cookie would show server-only IDs, such as UID2 tokens and LiveRamp -envelopes, in plain text there, and to the origin if the strip ever -regressed. So each value is sealed with AES-256-GCM: -`v1...`, with the nonce and ciphertext in -base64url without padding. `` names the sealing key: 1 to 8 -characters of `[a-z0-9]`, in plain text, because it is not secret. The keys -are dedicated secrets from the secret store, listed in -`[ec.identity.cookie_keys]` (section 10.1) and in `secret_fields()` (section -10.2). They are not derived from `ec.passphrase`. Each cookie's AES key is -derived from a keyring secret with a namespaced label -(`trusted-server/ts-eids-s/key/v1`, `trusted-server/ts-eids-r/key/v1`). The -nonce is 12 random bytes from the platform generator. The full EC ID is the -associated data, so it is authenticated but not stored: - -- A cookie copied from another visitor, or left over after an EC rotation, - fails to open and is ignored. -- The payload of `ts-eids-s` carries `kc` and `cf`. A `cf` that does not match - the current identity config makes its positive entries be ignored and - rebuilt from KV (section 5.6). `ts-eids-r` carries no `cf`. -- An old `ts-eids-s` replayed by its own visitor carries an old `kc`, which - forces a KV read (section 6.3), and the read overrides it. Replay can delay - a removal by at most `kv_revalidate_sec`. - -`ec_hash` is not used for binding, because it is the HMAC of the client IP -and is shared by every browser behind one address (`ec/generation.rs:83-93`). -A cookie that fails to open or decode is ignored and rebuilt by the next -request that reads KV. The failure is logged without the value. Sealing -needs an AES-GCM implementation, a new dependency (section 14), which the -UID2 provider needs anyway (section 12.3). - -**Key rotation.** TS seals every new cookie with the `current` key and -opens a cookie with any key in the keyring: the current key and the -previous keys still configured. A cookie whose `` is not in the -keyring fails to open, as any other cookie that fails to open. Rotation must -never drop an unapplied removal batch, so: - -- A writer whose phase 1 read finds a `ts-eids-r` cookie opened under a - previous key and not yet all applied re-seals it under the current key, in - place: same name, same batch, a fresh nonce and a new `Max-Age`. A partly - applied cookie is re-sealed whole. Every writer navigation reads KV when a - `ts-eids-r` cookie is present (section 6.3), so it re-seals or deletes - every one it carries. Stagers never re-seal. -- A `ts-eids-s` opened under a previous key is re-emitted under the current - key by the next request that reads KV and sets it (section 5.6). -- A rotation is two deploys: add the new key to the keyring, wait until - every POP serves that config, then make it `current`. A POP that does not - yet know a key ID would otherwise fail to open the new cookies, and its - writer would delete their batches. - -How long a previous key must stay configured: - -- **`ts-eids-s`:** at least `staged_max_age_sec` (default 7 days). By then - every staged entry sealed under it is too old to flush anyway, and a - cookie that fails to open is rebuilt from KV. A client that keeps - navigating moves to the current key within `kv_revalidate_sec`. -- **`ts-eids-r`:** one year, the cookies' `Max-Age`, after the key stops - being current. A client's batch moves to the current key at its first - writer navigation after the rotation, so a shorter retention covers every - client that navigates within it. Only the full year guarantees that no - browser can still hold a batch sealed under the removed key. - -Residual risk: a client that holds an unapplied batch and makes no writer -navigation before its key is removed loses that batch when it returns. Its -writer deletes the cookie as failing to open, and the removed ID can reach -that browser's bidstream again until the provider removes it again. A -client that never returns holds a batch nobody applies, rotation or not. An -emergency rotation that removes a compromised key at once accepts this loss -for every batch sealed under it. - -The hashes of section 9.3 keep their `ec.passphrase`-derived keys. - -**Why it is sealed.** Staged entries become KV writes, and sealing proves TS -wrote them. Without it, a visitor could plant an arbitrary ID in KV, where it -would also be sent from the server on routes without the cookie. Encryption -also keeps the IDs out of origin logs and browser extensions. - -**`ts-eids-s` payload.** Compact JSON. A source is written as its index into -the sorted list of configured sources, which `cf` fingerprints, so a config -change that reorders sources also invalidates the indexes. +that source, such as a browser-seeded one, is used only as the input to the +owner's refresh. This fails closed: an ID is re-fetched, never sent past +its real expiry. + +### 6.2 The `ts-eids-s` mirror + +`ts-eids-s` (`__Host-ts-eids-s`) is client-only. The shim writes it from +the mirror payload TS returns, and page scripts read it. The server never +sets, opens, parses or trusts it. Earlier revisions also had a family of +sealed, server-owned removal cookies, `ts-eids-r`; revision 15 removes them, +because every removal now reaches KV on the request that carries it +(section 5.3). + +**Readers.** The mirror is for page scripts that need the server IDs. This +spec names no consumer yet. Section 15 asks for one before rollout step 1 +ships the mirror; without one, step 1 ships without it. + +**Attributes.** `__Host-ts-eids-s`, with `Path=/; Secure; SameSite=Lax`, +and no `Domain`. It is not `HttpOnly`, because the shim writes it with +`document.cookie` and page scripts read it. Its `Max-Age` comes from the +mirror payload. + +**The mirror payload.** `/auction` returns it as `ext.ts_eids_s` in its +response (`ResponseExt`, `openrtb.rs:232`), and `/_ts/page-bids` as a +`ts_eids_s` member of its JSON body. It takes one of two shapes, or is +absent: ```json { - "v": 1, - "kc": 1760000000, - "cf": "1a2b3c4d", - "ids": [ - { - "s": 2, - "u": "ID5*example", - "w": "id5", - "t": 1760000000, - "e": 1767776000, - "r": 1760086400, - "a": 1, - "x": { "linkType": 2 }, - "pv": "3f2a9c1d04b7e6a8", - "sa": 1760000000, - "k": 0 - }, - { - "s": 4, - "u": "example-envelope", - "t": 1760000000, - "sa": 1760000000, - "b": 1, - "k": 0 - }, - { "s": 0, "d": 1 } - ], - "ps": { - "id5": { "n": 1760003600, "f": 1, "sg": 1 }, - "pull-ssp-example-com": { "p": 1760000000 } - } + "value": "eyJ2IjoxLCJlaWRzIjpbeyJzb3VyY2UiOiJleGFtcGxlLmNvbSIsInVpZHMiOlt7ImlkIjoiZXhhbXBsZS1hIiwiYXR5cGUiOjF9XX1dfQ", + "max_age": 86400 } ``` -| Field | Meaning | -| ------------- | ------------------------------------------------------------------------------------------ | -| `kc` | When a request last confirmed this cookie against the KV row | -| `cf` | Fingerprint of the identity config: providers, sources and owners | -| `m` | A request saw this EC's row missing. The next writer reads KV in phase 1 | -| `s` | The source, as an index into the configured source list | -| `u`…`x`, `rv` | Mirror the KV fields `uid`, `w`, `at`, `exp`, `ra`, `atype`, `ext`, `rv`. Staged: no `rv` | -| `pv` | The revision (`rv`) of the record this entry replaces. Absent: none | -| `sa` | When the entry was staged | -| `b` | `1` for a browser KV-gap entry (`FillMissing`) | -| `k` | `1` when the entry is known to be in KV, `0` when staged and not yet written | -| `d` | A stub: the entry was dropped for size and is still in KV | -| `ps.n/f/o/iv` | Mirror KV `ps`. `sg: 1` marks a staged change not yet written | -| `ps.oc` | A staged opt-out clear from a capture (section 8), with its time | -| `ps.p` | Pending marker: when an after-send dispatch happened whose result is not in the cookie yet | - -**`ts-eids-r` payload.** Removals as `{s, w, pv?, sa}`, where `s` is the -source domain itself, never an index: `ts-eids-r` carries no `cf`, so an -index could point at another source after a config change. An entry -without `pv` removes the source's record if `w` wrote it and acquired it at -or before `sa`. A tie goes to the removal, so an acquisition in the same -second never dismisses a revocation; an ID re-acquired in that second is -removed too and resolved again later. Opt-outs as `o: {: }`, and invalidated UIDs as -`iv: {: []}`. - -An entry is **applied**, and is skipped if the mutation meets it again, -when the live row shows: - -- a removal with `pv`: the record is gone or at another revision -- a removal without `pv`: the record is gone, written by another writer, or - acquired after `sa` -- an opt-out: KV's `ps.o.at` is at or after `at`, or KV's `ps.oc` is after - `at`, so a cleared opt-out is never re-applied. A clear in the same second - as the opt-out does not count: the opt-out still applies - -`iv` entries never keep a cookie alive: they are applied with the cookie's -other entries and then dropped with it. The writer's phase 1 read deletes -each cookie whose entries are all applied, and any `__Host-ts-eids-r-` -cookie that fails to open, for example one sealed under a key no longer in -the keyring. A cookie that is only partly applied is kept whole, and its -applied entries are skipped when re-applied. One opened under a previous -key is re-sealed (above). - -**Size.** Each cookie's sealed value is at most 512 bytes. The `v1.` -prefix, a key ID of at most 8 characters, the base64url nonce and their -separators take at most 29 bytes, base64url turns every 3 bytes into 4, and -the tag takes 16, so a cookie holds at most 346 bytes of payload JSON. A -removal costs 56 bytes plus its source domain -and provider ID: about four fit with typical names (a 15-byte source and an -8-byte provider), and one with a 253-byte source domain. An `iv` hash costs -19 bytes, and an opt-out about 15 bytes plus the provider ID. - -A batch is never coarsened to fit. It is not merged into another -response's cookie, its removals are not widened, its `iv` hashes are not -dropped, and an invalidation never becomes an opt-out. - -**Overflow.** A response to a real browser writes its batch to KV before -send, instead of setting a cookie, when the batch does not fit one cookie -or the request already carries four openable `ts-eids-r` cookies. A -response to a client that fails `looks_like_browser()` never makes this -write (section 4.7): it sets the batch in new cookies of its own, split as -needed, as when the write fails. This removal overflow -write is the second page-load exception to G4 (section 5.1). It reads the -row and applies only the batch's removals, opt-outs and `iv` hashes under -the rules of section 5.4, with a CAS and the withdrawal path's five -attempts. Like that path, it re-reads and tries again only when the CAS -generation no longer matches. Any other failure, Fastly's per-key limit -included, ends the write at once. When it lands, or the read shows the batch -already applied, no cookie is set. When it fails, the response sets the -batch in new cookies of its own, split across as many 512-byte cookies as it -needs, so the batch is never lost. - -Four is where TS stops relying on cookies, not a hard cap. Concurrent -responses each count from their own inbound snapshot, so five responses -that start with no removal cookie can set five. The count passes four only -while removal batches race or overflow writes fail, and the next writer -navigation deletes every applied cookie. A metric reports the number and -total size of `ts-eids-r` cookies per request (section 14). - -**Staged `ps` changes** merge at flush: `n` takes the later value, `f` the -cookie's, `iv` is the union, and an opt-out from `ts-eids-r` is kept unless -it is applied or stale under the rules above. A staged clear `ps.oc` is -dropped from `ts-eids-s` once KV holds an equal or later `ps.oc`. -A mirrored `o` in `ts-eids-s` yields to KV (section 4.7). - -**Pending markers.** While a marker is unanswered, the provider is not -dispatched again, and the next writer request reads KV in phase 1 (section -6.3). Until that read, the provider's cookie entries are not used in -auctions, because its after-send result may have removed them. A KV read -answers a marker when any of these hold, and the reader then clears it and -copies the result into the cookie, removals included: - -- `ps.n` for the provider is at or after `p`. -- `ps.o` for the provider is set. -- A source the provider claims has `at` at or after `p`, or lost the value - the cookie holds. -- The live row makes the provider ineligible (section 5.6). - -A marker that a read does not answer by `p + timeout_ms + pending_grace_sec` -is dropped: the dispatch was cancelled or its flush failed, and the provider -is eligible again. A marker cannot block a provider for good. - -**Size cap.** At most 2,048 bytes for the sealed value of `ts-eids-s`. When it -is over, entries are dropped in this order: - -1. Entries with `k: 1` become stubs `{s, d: 1}`. They are still in KV. A - stub makes the next request that needs the source read KV. -2. `ps` entries with no marker and no staged change. They are still in KV. -3. Provider entries with `k: 0`, in ascending `cookie_priority` (section - 10.1). Each one lost costs a later provider call. -4. Browser KV-gap entries (`b: 1`), last, because nothing re-derives one - unless the browser sends it on another `/auction` (section 5.3). - -Pending markers are never dropped. Each drop is logged with the provider and -source. - -**Who writes them.** - -- **`ts-eids-s`:** the writer request's HTML response only when needed - (section 5.2, phase 3), because any HTML response with `Set-Cookie` becomes - `private, max-age=0` (section 3.4); and stagers, whenever they save an - entry or their KV read changes the cookie (section 5.6). Their JSON - responses are already private. -- **`ts-eids-r`:** any response that produces a removal or opt-out, on any - route, including the unusable-consent path (section 5.3), adds a new - cookie for its batch, unless its overflow write landed (above; a client - that fails the real-browser check never makes one); and the writer - navigation's HTML response deletes the cookies its phase 1 read showed - applied and re-seals, in place, those opened under a previous key that - are not yet all applied. -- **A consent withdrawal, or any KV read that finds a tombstone,** sets - `Max-Age=0` on `ts-eids-s`. Once the tombstone is confirmed (written by - this request, or read), the `ts-eids-r` cookies are pointless and are - deleted too, because the tombstone removes everything. If the - withdrawal's tombstone write failed, they are kept. - -The browser shim never reads or writes either cookie. Both join `ts-ec`, -`ts-eids` and sharedId in the list of identity cookies the template cache -refuses as cache keys (`cookies/template_cache_policy.rs:35`), so no operator -can split the shared template cache per visitor with them. +```json +{ "clear": true } +``` + +- **`value`** when the request read a live, consenting row and the servable + server set is not empty. `max_age` is the smaller of 86,400 seconds, the + lifetime of `ts-eids`, and the time until the earliest `exp` in the + mirrored set, so the mirror never outlives an ID. +- **`clear`** when the request's consent denies EID persistence, when its + KV read finds a tombstone, `consent.ok = false` or a missing row, and + when the servable server set is empty. +- **Absent** when the request read no KV, such as a `/_ts/page-bids` call + whose auction does not run. The shim leaves the cookie as it is. + +The shim writes `value` as the cookie value with the given `Max-Age`, and +expires the cookie on `clear`. It skips the write when the value is +unchanged. + +`value` is the base64url encoding, without padding, of compact JSON: +`{"v": 1, "eids": [...]}`. The example above encodes one entry, +`{"source": "example.com", "uids": [{"id": "example-a", "atype": 1}]}`. +`eids` holds the server-side entries the auction merge would send for this +request (section 7.1), before browser precedence: every KV record that +passes the servability check, for sources with `bidstream_enabled = true`, +after `gate_eids_by_consent`. Each entry is an OpenRTB EID with its +`source`, its UIDs with `atype` and `ext`, and its EID-level fields, +provenance defaults included, exactly as bidders receive it. It never holds +`ps`, `ps.st`, refresh tokens or any other provider state, and no revision, +writer or timing field. + +**Size cap.** At most 2,048 bytes for the cookie value, so at most about +1,536 bytes of payload JSON. When the set does not fit, whole sources are +left out, in ascending `cookie_priority` (section 10.1) and then largest +first, and each omission is logged with the provider and source. A source's +entry set is mirrored whole or not at all, never trimmed. A left-out source +still reaches bidders through TS from KV. + +**Who writes it.** Only the shim. TS never sets `ts-eids-s`, not even on a +JSON response, so writer HTML responses stay cacheable. A withdrawal or +tombstone met on a writer navigation is cleared from the mirror at the +page's next `/auction` or `/_ts/page-bids`, whose consent check or KV read +returns `clear`. Until then the mirror can hold IDs TS no longer serves. +TS's own bidstream is unaffected, because the server never reads the +mirror. + +**No feedback.** The shim never copies mirror IDs into `ts-eids` or the +`/auction` body. A page script that does so anyway, for example by writing +them into Prebid's `ortb2` config, turns them into ordinary browser +candidates. Browser ingest stores them with `FillMissing`, one entry with +one UID, under today's rules (section 2.2), and the provider denials of +section 7.1 filter them before the auction, for owned and unowned sources +alike. So a stale copy of an ID a provider removed is not sent. A mirror ID +never becomes an owner write and never carries its writer, timing or +EID-level fields back into KV. + +**Template cache.** `ts-eids-s` joins `ts-ec`, `ts-eids` and sharedId in +the list of identity cookies the template cache refuses as cache keys +(`cookies/template_cache_policy.rs:35`), so no operator can split the +shared template cache per visitor with it. **Outbound requests.** TS's identity cookies never reach third parties. One -list, `TS_IDENTITY_COOKIE_NAMES` (`ts-ec`, `ts-eids`, `__Host-ts-eids-s`, -every `__Host-ts-eids-r-` cookie by prefix, `ts-ec-pull-complete`), is stripped from Prebid Server -requests and never added to proxy or provider requests. The publisher origin -is not a third party, but it gets only the small ones: +list, `TS_IDENTITY_COOKIE_NAMES` (`ts-ec`, `ts-eids`, `__Host-ts-eids-s`), +is stripped from Prebid Server requests and never added to proxy or +provider requests. The publisher origin is not a third party, but it gets +only the small one: - **Publisher origin. ID cookies stripped.** The publisher proxy removes - `ts-eids`, `__Host-ts-eids-s` and every `__Host-ts-eids-r-` cookie - (`TS_ORIGIN_STRIPPED_COOKIE_NAMES`) from the `Cookie` header it forwards - (`publisher.rs:3124-3147`), and drops the header when nothing is left. - `ts-ec` and `ts-ec-pull-complete` still reach the origin, as today, - because publishers' servers may read `ts-ec`. Without the strip the ID cookies - add up to about 5.5 KB to every origin request, and up to 2 KB more while - removal batches wait, and common origin - servers reject a header line over about 8 KB with 400 or 431, which would - fail the page itself. Stripping `ts-eids` from the origin is a behaviour + `ts-eids` and `__Host-ts-eids-s` (`TS_ORIGIN_STRIPPED_COOKIE_NAMES`) from + the `Cookie` header it forwards (`publisher.rs:3124-3147`), and drops the + header when nothing is left. `ts-ec` still reaches the origin, as today, + because publishers' servers may read it. Without the strip the ID cookies + add up to about 5 KB to every origin request, and common origin servers + reject a header line over about 8 KB with 400 or 431, which would fail + the page itself. Stripping `ts-eids` from the origin is a behaviour change: an origin that read it must read `x-ts-eids` or ask TS instead. A metric still reports the inbound `Cookie` header size (section 14). - **Prebid Server.** Stripped in the Prebid transport before the @@ -2403,86 +2359,34 @@ is not a third party, but it gets only the small ones: header (section 4.7). The strip iterates every `Cookie` field, as `gpt_diagnostics.rs:394-416` does, -and drops the header when nothing is left. Only the routes of section 6.3 -open the cookies, lazily, at most once per request, keeping the result on -`EcContext`. Assets and other routes never decrypt them: the origin and -Prebid Server strips match cookie names only. The derived keys, `cf` and -the source index are computed once per settings load, not per request. +and drops the header when nothing is left. It matches cookie names only; no +route opens `ts-eids-s`. The configured source list is computed once per +settings load, not per request. ### 6.3 Read order -For a request with a valid EC that uses identity (the writer request, -`/auction`, `/_ts/page-bids`, a proxy capture, or any request dispatching a -hold-mode provider): - -1. If `ts-eids-s` opens, use its IDs for owned sources (skipping expired - entries, stubs and the entries of providers with an unanswered marker), - its staged entries, and its `ps` for eligibility (section 4.7). Whether - or not it opens, apply `ts-eids-r` on top: its removals hide the records - they name, from the cookie view and from KV, and its opt-outs block their - providers. -2. **The writer's phase 1** reads the KV snapshot when any of these hold: - - `ts-eids-s` is missing, fails to open, or has a `cf` that does not - match - - `now - kc` exceeds `kv_revalidate_sec` - - `m` is set - - any `ts-eids-r` cookie is present, so the writer can delete the - applied ones - - any pending marker is unanswered - - the navigation's auction needs a source that is only a stub - - a hold-mode provider is due, so the read is its pre-disclosure - revalidation - - an after-send provider without private state is eligible on the - cookie view and has a candidate source that is unowned or only a stub, - so its dry run sees what KV holds (section 5.2). A valid pull-complete - marker removes a generic-pull provider's sources from the cookie-view - candidates, as today, and step 3's cookie-only `ps.n` keeps this from - firing on every navigation. - - the navigation runs an auction that needs the row, as today's preload - decides (`auction_needs_row`, `publisher.rs:3110-3117`). Today that is - every auction navigation with unowned bidstream sources, because the - cookie caches no unowned values - - consent is withdrawn and the tombstone path needs the row, as today - - `/auction` and `/_ts/page-bids` keep their existing reads and gain no new - triggers. - -3. **Any request that read KV** merges the two as below and sets `kc = now`. - It re-emits the cookie only when the merge changed it or the inbound `kc` - is older than half of `kv_revalidate_sec` (section 5.6): - - A tombstone, or `consent.ok = false`, empties the server set and expires - the cookie. - - A missing row, unless this request is about to recover it, replaces - `ts-eids-s` with a minimal cookie holding only `m: 1`: no positive - entries and no markers (section 5.6). - - For each owned source, the KV record wins, with one exception: a staged - provider entry that `OwnerWrite` would still apply (its `pv` matches, or - it has no `pv` and KV holds only a value the provider did not write) - stays `k: 0`. - - A `k: 1` entry whose source KV no longer holds, or holds at a different - revision, is dropped or replaced. - - A staged entry KV already holds becomes `k: 1`. A `b: 1` entry is dropped - once KV holds any value for its source. - - A stub is replaced by the KV record. - - Answered pending markers are cleared, and `m` is cleared when the row is - present. - - A provider the row shows ineligible because all its unowned candidate - sources are present gets `ps.n = now + refresh_after_sec` in the - cookie only, stored with the KV `n` it saw. A later read that shows a - different KV `n` (an `invalidate` sets `n = now`, for example) drops - the cookie-only value. Owned sources need no such value: the cookie - caches them with their `ra` and `exp`, so the cookie view already knows - when they are due. - - On the writer only, each `ts-eids-r` cookie whose entries the row shows - all applied is deleted, and each one opened under a previous key that - is not yet all applied is re-sealed under the current key (section - 6.2). Stagers leave the `ts-eids-r` cookies alone. - - The writer's HTML response carries the result only under phase 3's rules, - which include "the phase 1 read changed the cookie". - -Missing cookie, KV present: the writer request reads KV and rebuilds the -cookie. The visitor loses nothing. +The server-side view of provider IDs is the KV row, and nothing else. For a +request with a valid EC that uses identity (the writer request, `/auction`, +`/_ts/page-bids`, or a capture removal write): + +1. **The writer's phase 1** reads the KV snapshot on every returning-EC + navigation. The read finds orphans (section 5.5), feeds eligibility + (section 4.7), filters browser candidates (section 7.1) and serves the + withdrawal path when consent is withdrawn. +2. `/auction` and `/_ts/page-bids` keep their existing reads and gain no new + triggers. A stager navigation that runs an auction, such as a prefetch, + preloads as today, without the pull-complete marker's skip (section + 10.4). +3. **Any request that read KV** uses the row as the base: + - A tombstone, or `consent.ok = false`, empties the server set. An + `/auction` or `/_ts/page-bids` response returns `clear` for the + mirror. + - A missing row, unless this request is about to recover it, empties the + server set and stores nothing (section 5.6). + +A request that read no KV, such as a proxied Lockr API call with no +revocation, has no server set. It needs none, because it persists no +positive ID (section 5.3). ## 7. Auction merge and ownership @@ -2490,8 +2394,8 @@ cookie. The visitor loses nothing. `merge_auction_eids` (`auction/endpoints.rs:505`), and the navigation and page-bids merge (`publisher.rs:5547-5568`), take a third input: the server -set. It holds IDs from `ts-eids-s` (including staged entries) and KV, with -expired entries, stubs and removed entries left out. The server set requires +set. It holds the KV records the request read, with expired records left +out. It never holds anything from `ts-eids-s`. The server set requires what today's KV path requires: `ec_allowed()`, and a live consenting row whenever the request read KV (`endpoints.rs:446-460`). Each server-side ID must also pass, for this request, the consent policy of the provider that @@ -2502,7 +2406,7 @@ not disclosed once that provider's policy fails. A record whose writer is disabled or no longer configured has no policy left to check, so it is never served, and it counts as absent for `FillMissing` and for unowned `Initial`, so the browser or another provider can replace it. A record -without `w` (browser ingest or legacy) needs the owner's policy for an owned +without `w` (browser ingest or version 1) needs the owner's policy for an owned source, and only the generic gates for an unowned one, as today. For each source, the value sent to bidders is: @@ -2518,45 +2422,90 @@ current value. It may be stale, which is the problem the EID fidelity work fixed (D9). For a provider whose IDs expire, a browser-seeded record is never sent from the server at all (section 6.1). -A server value uses its stored `atype`, else `[[ec.partners]].openrtb_atype`, -and its stored UID-level `ext`. A value the provider wrote also gets the -EID-level `inserter`, `matcher` and `mm` the provider declared for that -source (`with_provenance`, section 4.3). A browser value keeps the browser's -own EID-level fields, as the EID fidelity work requires. - -**Owner denials apply to browser candidates.** Before the precedence above, -the browser's candidate for an owned source passes the owner's denials, read -from KV and from `ts-eids-r` alike, so they hold before the removal reaches -KV: - -- A browser UID whose keyed hash is in the owner's `ps.iv` is dropped. Once - the owner invalidates UID A, an A still in the `/auction` body or in - `ts-eids` is not sent. -- While the owner has a sticky `ps.o`, the browser's value for every source - that provider owns is dropped, until the opt-out clears (section 8). +**How a server value goes out.** A server value is the source's whole entry +set, one outbound EID per stored entry, in stored order, after the browser +entries (EID fidelity spec §4.5). A one-UID record without EID-level +fields is one entry, exactly as today (`ec/eids.rs:64-76`). For each entry: + +- Each UID uses its stored `atype`, else `[[ec.partners]].openrtb_atype`, + and its stored UID-level `ext`. +- The entry keeps its stored EID-level fields. For a value a provider + wrote, an entry that sets none of `inserter`, `matcher` and `mm` gets + that provider's `with_provenance` default for the source (section 4.3). +- An owner's valid value replaces every browser entry for the source, all + of them when the browser repeated the source. Its set is never mixed + with browser entries. + +A browser value keeps the browser's own entries and EID-level fields, as the +EID fidelity work requires. A browser-seeded or version 1 record is one +entry with one UID, so everything that predates this spec goes out +unchanged. + +This needs the output path to carry a set. Today `resolve_partner_ids` +returns `ResolvedPartnerId { uid, openrtb_atype }` and `to_eids` builds one +EID from each (`ec/eids.rs:21-28`, `:64-78`), for the auction +(`auction/endpoints.rs:465-466`) and the admin view +(`ec/admin.rs:485-486`). `ResolvedPartnerId` gains the record's servable +entry set and its writer, and `to_eids` emits one `Eid` per stored entry +under the rules above. `build_eids_header` and both callers use it +otherwise unchanged. For a one-UID record without `w`, the output is +byte-identical to today's. + +**Provider denials apply to browser candidates.** Before the precedence +above, the browser's candidates pass the denials of the providers that +claim their source, read from KV: + +- A browser UID whose hash is in the `ps.iv` or `ps.ov` of a provider that + claims the + source is dropped, for owned and unowned sources alike. Once a provider + invalidates UID A, or removes it with an opt-out, an A still in the + `/auction` body or in `ts-eids` is not sent. +- While the owner of a source has a sticky `ps.o`, the browser's value for + that source is dropped, until the opt-out clears (section 8). The same + holds for every source of a capture-only provider with a sticky `ps.o`, + owned or not, because its IDs reach TS only through the browser. These run before `gate_eids_by_consent`, which knows nothing of -provider-specific rejections. `ps.iv` keeps the four most recent hashes, so -an older rejected UID can be sent again once evicted. Unowned sources keep -today's browser-first rule: no provider can veto them. +provider-specific rejections. `ps.iv` and `ps.ov` each keep the 128 most recent hashes, so an older rejected UID can be sent again once evicted. A removed set adds a +hash for each of its UIDs (section 7.2), so a browser entry is filtered UID +by UID, and an entry left with no UID is dropped. Every writer navigation, +`/auction` and `/_ts/page-bids` auction reads KV and filters fully. +Unowned sources otherwise keep today's browser-first rule: a provider can +veto only the UIDs it removed itself, never the source. + +The browser's candidates include any `ts-eids-s` mirror ID a page script +fed back through Prebid (section 6.2). It is filtered like any other +browser value, so a stale mirror copy of an ID a provider has since removed +is not sent. Sources with `bidstream_enabled = false` are left out, as `resolve_partner_ids` does today (`ec/eids.rs:36`). The merged set is consent-gated (`gate_eids_by_consent`) and reported in -`x-ts-eids` exactly as today, with the same 4 KB cap. `x-ts-eids` therefore -shows server-only IDs. The calling script can read it, and so can any page -script that wraps `fetch`, so `HttpOnly` on `ts-eids-s` does not hide server -IDs from the page. The header stays for validation and troubleshooting, and -this exposure is accepted. - -One shared servability check covers expiry, the rollback rule, the -writer's and owner's consent policies, and opt-outs: a record whose writer has `ps.o` set, in KV or -in `ts-eids-r`, is never served, even before the removal reaches KV. `resolve_partner_ids` (`ec/eids.rs:36`), the admin view +`x-ts-eids` exactly as today, with the same 4 KB cap. A full entry set can +reach that cap sooner, and `x-ts-eids-truncated` is set as today. The +header's truncation stays per entry, as today (`encode_eids_header`, +`ec/eids.rs:106`), because the header also carries the browser's entries, +whose truncation must not change. So the header can hold part of a set; +`user.eids` always holds it whole. `x-ts-eids` therefore shows server-only +IDs, EID-level fields included. The calling script can read it, and so can +any page script that wraps `fetch`. The `ts-eids-s` mirror shows the same +server-side entries to every page script (section 6.2). The header stays +for validation and troubleshooting, and both exposures are accepted +(section 9.3). + +One shared servability check covers expiry, the rollback rule, the writer's +and owner's consent policies, and opt-outs: a record whose writer has `ps.o` +set is never served. `resolve_partner_ids` (`ec/eids.rs:36`), the admin view (`ec/admin.rs:485`) and `/_ts/api/v1/identify` all use it. `/identify` reads `entry.ids` directly today (`ec/identify.rs:99-103`), so it switches to the -check. The stale module comment at `ec/eids.rs:1-5` is corrected. So expired -IDs are never returned by any of them (G7). +check. Its response shape does not change: `uid` is the first UID of the +first entry of the source's record, when that record passes the +servability check, and `eid` is built from that UID with the partner's +`openrtb_atype`, as today (`ec/identify.rs:127-135`). A partner calling +`/identify` gets the one UID it gets today, never a provider's full set. The +stale module comment at `ec/eids.rs:1-5` is corrected. So expired IDs are +never returned by any of them (G7). **Consent for `/identify`.** The policies are checked against the visitor's consent on this request, never against the row's stored `consent.tcf` and @@ -2584,15 +2533,27 @@ no consent update. - **Owned source.** The owner is authoritative. It refreshes when due (section 4.7, step 4), and its valid value wins in the auction. Browser - ingest and KV-gap entries may only fill the source when it is absent or - its record is expired. + ingest may only fill the source when it is absent or its record is + expired. That includes a `ts-eids-s` mirror ID the page fed back + (section 6.2). That seeds it: the owner treats a value it did not write as due, and refreshes it at once. No other provider may claim the source (section 10.2). -- **Invalidated values are not re-seeded.** When the owner invalidates a - value, its keyed hash goes into `ps.iv`. Browser ingest and KV-gap entries - skip a UID in `ps.iv`, so the browser cannot re-seed the value the owner - just rejected, and no invalidate-reseed-refresh loop starts. +- **Removed values are not re-seeded.** When a provider's invalidation or + opt-out removes a record, the hash of each UID in the removed set goes + into that provider's `ps.iv` or, for an opt-out, its `ps.ov` (section + 5.4). Browser ingest skips a UID in the `ps.iv` or `ps.ov` of any provider + that claims the source, so the browser cannot + re-seed a value a provider just rejected, and no + invalidate-reseed-refresh loop starts. +- **A provider replaces its own set whole.** For an owned source, a result + that names it replaces every stored entry for it, so the owner drops one + entry by returning the set without it. For an unowned source a result + only fills an absent or expired record (`FillMissing`) and never + replaces one, so an unowned provider cannot drop an entry. Browser ingest + never adds an entry to a + provider's set: it fills a source only when it is absent or expired, as + today. - **Unowned source.** Any provider claiming it, and the browser, write with `FillMissing` only. Unowned sources are never refreshed. This is exactly today's pull-sync contract. The auction is browser-first. @@ -2602,54 +2563,59 @@ no consent update. ## 8. Errors and outcomes -Every effect below goes into the mutation (writer request), or a stager's -cookies: positive changes into `ts-eids-s`, removals and opt-outs into -`ts-eids-r`. - -| Result or error | Stored IDs | `ps` | Log | -| ------------------------------------------------------------------------------ | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | ----- | -| `Resolution::found` | `OwnerWrite` or `FillMissing` per source | `f = 0`; `st` replaced or cleared when the resolution says so | debug | -| A stateful result whose IDs or state lost to a newer write (section 5.4) | unchanged; the result's removals still apply | unchanged | debug | -| `replacing_state` oversized or unserializable | no IDs stored; the result's removals still apply | `st` unchanged; `f` and `n` as for `Malformed` | warn | -| `Resolution::no_match` | unchanged | `n = now + retry_in` (default 24 h, max 7 d) | debug | -| `Resolution::invalidate`, `also_invalidate` | `OwnerRemove` with the revision the request read | UID hash added to `iv`; `n = now`: eligible for `Initial` | info | -| `Resolution::opted_out` | `WriterRemove` for every record this provider wrote, owned or unowned | `o = {at: now, fp}`, `st` cleared | info | -| `with_retry_in` on any resolution | as above | `n` is at least `now + retry_in` | debug | -| `request` returns `None`, or `InputUnavailable` | unchanged | `n = now + refresh_after_sec` | debug | -| A recorded `InvalidState`, whatever the provider returned | as the result says | `st` cleared, `f` unchanged, `n = now + refresh_after_sec` unless the result sets `n` | warn | -| Hold-mode budget spent before the answer | unchanged | `n = now + hold_retry_sec` | debug | -| `Rejected` with a 4xx other than 429, `Config`, `UndeclaredInput`, `Malformed` | unchanged | `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)` | warn | -| `Rejected` with 429 or 5xx, `Transport`, timeout | unchanged | `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`; breaker `record` | info | -| Breaker open | not dispatched | unchanged | debug | -| Capture: any result or error | IDs and removals as above | only `o`; never `n`, `f` or the breaker | debug | -| Capture with `capture_enabled = false` | positive IDs discarded; removals as above | only `o` | debug | -| Captured `opted_out`, `revoke_scope = "global"`, real browser | the withdrawal tombstone removes the row (section 5.1) | removed with the row | info | +Every resolving provider runs on the writer, so every effect below goes +into the writer's mutation. A capture, the only provider result on a +stager, stores its removals, opt-out and clear through the capture removal +write (section 5.3), and nothing else. + +| Result or error | Stored IDs | `ps` | Log | +| ------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | ----- | +| `Resolution::found` | `OwnerWrite` or `FillMissing` per source | `f = 0`; `st` replaced or cleared when the resolution says so | debug | +| A stateful result whose IDs or state lost to a newer write (section 5.4) | unchanged; the result's removals still apply | unchanged | debug | +| `replacing_state` oversized or unserializable | no IDs stored; the result's removals still apply | `st` unchanged; `f` and `n` as for `Malformed` | warn | +| `Resolution::no_match` | unchanged | `n = now + retry_in` (default 24 h, max 7 d) | debug | +| `Resolution::invalidate`, `also_invalidate` | `OwnerRemove` of the whole set, with the revision the request read | on apply, a hash per UID of the removed set added to `iv`; `n = now` | info | +| `Resolution::opted_out` | `WriterRemove` for every record this provider wrote, owned or unowned; for a capture-only provider, also every record of its sources no provider wrote | `o = {at: now, fp}`, `st` cleared; a hash per UID of each removed record added to `ov` | info | +| `with_retry_in` on any resolution | as above | `n` is at least `now + retry_in` | debug | +| `request` returns `None`, or `InputUnavailable` | unchanged | `n = now + refresh_after_sec` | debug | +| A recorded `InvalidState`, whatever the provider returned | as the result says | `st` cleared, `f` unchanged, `n = now + refresh_after_sec` unless the result sets `n` | warn | +| Hold-mode budget spent before the answer | unchanged | `n = now + hold_retry_sec` | debug | +| `Rejected` with a 4xx other than 429, `Config`, `UndeclaredInput`, `Malformed` | unchanged | `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)` | warn | +| `Rejected` with 429 or 5xx, `Transport`, timeout | unchanged | `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`; breaker `record` | info | +| Breaker open | not dispatched | unchanged | debug | +| Capture: any result or error | removals as above; positive IDs counted in `capture_unpersisted`, never stored | only `o`, `oc`, `iv` and `ov`; never `n`, `f` or the breaker | debug | +| Captured `opted_out`, `revoke_scope = "global"`, real browser | the withdrawal tombstone removes the row (section 5.1) | removed with the row | info | **Sticky opt-outs.** `ps.o = {at, fp}` records a provider-reported opt-out and the consent fingerprint at that time. Server-side calls do not carry the vendor's own opt-out cookie, so `o` is TS's only record of it. Applying it -removes every record the provider wrote (section 5.4), and while it is set -no record that provider wrote is served (section 7.1). It is cleared only by -an affirmative re-grant for this provider: +removes every record the provider wrote, and for a capture-only provider +every record of its sources that no provider wrote (section 5.4). While it +is set no record that provider wrote is served, and the browser's values it +denies are dropped (section 7.1). It is cleared only by an affirmative +re-grant for this provider, and clearing it empties `ps.ov` (section 5.4): - **A provider with a GVL ID:** a TC string whose `LastUpdated` is after `at` and that grants the provider's vendor and purposes. - **Any provider, under GPP:** the applicable US section showing no sale, sharing or targeted-advertising opt-out for the signals the provider's - policy reads, where the fingerprint at `at` showed one. GPP carries no - creation time, so the clear is by section values. + policy reads, where `fp` at `at` recorded one. GPP carries no creation + time, so the clear is by section values: the opt-out clears when `fp` + records at least one US opt-out bit (sale, sharing, targeted + advertising, GPC) and the current consent shows none of the opt-outs + `fp` records. - **A capture-only provider without a GVL ID, such as Lockr:** a later 2xx - capture that returns IDs, under consenting cookie and body consent. That - is the vendor issuing again after the visitor re-consented through it. - The capture's proxy request is a stager, so it stages `ps.oc` beside the - IDs in `ts-eids-s`, and the next writer applies the clear before the IDs - (section 5.4). If a concurrent `/auction` overwrites that cookie, both - are lost together, and the next capture stages them again. + capture that returns IDs, under the request's consent. That is the + vendor issuing again after the visitor re-consented through it. The + capture removal write stores the clear as `ps.oc = now` on that request, + under the ordering of section 5.4 (section 5.3). The captured IDs + themselves are not stored; once the clear is in KV, the browser path + carries them through browser ingest. - **An operator,** through the admin endpoints. A change of jurisdiction, a newly present GPP string, or a privacy signal the -provider relaxes never clears it. A clear is written to KV, and the -browser's mirrored copy yields to it (section 4.7). A consent withdrawal's +provider relaxes never clears it. A clear is written to KV as `ps.oc`. A +consent withdrawal's tombstone removes everything, provider opt-outs included: after it, the visitor starts again with a new EC. @@ -2659,21 +2625,22 @@ visitor starts again with a new EC. `opted_out` from a real browser runs today's consent-withdrawal tombstone path on that request instead (section 5.1): the row is tombstoned, the EC cookie is cleared, and everything is removed, provider opt-outs included. -It uses the withdrawal's existing exception, so G4 keeps exactly two -page-load exceptions. From a client that fails `looks_like_browser()`, a -global revoke is staged as a provider opt-out in that client's own -`ts-eids-r` (section 4.7), so a replayed `ts-ec` cannot tombstone another -browser's row. An opt-out from `resolves` is always provider-scoped. +It uses the withdrawal's existing exception, so it adds no page-load exception to G4. From a client that fails `looks_like_browser()`, a +global revoke is dropped (section 4.7), so a replayed `ts-ec` cannot +tombstone another browser's row. An opt-out from `resolves` is always +provider-scoped. -The **consent fingerprint** is the first 16 hex digits of an HMAC, keyed from -`ec.passphrase`, over the consent signals the provider's policy reads: +The **consent fingerprint** `fp` is a bitmask, written as hex, of the +consent signals the provider's policy reads: - its TCF purpose bits and vendor bit - the US sale, sharing and targeted-advertising opt-outs it does not relax - GPC and DNT, unless relaxed -It is not computed over the raw consent string, which changes whenever a CMP -re-saves it. +It is stored in plain form: the same row already holds the raw consent +strings (`KvConsent`, `ec/kv_types.rs:69-80`), so a keyed hash would protect +nothing, and a plain bitmask can be compared bit by bit. It is not the raw +consent string, which changes whenever a CMP re-saves it. Rules: @@ -2721,17 +2688,19 @@ Rules: - **Before disclosure:** the live-row read and eligibility recomputation described in section 4.7. - **At write:** the CAS on that read's generation. -- **At flush and on every cookie write:** staged positive entries, and IDs - written into `ts-eids-s`, pass the generic gate and the writing provider's - policy against the current consent (section 5.3). Removals and opt-outs - never need consent to apply (section 5.6). +- **In the mirror payload:** the IDs returned for `ts-eids-s` pass the + servability check and the generic gate against the request's consent + (section 6.2). Removals and opt-outs never need consent to apply + (section 5.6). - **In the bidstream and `/identify`:** server-side IDs pass their writer's policy and, for an owned source, the owner's, against the request's visitor consent (section 7.1). Owner denials also filter the browser's candidates. - **On withdrawal:** the tombstone is written synchronously on whichever - request carries it, and the response expires `ts-eids-s` (section 5.1). A - global revoke from a real browser takes the same path (section 8). + request carries it (section 5.1), and the next `/auction` or + `/_ts/page-bids` response tells the shim to clear `ts-eids-s` (section + 6.2). A global revoke from a real browser takes the same path (section + 8). ### 9.2 Outbound requests @@ -2760,39 +2729,48 @@ Rules: - Provider errors are logged with `current_context()` only, never with `{:?}` of the report, whose attachments could carry a URL or body. - `ProviderRequest`, `ProviderResponse`, `ResolveInput`, `CaptureInput`, - `StoredId`, `ResolvedId`, `ProviderConsent` and `HashedEmail` have - redacting `Debug` implementations. `ConsentContext` gets one too, because - it derives `Debug` over the raw consent strings today + `StoredId`, `ResolvedEid`, `ResolvedUid`, `ProviderConsent` and + `HashedEmail` have redacting `Debug` implementations. `ConsentContext` + gets one too, because it derives `Debug` over the raw consent strings today (`consent/types.rs:109`). - The Lockr proxy logs only the path of the upstream URL, not its query (`integrations/lockr.rs:202-204`). -- Hashes of identifiers (the rate-limit visitor key, `pv`, `ps.iv`, the - consent fingerprint) are HMACs with keys derived from `ec.passphrase`, so - they cannot be linked to an EC ID without the key. They do not move to the - rotating cookie keys (section 6.2). `ps.iv`, the consent fingerprint and - the legacy revision hash a `pv` can hold are stored and compared on later - requests, so each rotation would break them: a rejected UID could be - re-seeded, an opt-out's fingerprint would never match again, and a staged - removal with a legacy `pv` would look applied and be skipped. The visitor - rate-limit key would survive a rotation, because its windows last 60 - seconds, but it stays with the others. -- Cookie sealing keys come from the secret store and are never logged. A - key ID may be logged. +- The rate-limit visitor key is an HMAC of the full EC ID with a key + derived from `ec.passphrase`, because it leaves the row for the rate + limiter. Its windows last 60 seconds, so a passphrase rotation costs + nothing. Nothing stored in the row is keyed: `ps.iv` and `ps.ov` hold unkeyed UID hashes (section 6.1) and the consent fingerprint is a plain bitmask + (section 8), because the same row already holds the raw UIDs and consent + strings. So a passphrase rotation breaks nothing stored. - `CaptureInput` never exposes request bodies or request headers. Proxied - requests can contain an email or HEM. TS forwards them, and reads from a - body only the consent pointers a descriptor declares (section 4.7). It - never reads anything else for identity. + requests can contain an email or HEM. TS forwards them, and never reads a + proxied request body for identity. - Provider state stays in KV. It is never sent to bidders, put in a cookie or returned by the admin lookup. +- A provider's EID-level `ext` and UID-level `ext` are vendor data, such as + segments or match metadata. They are stored in KV and never in `ts-eids`, + which keeps dropping EID-level `ext` (EID fidelity spec §4.3). They reach + bidders, `x-ts-eids` and the `ts-eids-s` mirror under the same consent + gates as the UIDs. A vendor spec that stores such data states what it + holds, as part of its privacy review. +- **Server-only IDs are readable by page scripts.** `ts-eids-s` is not + `HttpOnly` and is not sealed, so every script on the page can read the + servable server set, including UID2 advertising tokens and LiveRamp + envelopes. `x-ts-eids` already exposes the same IDs to the calling script + and to any script that wraps `fetch` (section 7.1), so the mirror adds no + new class of reader. The mirror holds only what bidders receive: never + `ps.st`, refresh tokens or other provider state (section 6.2). It is + stripped from origin and Prebid Server requests like the other ID + cookies. A vendor spec whose IDs must not reach page scripts says so, and + the operator gives its source `bidstream_enabled = false` or keeps the + provider off. ### 9.4 Caching -Responses that set `ts-eids-s` follow the same cookie-privacy rule as every -cookie-bearing response: `private, max-age=0`, with edge-cache headers -removed (`response_privacy.rs:122-145`). The Fastly terminal hook re-applies -it after late mutations (`adapter-fastly/src/main.rs:745`). Section 5.2 keeps -`ts-eids-s` off returning-visitor HTML except when something is due, changed -or stale. +This spec adds no `Set-Cookie` to any response. TS never sets `ts-eids-s`: +the shim writes it from JSON responses that are already private (section +6.2). Returning-visitor HTML keeps today's cacheability, and removing the +pull-complete marker (section 10.4) takes away the one `Set-Cookie` a +returning-visitor navigation could carry today (section 3.4). ## 10. Configuration @@ -2801,7 +2779,7 @@ or stale. ```toml # Coordinator policy: vendor-neutral, one place for every timing choice. [ec.identity] -# Replaces ec.pull_sync_concurrency. The old key is accepted and deprecated. +# Replaces ec.pull_sync_concurrency, which is removed (section 10.2). max_concurrency = 4 # Shared wall-clock budget for all hold-mode providers, in milliseconds. # Default 100 (section 4.7 explains why). @@ -2811,22 +2789,18 @@ hold_budget_ms = 100 hold_retry_sec = 300 # Per provider response, before and after decoding. max_response_bytes = 65536 -# Seconds a pending marker outlives its provider's timeout. Default 300. -pending_grace_sec = 300 -# Staged cookie entries older than this are not written. Default 604800. -staged_max_age_sec = 604800 -# A cookie not confirmed against KV for this long forces a KV read. -# Must be below the 24-hour tombstone TTL. Default 3600. -kv_revalidate_sec = 3600 [ec.identity.providers.id5] -# "hold" or "after_send". Default "after_send". ID5 keeps private state, so -# it runs only on the writer navigation: before the auction when that -# navigation runs one, after send when it does not (section 4.7, step 8). +# "hold" or "after_send". Default "after_send". Providers run only on +# writer navigations. Hold mode runs before the navigation's auction, and +# after send when the navigation runs none (section 4.7, step 7). mode = "hold" -# Hold mode: min(timeout_ms, remaining hold budget). Default: hold_budget_ms -# in hold mode, 1000 after send. +# Hold dispatch before the auction: min(timeout_ms, remaining hold budget). +# Default: hold_budget_ms. timeout_ms = 80 +# Any dispatch after send, including a hold-mode provider on a navigation +# that runs no auction. Default 1000. +after_send_timeout_ms = 1000 # Narrows the sources this provider may write. Default: every source its # descriptor declares. Narrowed here to the one source registered below. sources = ["id5-sync.com"] @@ -2839,31 +2813,19 @@ max_calls_per_minute = 1 # timeouts, 429s or 5xx responses happen within one 60-second window. # Default 50. breaker_failures_per_minute = 50 -# Lower is dropped first when ts-eids-s exceeds its cap. Default 0. +# Lower is left out of the ts-eids-s mirror first when it exceeds its cap. +# Default 0. cookie_priority = 10 [ec.identity.providers.lockr] -# Only for providers that declare `captures`. Default false: captured -# positive IDs are discarded and logged at debug; removals still apply. -capture_enabled = true +# Required for a capture-only provider: its opt-out removes browser-ingested +# records of every source listed here (section 5.4). +sources = ["id5-sync.com"] # Only for providers that declare `captures`. "provider" or "global". # Default "provider": a captured opt-out removes this provider's records. # "global" runs the consent-withdrawal tombstone instead (section 8). revoke_scope = "provider" -# Keys that seal ts-eids-s and ts-eids-r (section 6.2). No default: -# required when `ec.ec_store` is set. -[ec.identity.cookie_keys] -# The key ID that seals new cookies. It must be one of `keys`. -current = "k2" -# The current key and every previous key still accepted for opening. Each -# `secret` is a secret-store key name at rest, resolved at startup to 32 -# bytes in base64. -keys = [ - { id = "k2", secret = "ts-identity-cookie-key-k2" }, - { id = "k1", secret = "ts-identity-cookie-key-k1" }, -] - # Source policy: the existing table, with one new field. [[ec.partners]] name = "ID5" @@ -2879,10 +2841,8 @@ enabled = true partner_id = 1234 ``` -`[ec.identity]` and each `[ec.identity.providers.]` table are optional, -except `[ec.identity.cookie_keys]`, which is required when `ec.ec_store` is -set, because the cookies cannot be sealed without it. A provider with no -table uses the defaults. +`[ec.identity]` and each `[ec.identity.providers.]` table are optional. +A provider with no table uses the defaults. ### 10.2 Validation @@ -2902,7 +2862,9 @@ each provider's `sources` narrowing before checking ownership. Each of these is an error: - `owner` names a provider that is not enabled. -- `owner` names a provider whose descriptor does not include that source. +- `owner` names a provider whose descriptor does not include that source, + or a capture-only provider, which never writes a record in v1 (section + 4.7). - A descriptor fails to build (section 4.3). - An enabled provider claims a source with no `[[ec.partners]]` entry. - An enabled provider claims a source owned by a different provider. Use @@ -2910,27 +2872,26 @@ Each of these is an error: `id5-sync.com`, Lockr is narrowed to the sources no native provider owns. - `sources` names a source the provider's descriptor does not declare, or narrows the provider to no source at all. +- A capture-only provider has no `sources` setting. Its opt-out removes + browser-ingested records of every source it keeps, including IDs other + vendors' own modules produced (section 5.4), so the operator must list the + sources it is the delivery path for; the default of every declared source + is not allowed. - Two providers have the same ID (section 10.4). - A capture rule lies outside the registering integration's proxy routes. `IntegrationRegistry::with_plan` reports it at startup (section 4.3). - `[ec.identity.providers.]` names an unknown provider. -- `mode` is not `hold` or `after_send`. A provider with state runs only on - the writer request in either mode: in hold mode on writer navigations that - run an auction, and after send on those that do not (section 4.7, step 8). +- `mode` is not `hold` or `after_send`. - More than 32 providers are enabled, generic pull partners included. A KV row holds `ps` for at most 32 (section 6.1). - `timeout_ms` is greater than `hold_budget_ms` for a hold-mode provider. - `hold_budget_ms` is over 1,000, `max_concurrency` is 0, - `refresh_after_sec` is 0, or `kv_revalidate_sec` is not below 86,400. -- `capture_enabled` or `revoke_scope` is set for a provider whose - descriptor declares no `captures`, or `revoke_scope` is not `provider` or - `global`. -- `ec.ec_store` is set and `[ec.identity.cookie_keys]` is missing. -- `current` is not one of `keys`, two keys share an `id`, or an `id` is not - 1 to 8 characters of `[a-z0-9]`. -- At startup, after secret resolution, a key's secret does not decode from - base64 to exactly 32 bytes. Deploy validation sees only key names, so this - check runs in `validate_settings_for_runtime`. + `refresh_after_sec` is 0, or `after_send_timeout_ms` is 0 or over 10,000. +- `revoke_scope` is set for a provider whose descriptor declares no + `captures`, or is not `provider` or `global`. +- A removed key is set: `ec.pull_sync_concurrency`, or + `pull_sync_ttl_sec` or `pull_sync_rate_limit` on an `[[ec.partners]]` + entry. The error names the replacement (sections 10.1, 10.4). Validation never calls a provider. Live approvals, such as a LiveRamp placement or an approved origin, stay on the rollout checklist of each vendor @@ -2941,9 +2902,7 @@ secret, are `Redacted` config fields. At rest the config holds their secret-store key names, and they are resolved in memory before runtime settings are built (`secret_resolution.rs`). Each identity integration adds its secret paths to `TrustedServerAppConfig::secret_fields()` -(`config.rs:136-175`). The cookie keys' secrets are listed there too, at -`ec.identity.cookie_keys.keys[].secret`, as an optional path like the -partner tokens. `build_descriptor` must never read a secret field, +(`config.rs:136-175`). `build_descriptor` must never read a secret field, because deploy validation runs before resolution and would see key names. The harness's `check_descriptor_ignores_secrets` builds the descriptor from a config whose secrets are placeholder key names and from one with resolved @@ -2957,7 +2916,8 @@ secrets, so a descriptor that depends on a secret fails the test (section (`integrations/registry.rs:663`), and `.try_with_identity_provider(Result>)`, which wraps a constructor's result for the common case (section 4.6). An -integration can register more than one provider. The registry exposes `identity_providers()` to build the +integration can register more than one provider. The registry exposes +`identity_providers()` to build the coordinator. `handle_proxy` offers a proxied response to every provider whose descriptor has a capture rule matching it (section 4.7). A compile-time guard, `const _: Option<&dyn IdentityProvider> = None;`, keeps the trait @@ -2979,8 +2939,6 @@ Unchanged: - Fill-missing only, unless the operator sets `owner`. - The hourly rotation of partner order, which keeps dispatch fair under `max_concurrency` (`pull_sync.rs:143-156`). -- The pull-complete marker, which stays the signal that every pull partner - already has an ID (section 6.3). Its descriptor, built from the `[[ec.partners]]` entry: @@ -2990,12 +2948,9 @@ Its descriptor, built from the `[[ec.partners]]` entry: - A consent policy that blocks exactly what pull sync blocks today. The migration parity tests (section 13) pin it. - Default `after_send`, and unowned. -- Optional `owner = "pull-"` makes it authoritative, and the legacy - `pull_sync_ttl_sec` becomes its `refresh_after_sec`. -- `pull_sync_rate_limit` (per hour) maps to - `max_calls_per_minute = max(1, ceil(pull_sync_rate_limit / 60))`, which is - what the limiter enforces today. A `[ec.identity.providers.]` table - overrides it. +- Optional `owner = "pull-"` makes it authoritative. +- Its pacing comes from `[ec.identity.providers.pull-]`, like any + provider's (section 10.1). What changes: @@ -3010,6 +2965,16 @@ What changes: - The rate-limit key moves from `pull::` (`pull_sync.rs:373`) to the per-visitor key of section 4.7. Counters reset at deploy. +- `pull_sync_ttl_sec` and `pull_sync_rate_limit` are removed from + `[[ec.partners]]` (`settings.rs:430-439`). Their roles move to + `refresh_after_sec` and `max_calls_per_minute`. Today's hourly limit + already rounds to a per-minute one (section 3.4), so an operator who set + `pull_sync_rate_limit = N` sets `max_calls_per_minute = ceil(N / 60)`. +- The pull-complete marker (`pull_sync_marker.rs`) is removed. Every + returning writer navigation reads KV in phase 1 (section 6.3), so the + marker saves no read. TS stops setting and reading `ts-ec-pull-complete`; + a cookie a browser still holds expires within its one-hour `Max-Age` + (`pull_sync_marker.rs:22`). `dispatch_pull_sync` and `build_pull_sync_context` are removed. `run_edgezero_pull_sync_after_send` (`adapter-fastly/src/main.rs:618`) @@ -3030,7 +2995,8 @@ KV identity runs only on Fastly. Axum, Cloudflare and Spin build an `EcContext` for consent-gated routes, but have no `KvIdentityGraph`, no EC finalize and no EC generation (`publisher.rs:6896-6899`). -This design changes nothing on them except the Prebid Server and origin cookie strips, which +This design changes nothing on them except the Prebid Server and origin +cookie strips, which apply on every adapter. Providers are never dispatched there, and their `/auction` and `/_ts/page-bids` keep resolving client EIDs without KV. When EC identity is ported to them (#777 rows 52 to 54), phase 4 runs after send @@ -3045,7 +3011,8 @@ framework does not depend on any detail in them. Endpoints, payloads, consent parameters, size limits and semantics such as what Lockr's revocation covers are decided in each vendor's implementation plan, which replaces any shape marked "assumed" below. `IntegrationConfig` impls are -omitted; they match section 4.6. Endpoints come from operator config, shown with +omitted; they match section 4.6. Endpoints come from operator config, shown +with `example.com` hosts. Section 12.5 lists what writing them changed in the API. @@ -3185,10 +3152,9 @@ impl IdentityProvider for LiveRampProvider { }; let lifetime_days = if input.region()?.is("US", "CA") { 15 } else { 30 }; - let id = ResolvedId::new(source, envelope.value) - .with_atype(3) + let eid = ResolvedEid::new(source, ResolvedUid::new(envelope.value).with_atype(3)) .with_expires_in(Duration::from_secs(lifetime_days * 86_400)); - Ok(Resolution::found([id])) + Ok(Resolution::found([eid])) } } ``` @@ -3215,12 +3181,14 @@ What this shows: ### 12.2 Lockr (passive capture) -Lockr exercises capture from several exact paths, a revocation path that -must work after consent is denied, consent signals carried in the request -body, vendor key names mapped to sources, expiry in milliseconds, and -EID-level provenance. The paths and payloads come from the earlier Lockr -capture design (`spec/lockr-identity-capture`, its section 2), measured on a -live page. +Lockr exercises capture from several exact paths, a revocation path that must +work after consent is denied, an opt-out cleared by the vendor issuing +again, vendor key names mapped to sources, expiry in milliseconds, and a +complete +EID: every UID and the EID-level fields as the vendor returned them, with a +declared default for fields a response omits. The paths and payloads come from +the earlier Lockr capture design (`spec/lockr-identity-capture`, its section +2), measured on a live page. ```rust const LOCKR_API: &str = "/integrations/lockr/api/publisher/app/v2/identityLockr"; @@ -3235,8 +3203,7 @@ const KEY_SOURCES: &[(&str, &str)] = &[ impl IdentityProviderConfig for LockrConfig { fn build_descriptor(&self) -> Result> { - let mut captures = Captures::new() - .with_request_consent(["/consentString", "/gppString", "/ccpaString"]); + let mut captures = Captures::new(); for endpoint in ["page-view", "generate-tokens", "refresh-tokens", "sync-no-hem-ids", "revoke-consent"] { captures = captures.with_path(Method::POST, &format!("{LOCKR_API}/{endpoint}")); } @@ -3249,7 +3216,7 @@ impl IdentityProviderConfig for LockrConfig { builder .with_provenance( &"id5-sync.com".parse()?, - EidProvenance::new().with_inserter("Lockr-For-Publishers").with_matcher("").with_mm(2), + EidProvenance::new().with_inserter("Lockr-For-Publishers").with_mm(2), ) .build() } @@ -3283,6 +3250,10 @@ struct LockrEid { source: String, #[serde(default)] uids: Vec, + inserter: Option, + matcher: Option, + mm: Option, + ext: Option, } #[derive(Deserialize)] @@ -3324,21 +3295,41 @@ impl IdentityProvider for LockrProvider { return None; } - let uid = entry.eid.uids.first()?; - let mut id = ResolvedId::new(source, uid.id.clone()); - if let Some(atype) = uid.atype { - id = id.with_atype(atype); + let mut uids = entry.eid.uids.iter().map(resolved_uid); + let mut eid = ResolvedEid::new(source, uids.next()?); + for uid in uids { + eid = eid.with_uid(uid); + } + if let Some(inserter) = &entry.eid.inserter { + eid = eid.with_inserter(inserter); + } + if let Some(matcher) = &entry.eid.matcher { + eid = eid.with_matcher(matcher); } - if let Some(ext) = &uid.ext { - id = id.with_ext(ext.clone()); + if let Some(mm) = entry.eid.mm { + eid = eid.with_mm(mm); + } + if let Some(ext) = &entry.eid.ext { + eid = eid.with_ext(ext.clone()); } if let Some(at) = expires { - id = id.with_expires_at(at); + eid = eid.with_expires_at(at); } - Some(id) + Some(eid) }))) } } + +fn resolved_uid(uid: &LockrUid) -> ResolvedUid { + let mut resolved = ResolvedUid::new(uid.id.clone()); + if let Some(atype) = uid.atype { + resolved = resolved.with_atype(atype); + } + if let Some(ext) = &uid.ext { + resolved = resolved.with_ext(ext.clone()); + } + resolved +} ``` What this shows: @@ -3348,24 +3339,33 @@ What this shows: - **Revocation outlives consent.** `revoke-consent` arrives after the visitor has denied consent, so the request's consent gate would block it. A capture's removals must apply whatever the consent (section 12.5). It is - provider-scoped by default (`revoke_scope = "provider"`): it removes - Lockr-written records only, never the EC. An operator who sets + provider-scoped by default (`revoke_scope = "provider"`): it removes the + records of Lockr's sources that Lockr or browser ingest wrote, and blocks + the browser's values for them until Lockr issues again, never the EC. + Lockr never writes a record in v1, so without the browser-ingest part the + revoke would remove nothing (section 5.4). An operator who sets `revoke_scope = "global"` makes it a full withdrawal that tombstones the row, from real browsers only (section 8). Lockr's plan settles which scope Lockr's revocation means. -- **Capture sees a token once.** Lockr returns a token only when it first - issues one. On a reload its SDK sends the saved ID with `noGenerate`, and - the response carries no new token (#1246). So capture sees an ID only at - its first issue, and the browser path carries it on later page views: - Lockr writes the ID into Prebid's `ortb2` config, which the EID fidelity - work forwards in the `/auction` body and `ts-eids`. -- **Consent in the request body.** Lockr's SDK sends its own consent strings - in the body. The descriptor names them, the coordinator extracts and - decodes only those fields, and capture is gated by the stricter of the - cookie consent and the body consent. The provider never sees the body. -- **Provenance.** Lockr's ID5 EID carries `inserter`, `matcher` and `mm`, - which bidders read. The descriptor declares them once per source, and the - coordinator adds them to every server-side value Lockr wrote. +- **Capture sees a token once, and v1 does not keep it.** Lockr returns a + token only when it first issues one. On a reload its SDK sends the saved + ID with `noGenerate`, and the response carries no new token (#1246). + Lockr's API calls are stagers, so the captured IDs are discarded and + counted in `capture_unpersisted` (section 5.3). The browser path carries + the UID browser ingest picks for each registered source (section 2.2) to + KV: Lockr writes the ID into + Prebid's `ortb2` config, which the EID fidelity work forwards in the + `/auction` body and `ts-eids`, and the next writer ingests it (section + 2.3). +- **Complete EIDs.** Lockr's ID5 EID carries `inserter`, `matcher` and `mm`, + which bidders read, and can carry more than one UID. The capture copies + every UID and every EID-level field the response holds, so a stored set + is what Lockr issued, under the limits of section 4.5. In v1 capture + stores no set; the full set matters once a durable path exists (section + 15). The descriptor's + `with_provenance` stays as the default for a response entry that sets + none of `inserter`, `matcher` and `mm`; an entry that sets any of them + is Lockr's complete statement (section 4.3). - **Milliseconds.** `UnixTime::from_millis` converts Lockr's expiry, and a token already expired at capture time is skipped. - **Optional expiry.** Lockr gives an expiry only for tokens listed in @@ -3378,10 +3378,13 @@ What this shows: output, not a vendor's input. - **The opt-out lasts until Lockr issues again.** Lockr has no GVL ID for a TC string to re-grant, so its `ps.o` is cleared by a later successful - token capture under consenting cookie and body consent (section 8). -- **Ownership.** Lockr claims `liveramp.com` and `uidapi.com` too. Where native - providers own those, the operator narrows Lockr with `sources` (section - 10.2). + token capture under the request's consent, which the capture removal + write stores (section 8). +- **Ownership and narrowing.** Lockr claims `liveramp.com` and `uidapi.com` + too. A capture-only provider must list its `sources` (section 10.2), so + the operator keeps only the sources Lockr delivers on that site. A Lockr + revoke then removes and blocks only those, not IDs the ID5 or LiveRamp + modules produced for a source Lockr is not narrowed to. ### 12.3 UID2 (secrets, crypto, private state) @@ -3520,12 +3523,12 @@ impl IdentityProvider for Uid2Provider { "optout" => Ok(Resolution::opted_out()), "success" => { let identity = reply.body.ok_or_else(IdentityProviderError::malformed)?; - let id = ResolvedId::new(input.primary_source(), identity.advertising_token) - .with_atype(3) + let uid = ResolvedUid::new(identity.advertising_token).with_atype(3); + let eid = ResolvedEid::new(input.primary_source(), uid) .with_expires_at(UnixTime::from_millis(identity.identity_expires)) .with_refresh_at(UnixTime::from_millis(identity.refresh_from)) .with_refresh_until(UnixTime::from_millis(identity.refresh_expires)); - Ok(Resolution::found([id]).replacing_state(&Uid2State { + Ok(Resolution::found([eid]).replacing_state(&Uid2State { refresh_token: identity.refresh_token, refresh_response_key: identity.refresh_response_key, refresh_expires: identity.refresh_expires, @@ -3549,8 +3552,6 @@ What this shows: the refresh token lives. `with_refresh_until` keeps the expired record as a refresh anchor until `refresh_expires` (section 5.4), and the provider still checks `refresh_expires` itself. -- **No dry run.** UID2 keeps private state, so it gets no dry run (section - 5.2), and its `request` always builds the real, encrypted call. - **Randomness.** The envelope needs a random IV and nonce. A provider has no safe source of its own in a pure method, so `ResolveInput::random_bytes` supplies them, and the harness makes them deterministic. @@ -3573,7 +3574,10 @@ What this shows: `ext.linkType` per UID. It declares three sources (`id5-sync.com`, the EUID source its responses name, and `true-link-id5-sync.com`, `id5IdSystem.js:40-41`, `:130-170`) and uses `also_invalidate` for a - declared source a response no longer returns. + declared source a response no longer returns. Each source's EID goes + through `ResolvedEid` with the UID-level `ext` and EID-level fields the + response gives, so a response that returns several UIDs or entries for a + source stores them all. - **Generic pull** is built in (section 10.4). - **A local provider**, one that needs no outbound call, does not fit v1. An example is a publisher login ID read from a first-party cookie and @@ -3586,25 +3590,29 @@ What this shows: Writing the examples found these gaps. Each is now in the API of sections 4 to 6: -| Gap | Found by | Change | -| ------------------------------------------------------------ | ----------------- | -------------------------------------------------------------------------------- | -| `response` cannot know what `request` sent | LiveRamp, UID2 | `ProviderRequest::try_with_call_context`, `ProviderResponse::call_context` (4.5) | -| No safe randomness in a pure method | UID2 | `ResolveInput::random_bytes`, deterministic in the harness (4.2, 4.4, 13) | -| Secrets have no stated path | UID2 | Secret fields in `secret_fields()`; `build_descriptor` never reads them (10.2) | -| Text bodies | UID2 | `post_text`, `success_text` (4.5) | -| State cap too small for refresh tokens | UID2 | 1,024 bytes (4.5, 6.1) | -| Expiry in milliseconds | Lockr, UID2 | `UnixTime::from_millis` (4.4) | -| Prefix capture is too broad | Lockr | `Captures::with_path(method, path)` (4.3) | -| Revocation arrives after consent is denied | Lockr | Capture removals bypass the consent gate (4.7) | -| Consent signals in the request body | Lockr | `Captures::with_request_consent`; the stricter consent wins (4.3, 4.7) | -| EID-level `inserter`, `matcher` and `mm` cannot be expressed | Lockr | `ProviderDescriptorBuilder::with_provenance`, applied on output (4.3, 7.1) | -| A source string from a response must match a declared source | Lockr | `SourceDomain::matches` normalizes before comparing (4.3) | -| An oversized UID-level `ext` discarded the whole ID | Lockr (ID5 `pba`) | The `ext` is dropped and the ID kept (4.5) | -| Region tests need country and subdivision | LiveRamp | `Region::country`, `Region::is` (4.4) | -| An expired record stranded refreshable providers | UID2, LiveRamp | `Refresh` covers expired records; the provider decides (4.4, 4.7) | -| The dry run repeated expensive work and could not see state | UID2 | No dry run for providers with private state (5.2) | -| Every example parsed its endpoint twice | all | `Resolves::with_endpoint(&str)`, checked by `build()` (4.3) | -| Capture had two ways to say "nothing" | Lockr | `capture` returns `Resolution`; `no_match` means nothing (4.2) | +| Gap | Found by | Change | +| ----------------------------------------------------------------------- | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `response` cannot know what `request` sent | LiveRamp, UID2 | `ProviderRequest::try_with_call_context`, `ProviderResponse::call_context` (4.5) | +| No safe randomness in a pure method | UID2 | `ResolveInput::random_bytes`, deterministic in the harness (4.2, 4.4, 13) | +| Secrets have no stated path | UID2 | Secret fields in `secret_fields()`; `build_descriptor` never reads them (10.2) | +| Text bodies | UID2 | `post_text`, `success_text` (4.5) | +| State cap too small for refresh tokens | UID2 | 1,024 bytes (4.5, 6.1) | +| Expiry in milliseconds | Lockr, UID2 | `UnixTime::from_millis` (4.4) | +| Prefix capture is too broad | Lockr | `Captures::with_path(method, path)` (4.3) | +| Revocation arrives after consent is denied | Lockr | Capture removals bypass the consent gate (4.7) | +| EID-level `inserter`, `matcher` and `mm` cannot be expressed | Lockr | `ProviderDescriptorBuilder::with_provenance`, applied on output (4.3, 7.1) | +| A source string from a response must match a declared source | Lockr | `SourceDomain::matches` normalizes before comparing (4.3) | +| An oversized UID-level `ext` discarded the whole ID | Lockr (ID5 `pba`) | The `ext` is dropped and the ID kept (4.5) | +| Region tests need country and subdivision | LiveRamp | `Region::country`, `Region::is` (4.4) | +| A provider could return one UID per source, with fixed EID-level fields | Lockr, ID5 | `ResolvedEid` and `ResolvedUid`: several entries and UIDs per source, EID-level fields per entry, stored and sent whole (4.5, 6.1, 7.1) | +| An expired record stranded refreshable providers | UID2, LiveRamp | `Refresh` covers expired records; the provider decides (4.4, 4.7) | +| Every example parsed its endpoint twice | all | `Resolves::with_endpoint(&str)`, checked by `build()` (4.3) | +| Capture had two ways to say "nothing" | Lockr | `capture` returns `Resolution`; `no_match` means nothing (4.2) | + +Revision 15 removed two earlier entries. Request-body consent for capture +(`Captures::with_request_consent`) only gated positive captures, which v1 +no longer stores. The dry run, and with it `is_dry_run`, saved one KV +re-read in the rare case a stateless provider skipped its call. ## 13. Testing @@ -3630,7 +3638,11 @@ assert_eq!( let resolution = harness .respond(StatusCode::OK, &json!({ "uid": "example-uid", "signature": "example-signature" })) .expect("should parse the response"); -assert_eq!(resolution.ids()[0].uid(), "example-uid", "should return the uid"); +assert_eq!( + resolution.eids()[0].uids()[0].id(), + "example-uid", + "should return the uid" +); ``` `harness.respond_raw(status, headers, bytes)` covers empty bodies such as a @@ -3644,13 +3656,14 @@ sees what production would: - **Eligibility.** It applies the consent policy (eligibility step 1) and step 4: sources whose kind's needs are missing are left out, and with none left `request()` returns `Ok(None)` without calling the provider. -- **Provenance.** `ProviderDescriptor::provenance` gives the EID-level - fields; the harness adds nothing of its own. +- **Provenance.** Applied only by `output_eids`, as the bidstream applies + it (section 7.1). The `Resolution` getters show what the provider set. The harness API, in the test-support module, gated by -`#[cfg(any(test, feature = "test-support"))]`. A dependent crate never sees -`cfg(test)`, so a provider in its own crate enables `test-support` as a -dev-dependency feature of `trusted-server-core`: +`#[cfg(any(test, feature = "test-utils"))]`, the crate's existing +test-helper feature. A dependent crate never sees `cfg(test)`, so a +provider in its own crate enables `test-utils` as a dev-dependency feature +of `trusted-server-core`: ```rust impl<'p> ProviderHarness<'p> { @@ -3669,7 +3682,6 @@ impl<'p> ProviderHarness<'p> { /// The fixed clock. Default: a constant test time. pub fn with_now(self, now: UnixTime) -> Self; pub fn with_random_seed(self, seed: u64) -> Self; - pub fn as_dry_run(self) -> Self; /// Calls `request` and keeps the built request and its call context. pub fn request(&mut self) -> Result, Report>; @@ -3691,18 +3703,25 @@ impl<'p> ProviderHarness<'p> { headers: &[(&str, &str)], body: &[u8], ) -> Result>; - /// Runs a capture as the coordinator does, consent extraction included. + /// Runs a capture as the coordinator does, response decoding included. pub fn capture(&self, fixture: &CaptureFixture) -> Result>; /// Resolves an expiry or refresh against the fixed clock. pub fn resolve(&self, when: When) -> UnixTime; /// The replacement state in a result, decoded, for assertions. pub fn state_of(&self, resolution: &Resolution) -> Option; + /// The EIDs bidders would see for a result: the section 4.5 rules, then + /// the descriptor's provenance defaults (section 7.1). + pub fn output_eids(&self, resolution: &Resolution) -> Vec; /// Runs only the cases passed in, never the harness's own inputs. A /// provider declared `with_private_state()` fails here with a pointer to /// `check_conformance_with_state`. pub fn check_conformance(&self, cases: &ConformanceCases); pub fn check_conformance_with_state(&self, cases: &ConformanceCases); + /// The same checks, returning the first failure instead of failing the + /// test. `check_conformance` wraps it. A panic aborts a `wasm32-wasip1` + /// test run, so the harness's own negative tests use this. + pub fn try_check_conformance(&self, cases: &ConformanceCases) -> Result<(), ConformanceFailure>; } /// Inputs for one harness run. The harness's own `with_*` methods delegate @@ -3742,7 +3761,6 @@ pub struct CaptureFixture { /* private */ } impl CaptureFixture { pub fn new(method: Method, path: &str) -> Self; - pub fn with_request_body(self, body: &serde_json::Value) -> Self; pub fn with_response(self, status: StatusCode, body: &serde_json::Value) -> Self; } @@ -3772,6 +3790,8 @@ impl ConformanceCases { /// Test-only constructors, behind the same gate. impl StoredId { pub fn for_test(source: &SourceDomain, uid: &str) -> Self; + /// A stored full set. `uid()` becomes its first UID. + pub fn with_eids(self, eids: Vec) -> Self; pub fn written_by(self, writer: &IdentityProviderId) -> Self; pub fn expiring_at(self, at: UnixTime) -> Self; } @@ -3801,24 +3821,25 @@ harness never guesses which setup makes a provider call out: - A provider without `with_private_state()` never returns state. - Requests go only to declared hosts, over HTTPS, with no `Cookie` header and with valid header names and values. -- Returned IDs are only for declared sources, at most one per source. -- `ResolvedId::expiry()` and `refresh()` report what the provider set, and a - fixture ID of a source marked expiring carries an expiry. +- Returned entries are only for declared sources, and no fixture's result + hits a limit of section 4.5: no dropped UID or field, no discarded entry + and no set over `MAX_ENTRY_SET_BYTES`. A vendor whose real responses can + exceed a limit says so in its spec; the fixture does not hide it. +- `ResolvedEid::expiry()` and `refresh()` report what the provider set, and + a fixture set of a source marked expiring carries an expiry. - `capture` never returns a state change. - For every case with state, the harness also injects a stored state that is a bare JSON string, which no struct decodes, and checks that the provider neither panics nor treats it as valid state. A `#[serde(default)]` state type that decodes anything fails here. That the coordinator then clears it is a coordinator test. -- During the dry run, `is_dry_run()` is true and the built request is - discarded unsent. `as_dry_run()` fails the test for a provider declared - `with_private_state()`, which never gets one. `harness.check_conformance_with_state::(&cases)` also checks that the state type round-trips through `replacing_state` and `input.state()`. `check_descriptor_ignores_secrets::(&placeholder, &resolved)` builds the descriptor from a config with placeholder secret names and from one with -resolved secrets, and fails if the two differ (section 10.2). Vendor tests add a +resolved secrets, and fails if the two differ (section 10.2). Vendor tests +add a fixture response for each outcome the vendor can return, and assert it through the `Resolution` getters. @@ -3828,11 +3849,8 @@ The harness also: reproducible - carries the built request's call context into `respond`, as the coordinator does -- takes capture fixtures with a request body, and checks that only the - declared consent pointers are read from it and that a stricter body - consent blocks positive IDs but not removals -- applies a provider's provenance, so a test can assert the EID-level fields - bidders will see +- applies a provider's provenance in `output_eids`, so a test can assert the + EID-level fields bidders will see The examples in section 12 are the first harness cases: LiveRamp's initial and refresh 204s, Lockr's page-view and revoke-consent, and UID2's success, @@ -3841,50 +3859,66 @@ refresh and opt-out. **Coordinator.** - An eligibility matrix: consent × ownership × per-source kind × due state × - inputs × mode × writer or stager × new or returning EC. -- The merged eligibility view: a cookie-only `no_match` or result stops a - re-call on the next `/auction`; KV's `o` blocks dispatch even when the - cookie lacks it. + inputs × mode × writer or stager × new or returning EC. No stager + dispatches a resolving provider in any cell. +- The eligibility view is the phase 1 KV row alone: `ts-eids-s` never + changes eligibility, whatever it holds. +- On a site whose auctions all go through `/auction`, a hold-mode provider + runs after send on the next writer navigation that runs no auction, its + answer reaches KV, and the following `/auction` serves it from KV without calling the provider. Moved after send, it waits `after_send_timeout_ms`, and a miss backs off as a timeout, not as a hold cut-off. +- A `no_match` or 4xx on a writer stores `ps.n`, so a run of `/auction` and + `/_ts/page-bids` calls in the same SPA session never calls the provider + again before `ps.n`. - `sources` narrowing, owner takeover of a browser-seeded record, sticky `ps.o` surviving a jurisdiction change and cleared only by a newer grant, and `ps.iv` stopping a re-seed. - Hold budget: a fake HTTP client with controlled latency shows the auction - starts at the budget, an answer is saved with `k: 0`, and a cut-off call - sets `hold_retry_sec` without counting a failure. -- A dry-run `None` sets no pending marker. A `None` or `InputUnavailable` + starts at the budget, an answer joins the writer's mutation, and a + cut-off call sets `hold_retry_sec` without counting a failure. +- `request` is called once per dispatch. A `None` or `InputUnavailable` waits `refresh_after_sec`. -- Pending markers block re-dispatch, hide that provider's cookie entries - until a read, are answered by each of the three rules (including a - removal), and are dropped past their deadline. +- After-send results commit in the same phase 4 write. Two overlapping + writer navigations for one EC do not both dispatch a provider within the + burst cap's window. - A navigation answered with a status other than 2xx or 304 dispatches no - after-send provider and keeps hold-mode answers in its cookie. + after-send provider, stores no positive hold-mode answer, and flushes + those answers' removals, opt-outs and backoff, so a `no_match` or 4xx on a + 404 page is not called again before its `ps.n`. - An `UndeclaredInput` is logged as `Config` and backs off. - Only transport failures, timeouts, 429 and 5xx feed the breaker. 4xx and `Malformed` from many visitors never open it. - The burst cap is checked after every other eligibility step. - Passive capture runs from `handle_proxy` only for providers with a capture rule matching the request, decodes gzip and brotli bodies under the caps, - reads only the declared consent pointers from a request body, and never - touches `n`, `f` or the breaker. -- With `capture_enabled = false`, a capture's positive IDs reach neither - cookie nor KV and are logged at debug without values, while its removals - and opt-outs still apply. -- A capture from a client that fails `looks_like_browser()` stores no - positive ID, puts its removals and opt-outs only in its own `ts-eids-r`, - never makes the removal overflow write, and turns a global revoke into a - provider opt-out. A later writer navigation from that client applies the - batch. A replayed `ts-ec` on the proxy changes no KV row. + never reads the request body, and never touches `n`, `f` or the breaker. +- A capture stores no positive ID and increments `capture_unpersisted` + without logging a value. Its removals, opt-out and any clear are written + by the capture removal write on that request, before send. +- The capture removal write retries only a CAS conflict, up to five + attempts, tries once more after send when those fail, and on a second + failure increments `removal_lost` without logging a value. It + writes nothing for a tombstone, a missing row or an already applied + removal, and it runs on the unusable-consent path. +- A writer navigation whose phase 1 snapshot predates a capture removal + write of the same page load fails its CAS and is not retried; the next + navigation re-derives its browser ingest. +- A capture from a client that fails `looks_like_browser()` writes nothing, + drops its removals, opt-outs and global revoke, and logs them without + values. A replayed `ts-ec` on the proxy changes no KV row. - With `revoke_scope = "global"`, a captured opt-out from a real browser tombstones the row and clears the EC cookie on that request, as a consent withdrawal does. With `"provider"`, it removes only that provider's - records. -- A Lockr `revoke-consent` captured on the proxy removes every Lockr-written - record, owned or unowned, including sources the cookie holds only as - stubs, and survives a concurrent `/auction` rewriting `ts-eids-s`. -- Providers declared `with_private_state()` never run on stagers. A due - stateful provider gets a marker without a dry run, phase 4 rebuilds its - request with the live state, and a writer navigation without an auction - runs it after send. + records and, for a capture-only provider, the records of its sources no + provider wrote. +- A Lockr-style `revoke-consent` captured on the proxy removes the + browser-ingested record of each of Lockr's narrowed sources, adds their + UIDs to `ps.ov`, and drops the browser's values for those sources. A later + token capture clears the opt-out and empties `ps.ov`, and the next writer + navigation ingests the same stable UID again. A source Lockr is not + narrowed to keeps its browser value throughout. +- A due stateful provider is dispatched in phase 4 with its request built + from the live state, and a writer navigation without an auction runs it + after send. - A provider's `ResolveKind` is `Refresh` for a stored record whether it is still valid or expired, and an expired record counts as absent for `FillMissing`. @@ -3907,143 +3941,131 @@ refresh and opt-out. - A decodable GPP string with no applicable section, the header-only `DBAA` shape, does not block a strict policy where TCF does not apply. - A targeted-advertising opt-out without a sale opt-out produces no - server-side IDs in `user.eids`, no flush of that provider's entries, and no - IDs written into the cookie. + server-side IDs in `user.eids` and none in the `ts-eids-s` mirror + payload. +- The consent fingerprint is a plain bitmask: a GPP opt-out clears exactly + when `fp` records a US opt-out bit and the current consent shows none of + the opt-outs `fp` records, whatever else changed. **One-write invariant.** A new test store counts writes by mode (Add, -`IfGenerationMatch`, Overwrite) and is shared through a test-support feature +`IfGenerationMatch`, Overwrite) and is shared through the `test-utils` feature so adapter tests can use it. `KvIdentityGraph::counting` counts reads only (`ec/kv.rs:1596-1605`, `ec/kv_backend.rs:205-221`). -| Scenario | KV writes | -| -------------------------------------------------------------------- | ---------------------------------------------------- | -| New visitor, navigation | 1 (create, before send) | -| New visitor whose first ID collides | 1 (create with a re-minted ID) | -| Returning visitor, changes pending | 1 | -| Returning visitor, nothing changed | 0 | -| 304 revalidation with changes pending | 1 | -| Returning EC, navigation answered with 3xx, 4xx or 5xx | 0 | -| New visitor, prefetch or prerender navigation | 1 (create, before send) | -| Returning visitor, prefetch or prerender navigation | 0 | -| New visitor, client without `Sec-Fetch-Dest` | 1 (create, before send) | -| Returning visitor, iframe HTML or no `Sec-Fetch-Dest` | 0 | -| Integration-proxy navigation that mints an EC | 1 (create) | -| `/auction`, `/_ts/page-bids`, `/identify`, Lockr API proxy, assets | 0 | -| Consent withdrawal, on any request | CAS tombstone (up to 5 attempts) plus marker, per ID | -| Global revoke captured from a real browser | as a consent withdrawal | -| Removal batch from a real browser that overflows the removal cookies | CAS removal write before send (up to 5 attempts) | -| Capture removal, opt-out or global revoke from a non-browser client | 0 | -| CAS conflict or per-key limit on the flush | 1 attempt, not retried | +| Scenario | KV writes | +| ------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| New visitor, navigation | 1 (create, before send) | +| New visitor whose first ID collides | 1 (create with a re-minted ID) | +| Returning visitor, changes pending | 1 | +| Returning visitor, nothing changed | 0 | +| 304 revalidation with changes pending | 1 | +| Returning EC, navigation answered with 3xx, 4xx or 5xx | 0, or 1 when its hold-mode calls returned removals or backoff | +| New visitor, prefetch or prerender navigation | 1 (create, before send) | +| Returning visitor, prefetch or prerender navigation | 0 | +| New visitor, client without `Sec-Fetch-Dest` | 1 (create, before send) | +| Returning visitor, iframe HTML or no `Sec-Fetch-Dest` | 0 | +| Integration-proxy navigation that mints an EC | 1 (create) | +| `/auction`, `/_ts/page-bids`, `/identify`, assets | 0 | +| Lockr API proxy, no revocation | 0 | +| Captured revocation from a real browser | CAS removal write (up to 5 attempts), once more after send | +| Consent withdrawal, on any request | CAS tombstone (up to 5 attempts) plus marker, per ID; once more after send | +| Global revoke captured from a real browser | as a consent withdrawal | +| Capture removal, opt-out or global revoke from a non-browser client | 0 | +| CAS conflict or per-key limit on the flush | 1 attempt; its removals, if any, then written alone (up to 5 attempts) | **Withdrawal and removal.** - A withdrawal on `/auction` after the navigation's send leaves a tombstone, and the navigation's phase 4 drops its mutation and calls no provider. +- A withdrawal tombstone whose attempts fail before send is tried once more + after send. - After `opted_out` or `invalidate`, the removed ID stops reaching bidders - from the cookie by the next request that reads KV, and within - `kv_revalidate_sec` at most. -- A KV read that finds a tombstone expires `ts-eids-s`. -- A deletion tombstone makes every browser drop its cookie copy within - `kv_revalidate_sec`. - -**Failed flush.** After a refused write, staged entries and hold-mode answers -are still `k: 0` in the cookie, and the next navigation writes them. - -**Reads.** A returning visitor with a valid, recently confirmed `ts-eids-s`, -nothing due, staged or pending, and no registered source in `ts-eids` pays -zero KV reads on the navigation. `/auction` and `/_ts/page-bids` pay their -existing one read and no more. + through TS on the next request, and the next `/auction` or + `/_ts/page-bids` response leaves it out of the mirror payload. +- An `/auction` or `/_ts/page-bids` response whose KV read finds a + tombstone, a missing row or `consent.ok = false`, or whose consent denies + EID persistence, returns `clear` for the mirror, and the shim expires + `ts-eids-s`. +- A deletion tombstone is honoured at the visitor's next writer navigation + or `/auction`, whatever `ts-eids-s` holds. +- No response sets a server identity cookie other than `ts-ec`. + +**Failed flush.** After a refused write, the mutation's removals, opt-outs and clears are written alone with up to five CAS attempts, and a mutation without them makes no second write. When those attempts fail too, `removal_lost` is incremented without logging a value. The next writer navigation re-derives browser ingest from `ts-eids` and calls the still-due providers again within the burst cap. + +**Reads.** Every returning writer navigation pays exactly one KV read in +phase 1, plus the phase 4 re-read when an after-send provider dispatches. +`/auction` and `/_ts/page-bids` pay their existing one read and no more. No +request opens or parses `ts-eids-s`. **No lost EIDs (G10).** -- A registered source present only in the `/auction` body (beyond the - cookie trim) is saved as a KV gap and written by the next navigation. -- A gap too large for the cookie is saved again by the next `/auction` and - eventually written. -- Every source today's `/auction` ingest would write reaches KV. +- The shim's `ts-eids` trim keeps every registered source before any + unregistered one, and keeps the EID fidelity order within each group. +- A registered source present in the `/auction` body but beyond today's + trim is kept in `ts-eids` and written by the next navigation. +- When the registered sources alone exceed the trim, only those the trim + drops are lost, and the test pins which. +- Every source today's `/auction` ingest would write reaches KV whenever + the registered sources fit the trim. - An EID present only in Prebid's `ortb2` config, as Lockr writes its ID5, reaches `ts-eids` and KV. **Orphans and collisions (section 5.5).** -- A row deleted out of band is found within `kv_revalidate_sec` and recovered - by rotation before send. -- A stager that sees the row missing sets `m`, and the next writer recovers. +- A row deleted out of band is found by the next writer navigation's phase + 1 read and recovered by rotation before send. +- A stager that sees the row missing stores nothing and returns `clear` for + the mirror, and the next writer recovers. - A collision on the first create re-mints before any cookie is set. **Storage.** -- `KvPartnerId` and `ps` round-trip, with and without the new fields. -- The rollback rule: a stripped or browser-seeded record of an expiring - owned source is never sent from the server. +- A schema version 1 row reads as version 2, each `uid` as one entry with + one UID, and its next write stores version 2. A version 2 row and `ps` + round-trip. +- An owned expiring source's record is served from the server only when + its `w` is the owner and its `exp` has not passed. - Revision-checked `OwnerRemove` and `OwnerWrite` lose to a newer write. - Re-applying changes on a later writer keeps the other tab's change. + Re-applying changes on a later writer keeps the other tab's change. An + `OwnerRemove` of a version 1 record matches `Revision::Unset` until the + record's next write. - Two refreshes of an unchanged UID in the same second get different `rv`, so an invalidation read before the second refresh does not remove it. - An opt-out and a clear in the same second leave the provider opted out. + A clear with no opt-out to clear writes nothing, and a clear not later + than the opt-out is dropped. +- `ps.iv` and `ps.ov` hold unkeyed hashes, and they still match after an + `ec.passphrase` rotation, as does the consent fingerprint. - `ts-eids-s`: - - sealing and opening with the EC ID as associated data; a cookie sealed - for another EC, or tampered with, fails to open and is ignored without - logging its value - - rejection of a duplicate `__Host-ts-eids-s` - - a `cf` that does not match the identity config discards the cookie's - positive entries and keeps its removals - - a stale `kc` forcing a read - - the size cap's drop order, stubs forcing a read, and removals and - markers never dropped - - a staged refresh that `OwnerWrite` would apply survives the merge, and - one it would not is dropped - - the flush checks (freshness, writer still enabled and owning, consent - re-gate, opt-out, revision) - - a replayed old cookie cannot re-insert an opted-out ID, and is overridden - at its next revalidation -- `ts-eids-r`: - - an entry written by a removal-producing response survives a concurrent - `/auction` that rewrites `ts-eids-s` from an older copy - - two removal-producing responses in flight together both keep their - batches, each in its own cookie - - a batch that does not fit one cookie, or meets four openable cookies, - is written to KV before send; when that write fails it is set in new - cookies of its own, and no other cookie is rewritten - - concurrent responses that each create a first cookie, or each - overflow, from the same inbound snapshot all keep their batches - - a 253-byte source domain and a long provider ID fit the per-cookie - budget, and no batch is widened, stripped of `iv` or turned into an - opt-out to fit - - a partly applied cookie is kept whole - - a Lockr revoke followed by a re-consent capture, before and across - navigations, ends with Lockr's new IDs in KV and the old opt-out never - re-applied (`ps.oc` in KV), including after a failed flush - - a removal names its source by domain, so a config change that reorders - sources removes the right one - - a cookie that fails to open is deleted by the writer and never counted - - a removal without `pv` skips a record its writer re-acquired after `sa`, - and removes one acquired in the same second as `sa` - - it is honoured on the unusable-consent path, where `ts-eids-s` is - expired, until the next writer applies it - - the 512-byte sealed cap per cookie with an 8-character key ID, the - one-year `Max-Age`, and that stagers never delete one -- Cookie keys: - - a cookie sealed under a previous key opens; one whose key ID is not in - the keyring fails to open - - a writer re-seals an unapplied `ts-eids-r` cookie opened under a - previous key, in place, with the same batch, and deletes an applied one; - a stager never re-seals - - a `ts-eids-s` opened under a previous key is re-emitted under the - current key by the next request that reads KV - - across a rotation, with the new key added before it becomes current, - no unapplied removal batch is lost - - `ps.iv`, the consent fingerprint and a legacy `pv` hash still match - after a cookie key rotation + - the mirror payload holds only what bidders would receive from the + server set: no `ps`, `ps.st`, refresh token, revision, writer or timing + field, and nothing a consent gate or `bidstream_enabled = false` + excludes + - `value` is base64url JSON; `max_age` is at most 86,400 and never past + the earliest `exp`; an empty set returns `clear`; a request with no KV + read returns no payload + - the 2,048-byte cap leaves out whole sources in ascending + `cookie_priority`, never trims a set, and a left-out source still + reaches bidders + - the server never opens it: a forged, stale or malformed `ts-eids-s` + changes no eligibility, merge, `/identify` answer or KV write + - no response sets it with `Set-Cookie` + - the shim writes it as `__Host-ts-eids-s` with `Path=/`, `Secure` and + `SameSite=Lax`, without `HttpOnly`, skips an unchanged value, expires + it on `clear`, and never copies it into `ts-eids` or the `/auction` + body + - a mirror ID a page script feeds back through Prebid is ingested only as + a one-UID `FillMissing`, and the `ps.iv`, `ps.ov` and `ps.o` denials of + section + 7.1 filter it for owned and unowned sources - The admin lookup redacts `ps.st`. **Outbound requests.** Requests to Prebid Server, in every consent-forwarding mode, carry none of the `TS_IDENTITY_COOKIE_NAMES` cookies, keep every other cookie, and drop the header when nothing is left. -Requests to the publisher origin carry none of `ts-eids`, -`__Host-ts-eids-s` or any `__Host-ts-eids-r-` cookie, keep `ts-ec`, -`ts-ec-pull-complete` and every other cookie, and drop the header when -nothing is left. +Requests to the publisher origin carry neither `ts-eids` nor +`__Host-ts-eids-s`, keep `ts-ec` and every other cookie, and drop the +header when nothing is left. **Merge.** @@ -4051,9 +4073,9 @@ nothing is left. - A browser-seeded record never beats the browser's current value. - Expired IDs are never sent. - `x-ts-eids` matches what was sent. -- After the owner invalidates UID A, or reports an opt-out, a stale A in the - `/auction` body or in `ts-eids` is not sent, before and after the removal - reaches KV. +- After a provider invalidates UID A, or reports an opt-out that removes + it, a stale A in the `/auction` body or in `ts-eids` is not sent, for an + owned or an unowned source. - An unowned source a strict provider wrote stops reaching bidders and `/identify` once that provider's vendor consent is denied or a targeted-advertising opt-out applies. A record whose writer is disabled @@ -4066,484 +4088,252 @@ nothing is left. **Batch sync.** An update to an owned source applies only when absent. +**Complete provider EIDs (G13).** + +- A result with several entries for a source, several UIDs per entry and + every UID-level and EID-level field round-trips through the mutation, KV + and the merge, and reaches `user.ext.eids`, `x-ts-eids` and the + `ts-eids-s` mirror in the order the provider returned it. +- Each limit of section 4.5 drops exactly what it names: a blank `inserter`, + an `ext` over 2,048 bytes, a ninth entry, a 33rd UID, a duplicate entry. + A set over 4,096 bytes, measured by the shared `MAX_ENTRY_SET_BYTES` + function, is discarded whole and leaves the stored set as it is; for a + stateful provider the positive part of the result is rejected with its + removals kept; a stateless result all of whose sets are discarded is + handled as `Malformed`. +- For an owned source, a result that names it replaces its whole set; a + later result without one entry removes it. A source the result does not + name is left as it is. An unowned provider's set only fills an absent + record. +- The set's `exp`, `ra` and `ru` are the earliest its kept entries set, and + an expired set is never served. Expired means `now >= exp`. +- Descriptor provenance applies only to an entry with no EID-level field; + an entry with any keeps exactly what the provider set. +- `build()` rejects a blank provenance `inserter` or `matcher` and an `mm` + below 0. +- An owner's valid set replaces every browser entry for the source, + including repeated browser entries; an unowned set goes out only when the + browser did not send the source. +- An invalidation of a 32-UID set appends 32 hashes to `ps.iv` from the + live record it removes, and the filter then holds through `ps.iv`. Two + such invalidations keep all 64 hashes. +- A full set too large for the `ts-eids-s` mirror is left out of it whole, + never trimmed, and still reaches bidders from KV. +- `KvEntry::validate` rejects, on write, a record with an empty `eids`. On + read, a record that breaks a limit is dropped from the row with its + source logged, and the rest of the row reads normally. +- A stored entry with a field this build does not know reads normally. +- `/identify` returns the same `uid` and `eid` shape for a full set as for + a one-UID record: the first UID, with the partner's `openrtb_atype`. +- A mutation on a build that no longer configures a provider keeps that + provider's `ps.o`, `ps.oc` and `ps.ov` and drops its other `ps` fields. + +**Behaviour that predates this spec (G13).** Golden tests pin each path that +G13 must not change: + +- browser ingest writes a one-UID record holding the UID + `collect_eid_updates` picks today (for a repeated source, the first valid + UID of its last entry; the sharedId cookie last), and nothing for an + unregistered source, whatever EID-level fields or extra UIDs the browser + sent. The expected values are captured from `main`, not written from + this sentence +- the `x-ts-eids` header's per-entry truncation +- the `ts-eids` cookie's content and EID-level `ext` removal, and its trim + order apart from the registered-first rule G10 adds +- `/auction` body forwarding and the merge of browser entries +- a row read from `main`'s schema version 1 goes out to bidders exactly as + it does on `main`, and the golden requests of the EID fidelity spec + (§4.7) do not change + **Migration parity.** Today's `ec/pull_sync.rs` tests are ported against `GenericPullProvider`. Each behaviour change in section 10.4 gets its own test. **Config.** Each validation error in section 10.2, `build_descriptor` -running in `ts config validate` without secrets, acceptance of a config -with no `[ec.identity]` at all when `ec.ec_store` is unset, and acceptance -of one with only `[ec.identity.cookie_keys]` when it is set. The -strips-only release accepts `[ec.identity.cookie_keys]` (section 14). - -**First-auction metric.** The shim reports the race flag of section 14 on -the first `/auction` of a page load only, and only where the Lockr -integration is active. The log line carries the flag and no identifier -value. +running in `ts config validate` without secrets, and acceptance of a +config with no `[ec.identity]` at all. **Adapters.** Phase 4 runs after send on Fastly. Axum, Cloudflare and Spin -behave as before apart from the Prebid Server and origin cookie strips. The parity test crate -passes -(`cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`). +behave as before apart from the Prebid Server and origin cookie strips. The +parity test crate passes +(`cargo test --manifest-path +crates/trusted-server-integration-tests/Cargo.toml --test parity`). ## 14. Rollout -Three steps, each shippable on its own: - -0. **Strips first.** One release before any new cookie is set, ship only - the cookie strips (origin and Prebid Server, by name and by the - `__Host-ts-eids-r-` prefix) and the template cache's identity cookie - list. Stripping `ts-eids` from origin requests is a behaviour change for - any origin that read it. A rollback after step 1 then lands on code that - still keeps the new one-year cookies away from the origin and its header - limit. This release is the rollback floor. It also parses and validates - `[ec.identity.cookie_keys]` and lists its secrets in `secret_fields()`, - without using them. `Ec` denies unknown fields (`settings.rs:541-542`), - so without this a config holding the keys would fail to load after a - rollback to it. Operators provision the keys in the secret store and add - the table during this release, so step 1 finds them. +Two steps, each shippable on its own: + 1. **One write, no new vendors.** This step includes: - - the writer-request rule and the `IdentityMutation` single flush, with - no retry; the new visitor's create stays before routing, as today - - `/identify` and `/_ts/page-bids` stopping their writes - - `ts-eids-s`: the host-only cookie sealed with AES-256-GCM (a new - `aes-gcm` dependency, which the UID2 provider later reuses) under the - rotating cookie keys, `kc` revalidation, KV-gap staging on `/auction`, - pending and `m` markers, the size cap - - the `ts-eids-r` removal cookies, and their re-seal after a key - rotation (section 6.2) - - a metric for the inbound `Cookie` header size, and for the number and - total size of `ts-eids-r` cookies (section 6.2) - - a first-auction race metric for the open question in section 15. The - shim adds one boolean to the first `/auction` body of each page load, - only where the Lockr integration is active: whether the browser's - resource timing already shows a completed response from the Lockr - proxy's `page-view` path. TS counts first auctions with and without it. - It is counted once per page load and logs no identifier value. Capture - comes in step 2, but the race does not depend on it, so step 1 measures - it before step 2 relies on the cookie - - metadata-preserving `KvPartnerId` and `ps`, with `Default` and - constructors + - the outbound cookie strips (origin and Prebid Server) and the + template cache's identity cookie list. Stripping `ts-eids` from origin + requests is a behaviour change for any origin that read it + - the writer-request rule and the `IdentityMutation` single flush, with no retry except the removal-only one of section 5.2, phase 4; the new visitor's create stays before routing, as today + - `/auction`, `/_ts/page-bids` and `/identify` stopping their writes + - one after-send attempt for a withdrawal tombstone that failed before + send (section 5.1) + - the `ts-eids-s` mirror payload in `/auction` and `/_ts/page-bids` + responses, and the shim code that writes and clears the cookie, once + section 15 names a consumer; without one, this step ships without the + mirror + - the shim's registered-first `ts-eids` trim, with the registered source + list in the injected configuration (section 5.3) + - a metric for the inbound `Cookie` header size + - KV schema version 2 (section 6.1): the one record shape, `ps`, with + `Default` and constructors, and the read of version 1 rows. Full entry + sets are read, served and validated, though nothing in this step + writes one, so a rollback from step 2 to step 1 keeps every set a + provider wrote readable, and keeps each vendor's opt-out (below) - `GenericPullProvider` replacing pull sync, which brings the trait, the - descriptor, the harness and the write-counting test store + descriptor, the harness and the write-counting test store, and the + removal of the pull-complete marker and of the legacy pull sync keys + (section 10.4) - the shared servability check for `/identify` and the bidstream It changes behaviour on its own: fewer KV writes, `/auction`, - `/_ts/page-bids` and `/identify` stop writing, and new visitors reach - pull partners from their second page load. Write and read counters per - page load show the effect. + `/_ts/page-bids` and `/identify` stop writing, every returning writer + navigation reads KV once, and new visitors reach pull partners from + their second page load. Write and read counters per page load show the + effect. Operators move `pull_sync_ttl_sec`, `pull_sync_rate_limit` and + `ec.pull_sync_concurrency` to their replacements before deploying it, + because validation rejects them (section 10.2). 2. **Providers.** `.with_identity_provider` and `.try_with_identity_provider` for vendors, ownership and its - validation, hold mode, passive capture with `capture_enabled` off by - default and `revoke_scope` defaulting to `"provider"`, the real-browser - gate on capture effects, provider state, the rate-limiter `record` and - `count` methods and the circuit breaker, and the consent decoder - extensions of section 9.1. Vendor modules follow in their own specs. - Capture stays off for a provider until the checklist below has passed - once on a live service. - -**Capture provider checklist.** Before setting `capture_enabled = true`: - -- Each captured source has an `[[ec.partners]]` entry with + validation, hold mode, revoke-only passive capture with `revoke_scope` + defaulting to `"provider"`, the capture removal write with its + `capture_unpersisted` and `removal_lost` counters, the + real-browser gate on capture effects, provider state, the rate-limiter + `record` and `count` methods and the circuit breaker, the consent + decoder extensions of section 9.1, and the first full-set writes, from + provider results with more than one entry or UID or with EID-level + fields (G13). Vendor modules follow in their own specs. + +**Capture provider checklist.** Before enabling a capture provider on a +live service: + +- Each source it declares has an `[[ec.partners]]` entry with `bidstream_enabled = true` and the right `openrtb_atype`. Lockr's ID5 uses `1` (#1246). +- Its required `sources` list holds only the sources it is the delivery path + for, because its opt-out removes browser-ingested records of every source + it keeps, whoever produced them (sections 5.4, 10.2). - A source moving to an owner is moved off pull sync first. A `pull_sync_enabled` entry registers a generic pull provider that claims the source, and two providers cannot claim an owned source (section 10.2). -- Live acceptance: after an eligible request, `/_ts/admin/eids` lists the - source as matched (`ingest.matched`, `ec/admin.rs:532-536`), and then - `/_ts/admin/ec` shows the stored UID (`ec/admin.rs:276`). - -**Rotating the cookie keys.** Add the new key to `keys` and deploy. Once -every POP serves that config, make it `current` and deploy again. Keep the -previous key for the retention section 6.2 gives: `staged_max_age_sec` for -`ts-eids-s`, one year for `ts-eids-r`. Removing it sooner can drop an -unapplied removal batch of a client that has not navigated since. - -After providers write metadata, do not roll back past step 1. Step 1's reader -preserves the new fields. A rollback past it strips them: - -- `w`, `exp`, `ra` and `at`. The rollback rule of section 6.1 turns these into - re-fetches, not stale IDs, once rolled forward again. -- `ps.o`. Opted-out providers would be called again after rolling forward, - except where the cookie still holds the sticky `o`. -- `ps.st`. Providers with state start over. -- `ps.iv`. Rejected values could be re-seeded once. - -While rolled back, `exp` is not enforced at all. Rolling back past step 1 -also restores today's `/auction` ingest. Step 0's code ignores `ts-eids-s` -and the `ts-eids-r` cookies but still strips them, and the browser keeps -sending what it staged. +- Live acceptance: after a revocation captured on a live page, + `/_ts/admin/ec` (`ec/admin.rs:276`) shows the provider's `ps.o` and no + record for its sources. After the vendor issues again, it shows `ps.oc`, + and the next writer navigation ingests the source again. + +**Rolling back from step 2 to step 1.** Step 1 configures no vendor +providers, so: + +- Remove `owner` and the vendor `[ec.identity.providers.*]` tables from the + config first. Step 1's validation rejects an owner or a provider table it + does not know (section 10.2). +- Full sets stay readable but are not served while their writer is not + configured, and browser ingest may replace them (section 7.1). +- A mutation drops each vendor's `ps` fields except `o`, `oc` and `ov` + (section 5.4), so opt-outs and the IDs they removed survive, and `ps.iv` + and `ps.st` do not: invalidated values can be re-seeded once, and + stateful providers start over when rolled forward. + +**Rolling back past step 1 is unsupported.** Step 1 writes schema version 2, +and today's reader rejects any version but 1 (`ec/kv_types.rs:376-381`), so +an older build fails to read every row step 1 has written. Going back needs +a KV migration to version 1, which this spec does not provide. ## 15. Open questions 1. **How long single-page apps wait.** Some sites are single-page apps: after the first page loads, they change pages with JavaScript instead of loading a new document from TS. TS hears about those page changes only - through `/_ts/page-bids`, which never writes KV (section 5.1). So on those - sites, entries saved in `ts-eids-s` wait until the visitor's next full - page load. After launch, measure how long saved entries wait before they - are written, and revisit if the wait is long. -2. **Tombstone failures.** Today's withdrawal tombstone is a CAS write with - up to five attempts (`ec/kv.rs:1349-1430`). It can fail on CAS exhaustion, - on Fastly's per-key limit when another write to the same key landed - within about a second, on a read failure, or on a KV error. Should a - failed tombstone be retried after send, with a delay? -3. **How often the first auction misses Lockr's ID.** On the affected route, - Lockr's `page-view` call and the first `/auction` start close together. - How often does the first `/auction` leave before the proxied Lockr - response sets `ts-eids-s`? Then that auction lacks the captured ID, and a - first visit's first auction gets it only from the browser path. Step 1's - race metric (section 14) measures it before capture is turned on (#1260 - review). -4. **Global revoke and a granting CMP.** A global revoke clears the EC + through `/_ts/page-bids`, which never writes KV and calls no provider + (section 5.1). So on those sites, browser ingest and provider refreshes + wait until the visitor's next full page load. After launch, measure how + long they wait, and revisit if the wait is long. +2. **Lost removals.** A removal write that fails all its attempts is lost + (sections 5.2, 5.3). After launch, watch + `removal_lost`. If it is not close to zero, should a durable + fallback come back, and in what form? +3. **Global revoke and a granting CMP.** A global revoke clears the EC cookie, as a withdrawal does, but the CMP's consent may still grant. The next navigation then mints a new EC. Should a global revoke also stop minting until consent changes? +4. **Durable stager results.** Stagers persist no positive ID (section + 5.3), so a positive Lockr capture, with its full entry set, reaches KV + only as the one UID browser ingest picks. Should a durable path + come back for it, for example a capture-driven write within G4, if + `capture_unpersisted` shows that captures carry IDs or fields the + browser path does not? +5. **Who reads `ts-eids-s`, and does it meet the product rule?** The + product rule is that provider IDs go to a first-party cookie and to KV, + with the cookie read first. Since revision 14 the server never reads the + cookie, and no page script is named as its reader (section 6.2). Before + rollout step 1 ships the mirror, name its consumer and confirm with the + product lead that a client-only mirror meets the rule. Without a + consumer, the mirror costs about 2 KB on every request to the host and + exposes server-only IDs to page scripts for nothing. +6. **Providers on a new visitor's first navigation.** No provider runs on + a first navigation (section 4.7, step 7), so a visitor who leaves after + one page never gets a provider ID. One option keeps one write and never + uses an unconfirmed ID: mint the EC, run hold-mode providers (which need + no EC ID; only generic pull does), create the row with their answers, + then run the auction. It moves the create from before routing into + routing, and adds the hold budget to every first navigation that runs + an auction. Is that worth it? ## 16. Decisions log Decisions made while writing this spec, with the user: -| Decision | Choice | -| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Scope | Framework and trait. Vendors get their own specs. | -| HEM input | Typed slot, always empty in v1. Intake is a later spec. | -| Browser vs server precedence | Configured per source through `owner`. The owner's valid value wins. | -| Cookie for server IDs | Separate `ts-eids-s`, server-owned and HttpOnly, sealed with AES-256-GCM so the origin and its logs never see server-only IDs. | -| Structure | Trait plus coordinator, with an optional passive-capture capability. | -| Trait shape | One required method, `descriptor`. Capabilities are default methods. Synchronous. Additive by construction. | -| Inputs | Declared in the descriptor and enforced: a provider can read only what it declared. | -| Page load for the one-write rule | A navigation plus every request it causes. Only the navigation writes. Strict. | -| New-visitor write | One create before routing, as today, carrying browser EIDs. No provider runs on the first navigation. Changed in rounds 3 and 4. | -| New-visitor recovery | Today's rotation. The cookie-proof recreation of revision 3 is removed with the deferred create. | -| Failed write | Not retried. Changed from one merge-retry once Fastly's per-key limit was confirmed. | -| Browser EIDs beyond the cookie | `/auction` saves the KV gap in `ts-eids-s`. The next navigation writes it. | -| Delay on body-less routes | Accepted: a browser ID first seen in an `/auction` body reaches KV one page view later. | -| SPA views (`/_ts/page-bids`) | Stagers. They never write KV. | -| Hold budget | `hold_budget_ms` defaults to 100 ms, reduced from 300 ms (section 4.7 gives the reasons). | -| Outbound cookies | TS's identity cookies are stripped from Prebid Server requests. The origin keeps `ts-ec` but loses `ts-eids`, `ts-eids-s` and `ts-eids-r`, to stay under header limits. | -| Deletion | A tombstone with the 24-hour `TOMBSTONE_TTL` replaces the hard delete. No extra writes. | -| GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | -| `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | -| Consent withdrawal | Synchronous tombstone on any request, one of two G4 exceptions with the removal overflow write. Proposed in review; not yet confirmed. | -| Removal overflow | A synchronous, removal-only KV write before send, the second G4 exception. It retries only a CAS conflict and falls back to new cookies. Confirmed after round 10. | -| Equal-second ties | A removal or opt-out wins a tie with an acquisition or clear. An ID re-acquired in that second is resolved again later. Confirmed after round 10. | -| Server cookie lifetime | #1260's handoff cookie lived five minutes, only filled absent sources and was never authoritative. `ts-eids-s` lives a year, revalidates against KV every `kv_revalidate_sec` and can carry owner writes, so staged entries survive failed flushes and long SPA sessions. | - -Round 7 proposed these, and they are applied in revision 8 pending the -user's review of the spec: - -| Decision | Choice | -| ----------------------- | ----------------------------------------------------------------------------------------------------------- | -| Removals and opt-outs | One `__Host-ts-eids-r-` cookie per removal batch, up to four, so no concurrent response can lose one | -| Provider opt-out | Removes every record the provider wrote, with no revision check. Opted-out writers are never served | -| Expired records | `Refresh` covers a stored record, valid or expired. An expired record counts as absent for `FillMissing` | -| Stateful hold providers | Run only on the writer: before the auction when it runs one, after send when it does not | -| `capture` result | Returns `Resolution`; `no_match` means nothing was captured | -| Registration | `try_with_identity_provider` converts a constructor's error once; `build()` stays infallible | -| KV read cost | An auction navigation reads once, as today. Phase 4 reuses that snapshot unless an after-send provider runs | -| Provider count | At most 32 enabled providers, so a row's `ps` never fills up | - -Round 10 proposed these, and they are applied in revision 11 pending the -user's review of the spec: - -| Decision | Choice | -| ------------------- | ---------------------------------------------------------------------------------------------------- | -| Record revision | A random `rv` per record write | -| Stateful results | IDs, state and scheduling apply atomically. An invalid replacement state rejects the positive result | -| Refresh anchors | An expired owned record is kept, unserved, while its owner holds state, until its `ru` | -| Disclosure policy | The writer's policy always, plus the owner's. Records of disabled writers are not served | -| Browser candidates | The owner's `ps.iv` and `ps.o` filter the browser's value for owned sources | -| `/identify` consent | The request's visitor consent. No persisted visitor authorization | - -Round 11 proposed these, and they are applied in revision 12 pending the -user's review of the spec: - -| Decision | Choice | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| Real-browser gate | A non-browser capture stores no positive ID, stages removals only in its own `ts-eids-r`, and never makes the overflow write | -| Global revoke | `revoke_scope`, default `"provider"`. `"global"` runs the withdrawal tombstone, from real browsers only, under the existing exception | -| Cookie keys | Dedicated rotating keys from the secret store, `v1...`. Writers re-seal unapplied removal batches | - -## 17. Review log - -**Round 1** (2026-10-21) found 5 blocking issues, 16 significant gaps and 17 -accuracy problems. Revision 2 resolved them. Its findings covered the -deferred create's rotation risk, staged refreshes, pending-marker liveness, -provider state, the writer definition, hold-mode latency, concurrent cookie -writes, HTML cacheability, multi-source providers, inputs, `ext` level, -owner takeover, `/identify` and batch sync, the size cap, flush checks, rate -limits, the breaker, tombstones, `/auction` ingest and deploy-time -validation. - -**Round 2** (2026-10-21) answered the open questions and reworked the trait -for ergonomics and additivity (revision 3). - -**Round 3** (2026-10-21) ran four independent reviews: Rust API, write -protocol, code references, and privacy and consent. They found 7 blocking -issues and about 30 significant ones. Revision 4 resolves them as follows: - -| Finding | Resolution | -| ------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------ | -| Deferred create loses withdrawals, collision checks and orphan detection (B1, B2, B3, C1, D1) | Create before send, sections 5.2, 5.5 | -| Cookie IDs outlive KV removals (D2, B5) | `kc` revalidation, merge rules, G12, 6.2, 6.3 | -| `ts-eids-s` reaches Prebid Server (D3, C11) | Strip from Prebid Server; origin unchanged (revision 5), 6.2 | -| Stagers cannot save removals, state or `ps` (B4) | Removal entries, staged `ps`, state writer-only, 5.3 | -| Eligibility on stagers ignored the cookie (B6) | Merged eligibility view, 4.7 | -| Invalidate and re-seed loop (B7) | `ps.iv`, 7.2 | -| Writer could not clear markers or flip `k` (B8, C16) | Phase 3 cookie rule, 5.2 | -| Late hold results impossible; overruns tripped the breaker (B9, C2) | Cut-off calls, `hold_retry_sec`, 4.7, 8 | -| `request` returning `None` left providers due forever (A1) | `None` waits `refresh_after_sec`, dry run, 4.7, 8 | -| Capture backoff on unrecognised bodies (A2) | Capture never backs off, 4.7, 8 | -| `MissingInput` conflated a bug and a skip; state could drop IDs (A3) | `UndeclaredInput`, `InputUnavailable`, `with_state(&T)`, 4.4, 4.5 | -| EC ID and capture prefix unenforced (A4) | Crate-private EC ID, prefix check, 4.3 | -| Multi-source invalidation (A5) | `also_invalidate`, 4.5 | -| Registration snippet and config trait (A6) | `change_context`, `IdentityProviderConfig: IntegrationConfig`, 4.6, 10.2 | -| `#[non_exhaustive]` ineffective in-crate (A7) | Additivity by construction, opt-in kinds, 4.2 | -| Getters, setters and naming (A8, A13) | `with_*` setters, getters, renames, 4.3 to 4.5 | -| Second `ProviderId` (A9) | `IdentityProviderId`, 4.3 | -| Error type and header validation (A10, A11) | 4.5 | -| Lint and doc conventions; silent mis-wiring (A12, A14) | Conventions note, conformance checks, 4.2, 13 | -| Phase 4 lacked UA, page URL, DNT and a usable IP (C3) | Captured in phase 1, 4.4 | -| `/identify` bypassed the expiry filter (C4, D6) | Shared servability check, 7.1 | -| Missing writes; withdrawal path misdescribed (C5, B10, D13) | P3 table, 5.1, 13, 15 | -| Writer decision placement and prefetch visibility (C6) | 5.1 | -| Rate-limiter API and breaker (C7, B11, D12) | `record`/`count`, cap last, transport-only breaker, 4.7 | -| Capture buffering, encoding and cookie site (C8, C18) | 4.7, 5.3 | -| Unowned completeness signal (C9) | Pull-complete marker kept, 6.3, 10.4 | -| Domain-wide cookie and MAC binding (C10, D5) | `__Host-`, delimited MAC, `cf`, 6.2 | -| Clock failure, schema literals, write-counting tests (C12, C13, C14) | 4.4, 6.1, 13 | -| Policies gated only dispatch (D4) | Re-applied at flush, cookie, merge, `/identify`, 9.1 | -| Opt-out fingerprint lapsed (D8) | Sticky `o`, 8 | -| Strict policy failed open (D9) | Fail closed, more decoding, 9.1 | -| Minimization leaks: geo, jurisdiction, page URL, `x-ts-eids` (D10) | `Region`, `ProviderConsent`, sanitized URL, documented header, 4.4, 7.1 | -| Logging holes (D11) | 9.3 | -| `Sec-Fetch-Dest` fallback, undefined transitions, rollback losses (B12, B13, B14, B15, C15, C17) | 3.4, 5.1, 5.2, 6.2, 6.3, 14 | -| Reference corrections (C19) | Throughout | - -**Round 4** (2026-10-21) re-reviewed revision 4 with two independent -reviewers. They found 6 blocking issues, all with contained fixes, and -confirmed most round-3 fixes. Revision 5 resolves them: - -| Finding | Resolution | -| --------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| A minted ID was used in routing before its deferred create, on publisher and proxy routes | Create stays before routing; no provider on the first navigation, 4.7, 5.2 | -| A config change discarded staged removals and opt-outs | 5.6 | -| Flush checks could veto removals | Removals exempt, 5.6 | -| Orphan rotation re-bound positive IDs, undoing deletion | Missing row expires the cookie, 5.5, 5.6 | -| Markers could not be answered when phase 4 wrote nothing | Phase 4 always answers; unowned dry runs need KV, 5.6 | -| Markers set on non-2xx responses | 5.2 | -| Unowned hold answers became `OwnerWrite` | 5.3 | -| Writer never flipped its flushed entries to `k: 1` | Reconciling read, 5.6 | -| EC minting on non-writer navigations undefined | 5.6 | -| Withdrawal windows and `kc` stamping | 5.6 | -| G4 named withdrawal the only exception | G4 scoped to page loads | -| `cluster_size` had no trigger; size cap could fail | 5.6 | -| `pv` hash compared with a raw revision | `RevisionHash`, 5.4 | -| Capture plumbing, Prebid strip paths, default `capture`, `ProviderConsent`, `type Err`, `with_private_state()` naming | 4.2, 4.3, 5.6, 6.2 | - -**Round 5** (2026-10-21), a final review focused on trait ergonomics, found -3 blocking and 6 significant issues, all fixed in revision 6: - -- Phase 4 re-reads before dispatching whenever after-send providers are - marked, so a withdrawal after send is always seen (5.6). -- Conformance checks a provider against at least one working harness setup - and a fixture, so UID2- and LiveRamp-style providers pass (13). -- The capture-prefix check moves to `IntegrationRegistration::build()` (4.3). -- Undeclared input and state reads are recorded and caught even through - `.ok()` (4.4). -- Every capability default, including `request`, returns a - `not_implemented` marker (4.2, 4.5). -- State is enforced: `with_state` and `state()` need `with_private_state()` - (4.4, 4.5). -- Less boilerplate: `Resolves::with_endpoint`, `primary_endpoint`, - `ResolveInput::primary_source`; the example parses its endpoint once (4.6). -- Consistent names: `with_private_state`, `with_expiring_ids`, - `also_invalidate`; `found([])` is `no_match` (4.3, 4.5). -- Stale references, outbound-cookie wording, capture state, the additivity - claim for consent, mixed-kind inputs and the step 8 wording are corrected. - -**Round 6** (2026-10-21) was an example-driven review. LiveRamp ATS, Lockr -and UID2 were written as full providers against the API (section 12), and -everything they could not express was added. Section 12.5 lists the gaps: - -- a call context from `request` to `response` -- deterministic randomness -- a stated path for secrets -- text bodies -- a larger state cap -- millisecond times -- exact capture paths -- capture removals that bypass the consent gate -- consent from the request body -- EID-level provenance -- source matching -- keeping an ID whose `ext` is oversized -- region tests - -**Round 7** (2026-10-21) ran two final reviews in parallel. One covered the -API, the examples and feasibility against the code. The other covered the -protocol, privacy, consistency and efficiency. Together they found 2 -blocking, 13 significant and 11 minor issues, three of them found by both. -All are fixed in revision 8: - -- A provider opt-out removes every record that provider wrote, with no - revision check, and servability excludes records whose writer is opted - out. `OwnerRemove` is never skipped because of `ps.o` (5.4, 7.1). -- Capture removals and opt-outs go into a new append-only cookie, - `ts-eids-r`, which a concurrent `/auction` cannot overwrite. It is also - written on the unusable-consent path (5.3, 6.2). -- A due stateful provider, or one with an unowned or stubbed candidate - source, triggers the phase 1 KV read, so its dry run sees KV. A dry run - without loaded state never sets `ps.n` (5.2, 6.3). -- `Refresh` covers expired records, and an expired record counts as absent - for `FillMissing`. An expiring ID without an expiry keeps the stored `exp` - when its UID is unchanged, and is discarded otherwise (4.4, 4.5, 5.4). -- `ps` is pruned on every mutation, more than 32 providers is a config - error, and a mutation that fails validation falls back to its removals - (5.4, 6.1, 10.2). -- `ts-eids-s` is sealed with AES-256-GCM, compactly encoded and capped at - 2,048 bytes. Rollout adds a `Cookie` header size metric and an origin - header-limit checklist item (6.2, 14). -- A stateful hold-mode provider runs after send on writer navigations - without an auction (4.7, 10.1). -- Opt-out clearing: a mirrored `o` yields to KV, GPP clears by section - values, a capture-only provider clears on a later positive capture, and a - withdrawal resets every opt-out (8). -- Efficiency: the read-cost table is corrected, phase 4 reuses the phase 1 - snapshot, cluster evaluation runs only for `/identify` and after send, the - cookie is rewritten only when it changed, and `is_dry_run`, cached - `state()`, embedded-JSON `st` and one cookie decode per request remove - repeated work (4.2, 4.4, 5.2, 5.6, 6.1). -- Request-body consent is buffered in `handle_proxy` with - `collect_body_bounded` and merged by `consent::stricter_of` (4.7, 9.3). -- The capture-path check runs in `IntegrationRegistry::with_plan`, so - `build()` stays infallible (4.3, 10.3). -- API: `Resolves::with_endpoint(&str)`, an `Access` enum in - `UndeclaredInput`, a fallible `try_with_call_context` and `call_context`, - `expiry()` and `refresh()` getters, `StateChange`, `capture` returning - `Resolution`, `with_retry_in`, `replacing_state`, `CaptureInput` - accessors, and full listings for `Region`, `HashedEmail`, `UnixTime`, - `SourceDomain::matches` and `try_with_identity_provider` (4.2 to 4.6). -- Harness: `check_conformance(&ConformanceCases)` and - `check_descriptor_ignores_secrets` (13). -- Lockr parses into permissive local types and skips expired tokens (12.2). -- Minimization claims and stale wording are corrected (9.2, and - throughout). - -After round 7, at the user's request, `ts-eids`, `ts-eids-s` and `ts-eids-r` -are also stripped from publisher origin requests, which keep `ts-ec`. This -replaces the origin header-limit checklist item (6.2, 14). - -**Round 8** (2026-10-21) was a last overall review: two parallel reviews, -one on the protocol and design and one on the API, examples and -efficiency, plus an outside review. They found 1 blocking, 9 significant -and 22 minor issues, several found more than once. All are fixed in -revision 9: - -- Removal batches no longer share a cookie. Each removal-producing - response sets its own `__Host-ts-eids-r-` (up to four), stagers never - delete one, and only the writer deletes those its phase 1 read shows - applied. The cookies last a year, so a removal never expires before a - writer applies it (6.2, 6.3). -- Orphan rotation drops the rest of its mutation and skips phase 4, and - phase 4 applies the tombstone, missing and failed rules to a reused - snapshot too (5.2). -- A failed flush drops a stateful provider's staged IDs with its state, - and the loss of an after-send removal is bounded by the marker deadline - (5.2). -- Pull sync no longer reads KV on every navigation: the pull-complete - marker and a cookie-only `ps.n` stop the trigger (6.3). -- `cluster_size` is no longer driven by a `/identify` cookie flag. A writer - that is already writing adds it after send (5.6). -- A missing row has one outcome, a minimal `m: 1` cookie, and saves no KV - gap. The KV gap also fills expired sources (5.3, 5.6, 6.3). -- Lockr's opt-out can be cleared: a positive capture stages `ps.oc`, which - the writer applies before the IDs (5.4, 8). -- Expiry is declared per source (`with_expiring_source`, `is_expiring`). - Lockr marks none, because its expiry is optional (4.3, 12.2). -- `state()` returns `Result>` with a new `InvalidState` error, and - `capture` never changes state (4.4, 4.5, 4.7). -- The harness has a full listing, including `HarnessSetup`, `TestConsent`, - `CaptureFixture`, `ConformanceCases` and test-only constructors (13). -- Stateful providers skip the dry run, saving a phase 1 read per refresh. - Cookies open lazily, only on identity routes. Browser KV-gap entries are - dropped last at the size cap (5.2, 6.2). -- Smaller fixes: provenance getters, descriptor derives, the LiveRamp - refresh pacing, Lockr narrowing, the G10 and G12 bounds, and stale - wording. - -**Round 9** (2026-10-21) ran two parallel reviews, protocol and design, and -API, examples and harness, plus an outside review of developer experience. -They found 2 blocking, 11 significant and 16 minor issues. All are fixed in -revision 10: - -- A re-consent clear is stored in KV as `ps.oc`, so an old opt-out batch is - never re-applied, and "applied" is defined for every removal kind (5.4, - 6.1, 6.2). -- `ts-eids-r` names sources by domain, deletes cookies that fail to open, - and has a bounded fallback when four cookies are full (6.2). -- A stateful hold provider's answer stays out of the cookie, so its ID - never flushes without its state (5.2). -- The cookie-only `ps.n` applies to unowned sources and yields to a changed - KV `n` (6.3). Undispatched markers back off by at least 60 seconds (5.6). -- `cluster_size` is evaluated once per EC from any writer read, as today - (5.6). -- A strips-only release ships first and is the rollback floor (14). -- UID2's dead dry-run branch is gone; `InvalidState` always clears state and - has a section 8 row (4.4, 8, 12.3). -- The harness gains a raw resolve case, `HarnessSetup` methods, more - `TestConsent` setters, `state_of`, and stated kind and eligibility rules. - It is gated by `cfg(any(test, feature = "test-support"))`, so provider - crates can use it (13). -- `ProviderRequest::new` and `with_body` cover any method and body; both - header helpers follow one `Cookie` rule; `found` takes at most one ID per - source (4.5). -- Derives are corrected (`When` has no `Display`; `Hash` and `Ord` added), - and stale wording is fixed. - -**Round 10** (2026-10-21), an outside review of revision 10, found 6 blocking -issues and 1 question. Small executable models confirmed the ordering and -concurrency counterexamples. Revision 11 resolves them: - -| Finding | Resolution | -| -------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| The browser fallback could resend an ID the owner invalidated or opted out | Owner denials filter browser candidates for owned sources, 7.1 | -| Unowned provider-written IDs skipped the writer's consent policy | Writer's policy always, owner's too; disabled writers unserved, 4.7, 7.1, 9.1 | -| A re-read could pair one call's ID with another's state | `ProviderResult` applies atomically; invalid state rejects it, 4.5, 5.4, 8 | -| Pruning an expired ID stranded valid refresh state | Refresh anchors bounded by `ru`, `with_refresh_until`, 4.5, 4.7, 5.4, 6.1, 12.3 | -| Removal-cookie overflow could lose a batch under concurrency | Removal overflow write before send; measured byte budget; no coarsening, 6.2 | -| `hash(uid, at)` and equal-second ties could dismiss a revocation | Random `rv`; ties go to the removal or opt-out, 5.4, 6.1, 6.2 | -| The consent `/identify` checks against was undefined | The request's visitor consent; the stored snapshot authorizes nothing, 7.1 | - -The review's non-blocking notes are not applied here: the contract covers -the declared capabilities rather than every provider shape, HEM intake and -multi-step exchanges stay out of v1 (section 2.2), and the vendor examples -are illustrations, to be checked against protocol fixtures in their own -specs. - -**Round 11** (2026-10-23) folded in findings from -[#1246](https://github.com/IABTechLab/trusted-server/issues/1246) (Lockr IDs to -KV) and the review of -[#1260](https://github.com/IABTechLab/trusted-server/pull/1260) (server-side -identity foundation and Lockr capture). The Lockr ID5 EID lives in Prebid's -`ortb2` config, and capture sees it only at first issue. The first auction -can race the Lockr response. An empty GPP string must not block. Capture -should stay off until live acceptance. A replayed `ts-ec` could change a -row through the proxy, a revoke's scope needs an operator choice, and the -cookie keys should rotate apart from `ec.passphrase`. Revision 12 resolves -them: - -| Finding | Source | Resolution | -| --------------------------------------------------------------------------- | ------------ | -------------------------------------------------------------------------------------- | -| Lockr's ID5 EID is only in `ortb2`, which `getUserIdsAsEids()` misses | #1246 | Dependency on the EID fidelity collector stated; capture-once note, 12.2; G10 test, 13 | -| The first `/auction` can leave before the Lockr response sets `ts-eids-s` | #1260 review | Open question 3, 15; race metric in rollout step 1, 14 | -| A header-only GPP string (`DBAA`) blocked capture | #1260 review | Stated in 9.1; consent test, 13 | -| Capture defaulted to on | #1260 review | `capture_enabled`, default `false`, 4.7, 8, 10.1, 10.2, 14 | -| Owner, partner entry and pull sync prerequisites were implicit | #1260 review | Capture provider checklist, 14 | -| The handoff cookie's lifetime differed from `ts-eids-s` | #1260 | Decisions log row, 16 | -| A replayed `ts-ec` could enrich, strip or tombstone a row through the proxy | #1260 review | Real-browser gate on capture effects, G12, 4.7, 5.6, 6.2, 8, 13 | -| Global revoke had no operator gate | #1260 | `revoke_scope`, default `"provider"`, 5.1, 8, 10.1, 10.2, 12.2, 13 | -| Cookie keys came from `ec.passphrase` and could not rotate | #1260 | Dedicated rotating keys, re-seal, retention, 6.2, 9.3, 10.1, 10.2, 13, 14 | +| Decision | Choice | +| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Scope | Framework and trait. Vendors get their own specs. | +| HEM input | Typed slot, always empty in v1. Intake is a later spec. | +| Browser vs server precedence | Configured per source through `owner`. The owner's valid value wins. | +| Cookie for server IDs | Separate `ts-eids-s`, a client-only mirror the shim writes and the server never reads. Revision 14 replaced the server-owned, sealed, HttpOnly cookie of revisions 1 to 13. Revision 15 removed the `ts-eids-r` removal cookies as well. | +| Structure | Trait plus coordinator, with an optional passive-capture capability. | +| Trait shape | One required method, `descriptor`. Capabilities are default methods. Synchronous. Additive by construction. | +| Inputs | Declared in the descriptor and enforced: a provider can read only what it declared. | +| Page load for the one-write rule | A navigation plus every request it causes. Only the navigation writes and only the navigation calls providers. Strict. | +| New-visitor write | One create before routing, as today, carrying browser EIDs. No provider runs on the first navigation. Changed in rounds 3 and 4; open question 6 asks whether to revisit. | +| New-visitor recovery | Today's rotation. The cookie-proof recreation of revision 3 is removed with the deferred create. | +| Failed write | Not retried. Changed from one merge-retry once Fastly's per-key limit was confirmed. | +| Browser EIDs beyond the cookie | The shim's `ts-eids` trim keeps registered sources first, and the next navigation ingests them. Revision 14 removed the KV gap that `/auction` saved in `ts-eids-s`. | +| Delay on body-less routes | Accepted: a browser ID first seen in an `/auction` body reaches KV one page view later. | +| SPA views (`/_ts/page-bids`) | Stagers. They never write KV and call no provider. | +| Hold budget | `hold_budget_ms` defaults to 100 ms, reduced from 300 ms (section 4.7 gives the reasons). | +| Outbound cookies | TS's identity cookies are stripped from Prebid Server requests. The origin keeps `ts-ec` but loses `ts-eids` and `ts-eids-s`, to stay under header limits. | +| Deletion | A tombstone with the 24-hour `TOMBSTONE_TTL` replaces the hard delete. No extra writes. | +| GPP opt-outs | Sharing and targeted-advertising opt-outs are decoded, and the strict policy blocks them. | +| `x-ts-eids` | Kept as a full troubleshooting header. Its exposure of server-only IDs to page scripts is accepted. | +| Consent withdrawal | Synchronous tombstone on any request, a G4 exception with the two removal writes. Revision 15 adds one after-send attempt when it fails. Proposed in review; not yet confirmed. | +| Equal-second ties | A removal or opt-out wins a tie with an acquisition or clear. An ID re-acquired in that second is resolved again later. Confirmed after round 10. | +| Complete provider EIDs | A provider can return several EID entries per source, several UIDs each, with UID-level and EID-level fields. The set is stored and sent whole. Added in revision 13. | +| Entry-set writer | One writer per source. A result replaces the source's set whole, and one `rv` covers it, so ownership, revisions and removals stay per source. | +| Behaviour before this spec | G13 changes none of it. Browser ingest, `ts-eids`, batch sync, generic pull and `/identify` keep the behaviour the rest of this spec gives them and write one entry with one UID; only provider results write more. | +| Removal hashes | Derived when a removal applies, from the live record it removes. Unkeyed, ordered, at most 128 per list. Invalidations go to `ps.iv`, kept; opt-out removals go to `ps.ov`, emptied by the clear. | +| Provenance default | Applied only to an entry that sets no EID-level field, so vendor and config provenance never mix in one entry. Added in round 13. | +| Mirror lifetime | `ts-eids-s` lives at most a day and never past the earliest `exp` it holds. | + +Revision 15 applies these decisions, made by the user after the round 14 +review, which allowed breaking changes: + +| Decision | Choice | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Stager providers | None. Resolving providers run only on writer navigations, so every outcome, backoff included, is stored. Stagers serve KV | +| Removals | Written to KV on the request that carries them. `ts-eids-r`, its keyring and secrets, the overflow write and the strips-only rollout step are removed | +| Non-browser removals | Dropped, since earlier revisions could apply them only through a later real-browser navigation from the same client | +| Capture | Revoke-only: removals, opt-out and clear. Positive IDs are counted, never stored. `capture_enabled` and request-body consent are removed | +| Capture-only opt-out | Also removes the records of its sources that no provider wrote, and blocks the browser's values for them. A capture-only provider cannot own a source and must list its `sources` | +| Provider denials | A provider's `ps.iv` and `ps.ov` filter browser candidates for unowned sources too. An opt-out's removals go to `ps.ov`, which its clear empties | +| Mirror feedback | The shim never copies `ts-eids-s` into `ts-eids` or the `/auction` body | +| Dry run | Removed, with `is_dry_run` | +| KV schema | Version 2 with one record shape. Version 1 rows are read and upgraded on write. Rollback past rollout step 1 is unsupported | +| Stored hashes | `ps.iv` hashes are unkeyed and the consent fingerprint is a plain bitmask, because the row already holds the raw values. No legacy revision hash | +| Pull sync | The pull-complete marker and the legacy pull sync config keys are removed | +| Withdrawal retry | A tombstone that fails before send gets one more attempt after send | + +Decisions from earlier rounds, and the finding behind each revision, are in +the [review log](2026-10-20-server-side-identity-providers-review-log.md). From 13920fe7115c5ad704f3d04e0d450ba0c620e81e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 17:56:31 +0530 Subject: [PATCH 07/89] Close the spec section 8 gaps in the identity providers plan A found result now resets the failure count, and with_retry_in raises n on every resolution, not only no_match. Burst-capped calls get their own outcome with a 60-second wait, while a limiter error leaves ps unchanged. RemovalWriteOutcome becomes pub because FlushOutcome carries it, and the capture consent rules cite spec 4.7 instead of 5.5. The spec gains the 60-second burst-cap wait as a section 8 row and in section 4.7 step 8. The generic pull provider ID now carries a hash suffix when the domain already contains a hyphen, matching the plan, and the operator docs explain the rule. Signed-off-by: dhruv8sh --- ...26-10-22-server-side-identity-providers.md | 63 ++++++++++++++----- ...0-server-side-identity-providers-design.md | 16 +++-- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md b/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md index 9581b8980a..e6eaf186ae 100644 --- a/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md +++ b/docs/superpowers/plans/2026-10-22-server-side-identity-providers.md @@ -42,8 +42,8 @@ Copied from the spec and `AGENTS.md`. Every task implicitly includes these. - KV: `SCHEMA_VERSION` is **2**. Every record is `{w?, rv?, at?, exp?, ra?, ru?, eids: [{uids: [{id, atype?, ext?}], inserter?, matcher?, mm?, ext?}]}`, and `eids` is never empty. A version 1 record (`{uid, ...}`) reads as one entry holding that UID, and the row's next write stores version 2. Every UID-level and EID-level `ext` is a JSON object of at most **2,048** bytes as compact JSON. `inserter` and `matcher` are not blank and at most **255** bytes. A source holds at most **8** entries and **32** UIDs, and its entry set is at most **4,096** bytes as compact JSON (`MAX_ENTRY_SET_BYTES`). `ps.st` is at most **1,024** bytes of embedded JSON, a row has at most **32** `ps` entries, and `ps.iv` and `ps.ov` are each an ordered list of at most **128** hashes (`MAX_INVALIDATED_UIDS`), oldest first, truncated from the front before validation. One function, `entry_set_bytes`, measures `MAX_ENTRY_SET_BYTES` on the stored `eids` array, for both the coordinator and `KvEntry::validate`. Expired means `now >= exp`. `rv` is **16** hex digits (64 random bits). On read, a record that breaks a limit is dropped from the row and logged with its source only; only `serialize_entry` rejects. - `ProviderRequest` call context: at most **1,024** bytes of JSON. `IdentityProviderId`: `^[a-z][a-z0-9_-]{0,62}$`. - `[ec.identity]` defaults: `max_concurrency = 4`, `hold_budget_ms = 100`, `hold_retry_sec = 300`, `max_response_bytes = 65536`. Per-provider defaults: `mode = "after_send"`, `timeout_ms` = `hold_budget_ms`, `after_send_timeout_ms = 1000`, `refresh_after_sec = 86400`, `max_calls_per_minute = 1`, `breaker_failures_per_minute = 50`, `cookie_priority = 0`, `sources` = every source the descriptor declares (required for a capture-only provider). For a provider that declares `captures`: `revoke_scope = "provider"`. - - Removed config keys, each rejected with an error that names its replacement (spec 10.2): `ec.pull_sync_concurrency` (`[ec.identity].max_concurrency`), and `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]` (`refresh_after_sec` and `max_calls_per_minute = ceil(N / 60)` in `[ec.identity.providers.pull-]`). - - Backoff (spec 8): a `no_match` waits `n = now + retry_in`, default 24 h and at most 7 d. An input-caused failure waits `min(1 h × 2^(f-1), 24 h)`. A transport, 429 or 5xx failure waits `min(5 min × 2^(f-1), 1 h)`. A cut-off hold call waits `hold_retry_sec`. A skip waits `refresh_after_sec`. A due provider that was not dispatched gets `ps.n = max(ps.n, now + 60 s)`. + - Removed config keys, each rejected with an error that names its replacement (spec 10.2): `ec.pull_sync_concurrency` (`[ec.identity].max_concurrency`), and `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]` (`refresh_after_sec` and `max_calls_per_minute = ceil(N / 60)` in `[ec.identity.providers.]`). + - Backoff (spec 8): a `found` sets `f = 0`. A `no_match` waits `n = now + retry_in`, default 24 h and at most 7 d. `with_retry_in` on any other resolution makes `n` at least `now + retry_in`, after the row's own rule (so an `invalidate` gets `n = max(now, now + retry_in)`). An input-caused failure waits `min(1 h × 2^(f-1), 24 h)`. A transport, 429 or 5xx failure waits `min(5 min × 2^(f-1), 1 h)`. A cut-off hold call waits `hold_retry_sec`. A skip waits `refresh_after_sec`. A due provider the burst cap stops gets `ps.n = max(ps.n, now + 60 s)`. A provider the breaker or a limiter error stops is not dispatched and its `ps` is unchanged. - **G4 is strict.** A page load makes at most one KV write, never retried as a whole. The only exceptions (spec 5.1): - the consent-withdrawal tombstone path (a CAS with up to five attempts, then its completion marker, per withdrawn EC ID), which also runs for a `revoke_scope = "global"` capture from a real browser, and gets one more attempt after send when it fails before send (Task 27) - the removal-only retry of a failed flush (spec 5.2, phase 4, step 5; Task 29) @@ -56,7 +56,7 @@ Copied from the spec and `AGENTS.md`. Every task implicitly includes these. - **Behaviour that predates the spec does not change (G13).** G13 itself changes none of the paths that predate the spec. Browser ingest, `ts-eids`, batch sync, generic pull, `/auction` body forwarding and the `/identify` response keep the behaviour the rest of the spec gives them, and every record they write is one entry with one UID: one UID per registered source, picked by today's rule (`collect_eid_updates`, `ec/prebid_eids.rs:146-151`: for a repeated source, the first valid UID of its last entry, with the sharedId cookie applied last), and nothing for an unregistered source. Only a provider result writes more, and no Part 1 code writes more. A version 1 row goes out to bidders exactly as it does on `main`. Golden tests, named `golden_*` so `cargo test-fastly -- golden` runs them all, pin these paths (Tasks 6, 7, 28 and 30). - **Feature gate for test support.** Gate test-only code on `#[cfg(any(test, feature = "test-utils"))]`, the feature `trusted-server-core` already has (`crates/trusted-server-core/Cargo.toml`) and spec 13 names. `ec::identity::test_support` is a `pub mod` under that gate, and every item in it is `pub`, from the task that creates it, as existing `test-utils` items are (`IntegrationRegistry::from_request_filters`). A `pub(crate)` item behind the feature is dead code in the non-test library build that `cargo clippy-fastly` lints with `--all-features`, so `-D warnings` fails it. The Fastly adapter's dev-dependencies already enable the feature (`crates/trusted-server-adapter-fastly/Cargo.toml`). - **Adapter-facing API.** The Fastly adapter is a separate crate. It reaches identity state only through `pub struct IdentityRuntime` (Task 23) and `pub` opaque wrappers such as `BrowserSeed` (Task 25). Core types stay `pub(crate)`, and no `pub` item exposes a `pub(crate)` type: `private_interfaces` is an error under `-D warnings`. -- **Lints that bite plan code.** Backtick `OpenRTB`, CamelCase vendor names such as `LiveRamp` and `SharedID`, and other code-like words in doc comments (`clippy::doc_markdown`). `clippy::panic` is denied workspace-wide, tests included. On `wasm32-wasip1` a panic aborts the whole test binary under Viceroy and `#[should_panic]` tests are ignored, so a check that a test must see fail returns a `Result` instead of panicking. +- **Lints that bite plan code.** Backtick `OpenRTB`, CamelCase vendor names such as `LiveRamp` and `SharedID`, and other code-like words in doc comments (`clippy::doc_markdown`). `clippy::panic` is denied in production code; `clippy.toml` allows `panic!` in tests. On `wasm32-wasip1` a panic aborts the whole test binary under Viceroy and `#[should_panic]` tests are ignored, so a check that a test must see fail returns a `Result` instead of panicking. - Test discipline: 1. **Red first.** Run each new test against unchanged code and watch it fail for the expected reason before you implement. 2. **Pair absence with presence.** Each "X is gone" assertion sits next to a "control Y is still there" assertion on the same value. @@ -2001,7 +2001,7 @@ git commit --signoff -S -m "Record when browser ingest acquired each ID" -m "Ing - `ProviderDescriptor` and its getters: `builder`, `id`, `sources`, `primary_source`, `source_named`, `resolves`, `endpoint`, `captures`, `consent_policy`, `provenance`, `is_expiring` - `ProviderDescriptorBuilder`: `with_source`, `with_resolves`, `with_captures`, `with_consent_policy`, `with_expiring_ids`, `with_expiring_source`, `with_provenance`, `build` - `Resolves`: `new`, `with_endpoint(&str)`, `with_host(&str)`, `with_needs`, `with_needs_for`, `with_wants`, `with_kinds`, `with_private_state`, plus `pub(in crate::ec::identity) fn with_ec_id(self)` and crate-visible getters `hosts()`, `needs(kind)`, `wants()`, `kinds()`, `has_private_state()`, `gets_ec_id()` - - `Captures`: `new`, `with_path(Method, &str)`, `with_path_prefix(&str)`, plus crate-visible `matches(&Method, &str) -> bool`, `paths()`, `prefixes()`. There is no body-consent rule: capture is gated by the request's own consent, from its cookies and headers (spec 5.5). + - `Captures`: `new`, `with_path(Method, &str)`, `with_path_prefix(&str)`, plus crate-visible `matches(&Method, &str) -> bool`, `paths()`, `prefixes()`. There is no body-consent rule: capture is gated by the request's own consent, from its cookies and headers (spec 4.7). - `EidProvenance`: `new`, `with_inserter`, `with_matcher`, `with_mm`, `inserter`, `matcher`, `mm` - `ConsentPolicy`: `strict`, `with_tcf_purposes`, `with_tcf_vendor`, `allowing_dnt`, `allowing_gpc`, `allowing_us_opt_out`, plus `pub(in crate::ec::identity) fn eid_persistence_only()` for generic pull and crate-visible getters - `UsOptOut { Sale, Sharing, TargetedAdvertising }` @@ -2648,7 +2648,7 @@ impl Resolves { /// Revoke-only capture from responses TS proxies for this provider: it /// can remove IDs, opt out or clear an opt-out, and never stores a positive -/// ID (spec 5.5). +/// ID (spec 4.7). #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct Captures { paths: Vec<(Method, String)>, @@ -5368,7 +5368,7 @@ impl<'a> CaptureInput<'a> { } /// The request's consent, from its cookies and headers. The request - /// body is never read (spec 5.5). + /// body is never read (spec 4.7). #[must_use] pub fn consent(&self) -> &ProviderConsent { self.consent @@ -7236,7 +7236,7 @@ mod tests { "should hold the entries exactly as bidders receive them" ); assert_eq!(max_age, MIRROR_MAX_AGE_SEC, "should live a day without an exp"); - let MirrorPayload::Value { value, .. } = &payload else { unreachable!() }; + let MirrorPayload::Value { value, .. } = &payload else { panic!("should hold a value") }; assert_eq!(payload.to_json(), json!({ "value": value, "max_age": MIRROR_MAX_AGE_SEC }), "should serialize"); } @@ -7435,7 +7435,7 @@ git commit --signoff -S -m "Add the ts-eids-s mirror payload and its size cap" - - Produces: - `KvIdentityGraph::write_if_generation(&self, ec_id: &str, entry: &KvEntry, generation: u64) -> Result>`, `pub(crate)`: serializes with `serialize_entry` (so `validate` runs) and writes with `ENTRY_TTL` and `EcKvWriteMode::IfGenerationMatch(generation)`. Phase 4's flush (Task 29) uses it too, so every identity write goes through one path. - `pub(crate) const REMOVAL_WRITE_ATTEMPTS: u32 = MAX_CAS_RETRIES;` (5) - - `pub(crate) enum RemovalWriteOutcome { Written, NothingToWrite, Failed }`. `NothingToWrite` covers a tombstone, `consent.ok = false`, a missing row and removals that are already applied. `Failed` covers a failed read, a non-CAS write error such as Fastly's per-key limit, and five generation mismatches in a row. + - `pub enum RemovalWriteOutcome { Written, NothingToWrite, Failed }`. `NothingToWrite` covers a tombstone, `consent.ok = false`, a missing row and removals that are already applied. `Failed` covers a failed read, a non-CAS write error such as Fastly's per-key limit, and five generation mismatches in a row. It is `pub`, not `pub(crate)`, because the public `FlushOutcome::WriteFailed` (Task 29) carries it, and `private_interfaces` would fail the clippy gate otherwise. - `pub(crate) fn write_removals(graph: &KvIdentityGraph, ec_id: &str, removals: &IdentityMutation, context: &ApplyContext<'_>) -> RemovalWriteOutcome` - `pub(crate) fn log_removal_lost(ec_id: &str, removals: &IdentityMutation)`: logs at error with `log_id(ec_id)` and the provider IDs and sources from `removal_labels()`, never a UID. The caller increments `removal_lost` (Task 23 adds the counter to `RequestIdentity`). @@ -7616,7 +7616,7 @@ pub(crate) const REMOVAL_WRITE_ATTEMPTS: u32 = MAX_CAS_RETRIES; /// How a removal write ended. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum RemovalWriteOutcome { +pub enum RemovalWriteOutcome { /// The removals are in KV. Written, /// Nothing to write: a tombstone, a missing row, or already applied. @@ -7869,7 +7869,7 @@ git commit --signoff -S -m "Add [ec.identity] coordinator settings" -m "Adds coo - `pub(crate) struct EligibilityInput<'a> { provider: &'a dyn IdentityProvider, settings: &'a ProviderSettings, identity: &'a IdentitySettings, catalog: &'a ProviderCatalog, consent: &'a ConsentContext }`, so `eligibility` stays within 7 arguments - `pub(crate) fn eligibility(input: &EligibilityInput<'_>, view: &EligibilityView, facts: &RequestFacts, phase: Phase) -> Eligibility` - `pub(crate) fn fingerprint_bits(policy: &ConsentPolicy, consent: &ConsentContext, signals: PolicySignals) -> String`: the consent an opt-out's `fp` records, as a plain lowercase hex bitmask of at most 16 digits (`format!("{bits:x}")` of a `u64`), never a hash (spec 8, 9.3). Part 1 covers the signals TS decodes today, one bit each in this order: each TCF purpose the policy reads (1 to 10), the vendor bit when the policy names one, the US sale opt-out (from `us_privacy` or GPP), GPC and DNT. A signal the policy relaxes stays 0. Task 34 appends the sharing and targeted-advertising opt-outs after these and keeps these bits where they are. - - `pub(crate) enum CallOutcome { Resolved(Resolution), Skipped, InputUnavailable, InvalidState(Option), HoldCutOff, InputFailure, TransportFailure, BreakerOpen }` + - `pub(crate) enum CallOutcome { Resolved(Resolution), Skipped, InputUnavailable, InvalidState(Option), HoldCutOff, InputFailure, TransportFailure, BreakerOpen, BurstCapped, NotDispatched }`. `BurstCapped` is the burst cap's skip; `NotDispatched` is a limiter error's - `pub(crate) fn outcome_to_changes(provider: &dyn IdentityProvider, outcome: CallOutcome, view: &EligibilityView, context: &OutcomeContext<'_>) -> OutcomeChanges`, where `OutcomeChanges { changes: Vec, result: Option, state: ProviderStateChange }`. This is the section 8 table, row by row. `changes` holds every removal, and the positive changes of a provider without private state. `OutcomeContext` holds `now`, `catalog`, provider settings, `identity` settings, `expected_state: Option` and the consent fingerprint bits from `fingerprint_bits`. Eligibility steps, in order (spec 4.7): @@ -7897,13 +7897,15 @@ How `Resolved(resolution)` becomes changes (spec 4.5, "From entries to an entry 5. A result that named at least one source and all of whose sets are discarded is handled as `Malformed` (spec 4.5, section 8): `f += 1` with its backoff, and its removals still apply. Timing comes only from kept entries (Task 10). 6. `EntrySet.refresh_until` from a provider without private state is ignored with a warning. 7. Each kept set becomes `KvPartnerId { w: Some(provider), at: Some(now), exp, ra, ru, eids: set.entries, .. }`, with `exp`, `ra` and `ru` from the `EntrySet`. A one-UID result writes the same one-entry shape browser ingest writes. Descriptor provenance is not stored: Task 28 fills it in at output. + 7a. A `Resolution::found` that steps 4, 5 and 12 do not turn into `Malformed` sets `f = 0`, even when `entry_sets` dropped every entry and nothing is stored. It sets `f = 0` in the state it writes, in the `ProviderResult` for a provider with private state and in `state` otherwise. 8. Each record becomes `OwnerWrite` with `expected = Some(record_revision(stored))` (or `None` when no record) when this provider owns the source, else `FillMissing`. Either change replaces the source's set whole. A source the result does not name gets no change. 9. `invalidated` becomes `OwnerRemove { source, provider, expected: record_revision(stored) }` for each named source that has a record in the view, plus `n = now`. It carries no hash: `mutation::apply` (Task 15) hashes every UID of the live record it removes into the provider's `ps.iv`. A named source with no record makes no change. 10. `opted_out` becomes `WriterRemove { source, writer: provider }` for every source in the view whose record this provider wrote, and, for a capture-only provider, for every one of its `capture_only_sources` whose record has no writer. Plus `opt_out = KvOptOut { at: now, fp: fingerprint_bits }` and `st = Clear`. The mutation hashes each removed record into `ps.ov`. 11. When the provider has private state, the positive changes plus the state change plus `n` and `f` go into one `ProviderResult` with `expected_state`. Otherwise they go into `changes`, and `n` and `f` into `state`. 12. `state_rejected()` drops the positive part and applies the `Malformed` backoff. +13. `with_retry_in` on any resolution, not only `no_match`: after steps 1 to 12 set `n`, raise it to `max(n, now + retry_in)`, with `retry_in` capped at 7 d. A `found`, `invalidate` or `opted_out` with no `n` of its own gets `n = now + retry_in`. -The other rows of section 8: `Skipped` and `InputUnavailable` set `n = now + refresh_after_sec`. `InvalidState` clears `st`, keeps `f`, and sets `n = now + refresh_after_sec` unless its resolution sets `n`. `HoldCutOff` sets `n = now + hold_retry_sec` and never counts a failure; only a hold dispatch before the auction can be cut off. An after-send timeout is a `TransportFailure`. `InputFailure` is `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)`. `TransportFailure` is `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`. `BreakerOpen` changes nothing. Backoff arithmetic uses `saturating_pow` and `min`, so a large `f` cannot overflow. +The other rows of section 8: `Skipped` and `InputUnavailable` set `n = now + refresh_after_sec`. `BurstCapped` sets `n = max(n, now + 60)` and never counts a failure (spec 4.7 step 8). `InvalidState` clears `st`, keeps `f`, and sets `n = now + refresh_after_sec` unless its resolution sets `n`. `HoldCutOff` sets `n = now + hold_retry_sec` and never counts a failure; only a hold dispatch before the auction can be cut off. An after-send timeout is a `TransportFailure`. `InputFailure` is `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)`. `TransportFailure` is `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`. `BreakerOpen` and `NotDispatched` change nothing. Backoff arithmetic uses `saturating_pow` and `min`, so a large `f` cannot overflow. - [ ] **Step 1: Write the failing tests** @@ -7975,6 +7977,35 @@ Here are seven written out in full. Write the rest in the same shape: assert_eq!(long.state.n, Some(1_000 + 7 * 86_400), "should cap the wait at 7 days"); } + #[test] + fn found_resets_failures_and_retry_in_applies_to_every_resolution() { + let provider = fixed_provider("p", &["a.example.com"]); + let a: SourceDomain = "a.example.com".parse().expect("should parse the source"); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 3); + view.set_record_for_test(&a, record_for_test("example-old", Some("p"), 900)); + let fixture = OutcomeFixture::new(); + let context = fixture.context_at(1_000); + + let found = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new(&a, ResolvedUid::new("u"))]).with_retry_in(Duration::from_secs(600))), + &view, + &context, + ); + let invalidated = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::invalidate(&a).with_retry_in(Duration::from_secs(600))), + &view, + &context, + ); + + assert_eq!(found.state.f, Some(0), "should reset the failure count on found"); + assert_eq!(found.state.n, Some(1_000 + 600), "should wait at least retry_in after found"); + assert_eq!(invalidated.changes.len(), 1, "control: should remove the stored record"); + assert_eq!(invalidated.state.n, Some(1_000 + 600), "should raise invalidate's n = now to retry_in"); + } + #[test] fn transport_failure_backs_off_to_an_hour() { let provider = fixed_provider("p", &["a.example.com"]); @@ -8152,7 +8183,7 @@ Dispatch per call: - Any recorded undeclared read is also logged as `Config` and turns the outcome into `InputFailure`. 2. `check_request`. On failure: log `Config`, `InputFailure`. 3. Order: calls are dispatched in a stable order. Generic pull providers keep today's fairness rule (spec 10.4, `pull_sync.rs:143-156`): sorted by source domain, then rotated left by `(now / 3600) % len`, so the starting partner changes each hour when `max_concurrency` limits a batch. Other providers follow, by provider ID. -4. The burst cap: `rate_limiter.exceeded_per_minute(&format!("identity:{provider}:{}", visitor_key(keys, ec_id)), settings.max_calls_per_minute)`. Exceeded means not dispatched: the call's outcome is `Skipped` with `n = max(n, now + 60)` (Global Constraints), never a failure. A limiter error is logged and the call is skipped the same way. +4. The burst cap: `rate_limiter.exceeded_per_minute(&format!("identity:{provider}:{}", visitor_key(keys, ec_id)), settings.max_calls_per_minute)`. Exceeded means not dispatched: the call's outcome is `BurstCapped`, which sets `n = max(n, now + 60)` (Task 20), never a failure. A limiter error is logged and the call's outcome is `NotDispatched`: not dispatched, `ps` unchanged (Task 35). 5. `ensure` a backend with first-byte timeout `after_send_timeout_ms` after send, or for a hold dispatch `min(hold_timeout_ms, remaining budget)`, rounded up to the orchestrator's buckets (`auction/orchestrator.rs`, search for the bucket rounding). Do not follow redirects. Send with `send_async`, keeping at most `[ec.identity].max_concurrency` in flight. 6. Collect responses: check `Content-Length` against `max_response_bytes`, buffer, `decode_body`. Over the cap is `TransportFailure`. Build the `ProviderResponse` with the request's call context. Call `response`. - `Ok(resolution)` is `Resolved`, or `InvalidState(Some(resolution))` when the input recorded a state decode failure. @@ -8270,7 +8301,7 @@ Its `request` is a `GET` of `validated_pull_sync_url(partner.pull_sync_url.as_de - The body is `success_json::<{ uid: Option }>()`. `None`, an empty UID or one over `MAX_UID_LENGTH` (`is_valid_eid_uid`) is `no_match`, never `Malformed`, so a partner's "no user" answer waits the `no_match` 24 h instead of counting a failure. - Any other UID is `found([ResolvedEid::new(&source, ResolvedUid::new(uid))])`: one entry, one UID, no `atype`, no `ext` and no EID-level fields, so the coordinator stores the one-entry record pull sync stores today (G13). -A generic pull provider's settings are `settings.ec.identity.provider(&generic_pull_id(&partner.source_domain))` like any other provider's: `ProviderSettings::default()` unless the operator writes an `[ec.identity.providers.pull-]` table. Pacing moves there (spec 10.4): an operator who had `pull_sync_rate_limit = N` per hour sets `max_calls_per_minute = ceil(N / 60)`, and one who had `pull_sync_ttl_sec` sets `refresh_after_sec`. The changelog (Task 31) gives both conversions. +A generic pull provider's settings are `settings.ec.identity.provider(&generic_pull_id(&partner.source_domain))` like any other provider's: `ProviderSettings::default()` unless the operator writes an `[ec.identity.providers.]` table, where `` is `generic_pull_id(source_domain)` (below). The unknown-provider validation error lists every configured provider ID, so an operator who writes the plain form for a hyphenated domain sees the real ID. Pacing moves there (spec 10.4): an operator who had `pull_sync_rate_limit = N` per hour sets `max_calls_per_minute = ceil(N / 60)`, and one who had `pull_sync_ttl_sec` sets `refresh_after_sec`. The changelog (Task 31) gives both conversions. Until Task 29 deletes `dispatch_pull_sync`, `ec/pull_sync.rs` reads the same table: its rate check becomes `rate_limiter.exceeded_per_minute(&rate_key, provider_settings.max_calls_per_minute)` (`pull_sync.rs:200`), and its marker TTL uses `provider_settings.refresh_after_sec`. Its tests set those through `settings.ec.identity.providers` instead of the removed partner fields. @@ -8367,7 +8398,7 @@ Expected: PASS. ```bash git add -A crates/trusted-server-core/src -git commit --signoff -S -m "Model pull sync partners as GenericPullProvider" -m "Adds the provider harness basics and a KV store that counts writes by mode. Generic pull results stay one UID per source, a 404 or null UID stays a no-op, and one catalog validator rejects provider tables it cannot apply." -m "pull_sync_ttl_sec and pull_sync_rate_limit are removed from [[ec.partners]]: pacing comes from each pull partner's [ec.identity.providers.pull-] table, and a config that sets either key fails validation with an error naming its replacement." +git commit --signoff -S -m "Model pull sync partners as GenericPullProvider" -m "Adds the provider harness basics and a KV store that counts writes by mode. Generic pull results stay one UID per source, a 404 or null UID stays a no-op, and one catalog validator rejects provider tables it cannot apply." -m "pull_sync_ttl_sec and pull_sync_rate_limit are removed from [[ec.partners]]: pacing comes from each pull partner's [ec.identity.providers.] table, and a config that sets either key fails validation with an error naming its replacement." ``` --- @@ -9005,7 +9036,7 @@ Phase 4 (`run_after_send`, spec 5.2 steps 1 to 5): 4. Recompute eligibility steps 4 and 5 for each after-send provider from the live row, rebuild its request from it (the live `ps.st` for a stateful provider), dispatch with `coordinator::dispatch(.., budget: None)`, and turn the outcomes into changes added to the mutation. 5. If the live row has no `cluster_size`, add it with `compute_cluster_size` and `set_cluster_size`. 6. `mutation::apply` against the live entry. If `changed`, write once with `IfGenerationMatch(generation)`: `Written`, or `NoChange` when nothing changed. Never retry the whole flush. -7. On a failed write (a precondition failure, Fastly's per-key rate limit or any other error), log at `warn` with `log_id`. When `mutation.removals()` (Task 15) is not empty, run `write_removals(graph, ec_id, &removals, &context)` (Task 18): a re-read, the removal rules of section 5.4, and up to five attempts. `Failed` there calls `log_removal_lost` and `RequestIdentity::record_removal_lost` (Task 23) once per lost removal. Return `WriteFailed { removals }` with that outcome, or `NothingToWrite` when the mutation held no removal. +7. On a failed write (a precondition failure, Fastly's per-key rate limit or any other error), log at `warn` with `log_id`. When `mutation.removals()` (Task 15) is not empty, run `write_removals(graph, ec_id, &removals, &context)` (Task 18): a re-read, the removal rules of section 5.4, and up to five attempts. `Failed` there calls `log_removal_lost` and `RequestIdentity::record_removal_lost` (Task 23) once per lost removal. Return `FlushOutcome::WriteFailed { removals }` with that `RemovalWriteOutcome`, using `RemovalWriteOutcome::NothingToWrite` when the mutation held no removal. `mutation::apply` writes each source's record whole (Task 15), so a multi-entry record the mutation does not touch keeps its `eids` byte for byte, and a version 1 row upgraded on write serializes as Task 6's golden expects. @@ -9160,7 +9191,7 @@ Add `identity_measurement_records_counts_for_a_writer_navigation` (a writer navi - [ ] **Step 2: Docs** - `edge-cookies.md`: add a section for `__Host-ts-eids-s`: a client-only mirror of the server IDs that the shim writes from `/auction` and `/_ts/page-bids` responses for page scripts, with `Path=/; Secure; SameSite=Lax`, not `HttpOnly`, a `Max-Age` of at most one day and never past the earliest ID expiry, and a 2,048-byte cap that leaves out whole sources by `cookie_priority`. The server never reads it, and it is stripped from origin and Prebid Server requests (Tasks 1 to 4). Note that `ts-ec-pull-complete` is gone. Describe the writer-request rule in plain words: only a full page navigation from a returning visitor writes KV, once, after the page is sent. Say that the shim's `ts-eids` trim now keeps registered sources first. -- `configuration.md`: the `[ec.identity]` table, with every key, its default and its meaning, from spec 10.1. Name `revoke_scope` as applying from Part 2. Name the removed keys and their replacements: `ec.pull_sync_concurrency` becomes `[ec.identity].max_concurrency`; `pull_sync_ttl_sec` becomes `refresh_after_sec` and `pull_sync_rate_limit = N` becomes `max_calls_per_minute = ceil(N / 60)`, both in `[ec.identity.providers.pull-]` (Tasks 19 and 22). Validation rejects the old keys. The capture provider checklist lands with capture, in Task 41. +- `configuration.md`: the `[ec.identity]` table, with every key, its default and its meaning, from spec 10.1. Name `revoke_scope` as applying from Part 2. Name the removed keys and their replacements: `ec.pull_sync_concurrency` becomes `[ec.identity].max_concurrency`; `pull_sync_ttl_sec` becomes `refresh_after_sec` and `pull_sync_rate_limit = N` becomes `max_calls_per_minute = ceil(N / 60)`, both in `[ec.identity.providers.]` (Tasks 19 and 22). Give the ID rule with examples: `ssp.example.com` is `pull-ssp-example-com`, and a domain that contains `-`, or an ID over 63 characters, also gets `-` and 8 hex digits of the domain's SHA-256 (spec 10.4). Validation rejects the old keys. The capture provider checklist lands with capture, in Task 41. - `ec-setup-guide.md`: pull partners now run from a visitor's second page load, back off on errors, and are paced by their provider table. - `api-reference.md`: `/_ts/api/v1/identify`, `/auction` and `/_ts/page-bids` no longer write KV, and `/identify` omits expired IDs. `/identify`'s response shape is unchanged: one UID per source, even for a record that holds several. Document `ext.ts_eids_s` in the `/auction` response and `ts_eids_s` in the page-bids body, with the `value`, `clear` and absent cases (spec 6.2). - `configuration.md`, a "Rolling back" subsection after the `[ec.identity]` table (spec 14). This release moves KV to schema version 2, and a release before it cannot read a version 2 row, so a rollback past it is unsupported once it has written. For a later vendor release rolled back to this one: remove `owner` and every vendor `[ec.identity.providers.*]` table from the config first, because this release rejects an owner or a provider table it does not know; entry sets stay readable but are not served while their writer is not configured; and a write prunes an unconfigured provider's `ps` to `o`, `oc` and `ov`, so opt-outs survive while `ps.iv` and `ps.st` do not. diff --git a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md index 4074a4ae3b..c0c981480c 100644 --- a/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md +++ b/docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md @@ -1507,7 +1507,9 @@ request when all of these hold: `max_calls_per_minute` on the key `identity::`, where `visitor` is an HMAC of the full EC ID with a key derived from `ec.passphrase`. It is not `ec_hash`, which everyone behind one IP shares - (section 3.4). + (section 3.4). A due provider the cap stops is not dispatched and gets + `n = max(n, now + 60 s)`, one limiter window, and never a failure + (section 8). Pacing per visitor comes from `ps.n` and the refresh rules in step 4, not from the limiter. The limiter only caps bursts, because its windows are 60 @@ -2583,6 +2585,7 @@ write (section 5.3), and nothing else. | `Rejected` with a 4xx other than 429, `Config`, `UndeclaredInput`, `Malformed` | unchanged | `f += 1`, `n = now + min(1 h × 2^(f-1), 24 h)` | warn | | `Rejected` with 429 or 5xx, `Transport`, timeout | unchanged | `f += 1`, `n = now + min(5 min × 2^(f-1), 1 h)`; breaker `record` | info | | Breaker open | not dispatched | unchanged | debug | +| Burst cap exceeded (section 4.7 step 8) | not dispatched | `n = max(n, now + 60 s)`; never a failure | debug | | Capture: any result or error | removals as above; positive IDs counted in `capture_unpersisted`, never stored | only `o`, `oc`, `iv` and `ov`; never `n`, `f` or the breaker | debug | | Captured `opted_out`, `revoke_scope = "global"`, real browser | the withdrawal tombstone removes the row (section 5.1) | removed with the row | info | @@ -2927,9 +2930,14 @@ object-safe. Each `[[ec.partners]]` entry with `pull_sync_enabled = true` registers a built-in `GenericPullProvider`. Its ID is `pull-` plus the source domain with -dots replaced by `-` (for example `pull-ssp-example-com`). An ID that would -exceed 63 characters is cut and given an 8-character hash suffix of the full -domain, so IDs stay unique and valid. +dots replaced by `-` (for example `pull-ssp-example-com`). A domain that +already contains `-` also gets `-` and the first 8 hex digits of the SHA-256 +of the full domain, so `a-b.example.com` and `a.b.example.com` get different +IDs. An ID that would exceed 63 characters is cut and given the same hash +suffix, so IDs stay unique and valid. +No earlier build stored a generic pull provider ID: before this spec, pull +sync kept no per-provider state and wrote no `w`, so the hash rule renames +nothing that is persisted. Unchanged: From 5c3bd5ee958ab1f67f83afe06c4812a90fb2a320 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:10:44 +0530 Subject: [PATCH 08/89] Add identity cookie lists and a cookie strip by name Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/constants.rs | 3 + crates/trusted-server-core/src/cookies.rs | 124 +++++++++++++++++++- 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index 1fd3c2382c..458786db01 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -6,6 +6,9 @@ pub const COOKIE_TS_EC_PULL_COMPLETE: &str = "ts-ec-pull-complete"; /// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; +/// Client-only mirror of the servable server-side EIDs, written by the shim +/// and never read by the server (spec 6.2). +pub const COOKIE_TS_EIDS_S: &str = "__Host-ts-eids-s"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; pub const COOKIE_SHAREDID: &str = "sharedId"; diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 3fe6719ef7..24bb261c7f 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -11,7 +11,10 @@ use error_stack::{Report, ResultExt as _}; use http::Request; use http::header; -use crate::constants::{COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_US_PRIVACY}; +use crate::constants::{ + COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_TS_EC, COOKIE_TS_EIDS, + COOKIE_TS_EIDS_S, COOKIE_US_PRIVACY, +}; use crate::error::TrustedServerError; /// Cookie names carrying privacy consent signals. @@ -26,6 +29,19 @@ pub const CONSENT_COOKIE_NAMES: &[&str] = &[ COOKIE_US_PRIVACY, ]; +/// Trusted Server identity cookies that never reach a third party. +/// +/// Stripped from Prebid Server requests and never added to proxy or provider +/// requests. +pub const TS_IDENTITY_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; + +/// Identity cookies removed from publisher-origin requests. +/// +/// `ts-ec` still reaches the origin. The ID cookies +/// are removed so that they cannot push the origin's `Cookie` header past +/// common 8 KB header limits. +pub const TS_ORIGIN_STRIPPED_COOKIE_NAMES: &[&str] = &[COOKIE_TS_EIDS, COOKIE_TS_EIDS_S]; + /// Parses a cookie string into a [`CookieJar`]. /// /// Returns an empty jar if the cookie string is unparseable. @@ -104,6 +120,52 @@ pub fn strip_cookies(cookie_header: &str, cookie_names: &[&str]) -> String { .join("; ") } +/// Strips cookies whose name is exactly one of `names` from a `Cookie` header +/// value. +/// +/// Returns an empty string when nothing is left. +#[must_use] +pub fn strip_cookie_pairs(cookie_header: &str, names: &[&str]) -> String { + cookie_header + .split(';') + .map(str::trim) + .filter(|pair| !pair.is_empty()) + .filter(|pair| { + let name = pair.split('=').next().unwrap_or_default().trim(); + !names.contains(&name) + }) + .collect::>() + .join("; ") +} + +/// Strips matching cookies from every `Cookie` field in `headers`. +/// +/// All `Cookie` fields are read, matching pairs are removed, and the rest is +/// written back as one field. A field that is not valid UTF-8 is dropped +/// rather than forwarded, because it cannot be checked. The header is removed +/// when nothing is left. +pub fn strip_cookie_headers(headers: &mut http::HeaderMap, names: &[&str]) { + let retained = headers + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .map(|value| strip_cookie_pairs(value, names)) + .filter(|value| !value.is_empty()) + .collect::>() + .join("; "); + + headers.remove(header::COOKIE); + if retained.is_empty() { + return; + } + match http::HeaderValue::from_str(&retained) { + Ok(value) => { + headers.insert(header::COOKIE, value); + } + Err(_) => log::debug!("Dropping Cookie header that is invalid after stripping"), + } +} + /// Copies the `Cookie` header from one request to another, optionally /// stripping consent cookies. /// @@ -451,4 +513,64 @@ mod tests { let stripped = strip_cookies(header, CONSENT_COOKIE_NAMES); assert_eq!(stripped, "session=abc=123=def"); } + + #[test] + fn strip_cookie_pairs_removes_exact_names_only() { + let header = + "ts-ec=abc; ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ; other=1; ts-eids-extra=keep"; + + let stripped = strip_cookie_pairs(header, TS_ORIGIN_STRIPPED_COOKIE_NAMES); + + assert_eq!( + stripped, "ts-ec=abc; other=1; ts-eids-extra=keep", + "should drop the ID cookies and keep ts-ec and a cookie that only shares a prefix" + ); + } + + #[test] + fn strip_cookie_headers_joins_every_cookie_field_and_drops_non_utf8() { + let mut headers = http::HeaderMap::new(); + headers.append( + header::COOKIE, + http::HeaderValue::from_static("ts-ec=abc; ts-eids=W10"), + ); + headers.append( + header::COOKIE, + http::HeaderValue::from_bytes(b"bad=\xff").expect("should build an opaque header"), + ); + headers.append(header::COOKIE, http::HeaderValue::from_static("other=1")); + + strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES); + + let values: Vec<_> = headers.get_all(header::COOKIE).iter().collect(); + assert_eq!(values.len(), 1, "should collapse into one Cookie header"); + assert_eq!( + values[0], "other=1", + "should keep only the unrelated cookie" + ); + } + + #[test] + fn strip_cookie_headers_removes_header_when_nothing_is_left() { + let mut headers = http::HeaderMap::new(); + headers.insert( + header::COOKIE, + http::HeaderValue::from_static("ts-ec=abc; __Host-ts-eids-s=eyJ2IjoxfQ"), + ); + headers.insert( + header::USER_AGENT, + http::HeaderValue::from_static("ExampleBrowser/1.0"), + ); + + strip_cookie_headers(&mut headers, TS_IDENTITY_COOKIE_NAMES); + + assert!( + headers.get(header::COOKIE).is_none(), + "should remove the empty Cookie header" + ); + assert!( + headers.get(header::USER_AGENT).is_some(), + "should leave other headers alone" + ); + } } From f0a1e4f346e410004361a0f07e469be344c5b6b6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:14:44 +0530 Subject: [PATCH 09/89] Strip identity cookies from Prebid Server requests in every mode Every Cookie field is now read and checked, and a field that is not valid UTF-8 is dropped instead of forwarded. Tests that asserted a malformed Cookie field is forwarded unchanged now expect it to be dropped. Signed-off-by: dhruv8sh --- .../src/auction/routing.rs | 23 ++- .../src/integrations/prebid.rs | 170 ++++++++++++++---- 2 files changed, 154 insertions(+), 39 deletions(-) diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs index 2b8135da3c..97f74b334d 100644 --- a/crates/trusted-server-core/src/auction/routing.rs +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -203,15 +203,16 @@ impl ProviderSlotInput { /// the same legacy handling. Client-supplied `X-Forwarded-For` is never read. #[derive(Debug, Clone, Default)] pub(crate) struct PrebidTransportHeaders { - cookie: Option, + cookies: Vec, user_agent: Option, referer: Option, accept_language: Option, } impl PrebidTransportHeaders { - pub(crate) fn cookie(&self) -> Option<&HeaderValue> { - self.cookie.as_ref() + /// Every inbound `Cookie` field, in order. + pub(crate) fn cookies(&self) -> &[HeaderValue] { + &self.cookies } pub(crate) fn user_agent(&self) -> Option<&HeaderValue> { @@ -228,7 +229,12 @@ impl PrebidTransportHeaders { fn snapshot(request: &Request) -> Self { Self { - cookie: request.headers().get(header::COOKIE).cloned(), + cookies: request + .headers() + .get_all(header::COOKIE) + .iter() + .cloned() + .collect(), user_agent: request.headers().get(header::USER_AGENT).cloned(), referer: request.headers().get(header::REFERER).cloned(), accept_language: request.headers().get(header::ACCEPT_LANGUAGE).cloned(), @@ -1336,7 +1342,14 @@ mod tests { Some(attested), ); let headers = routed.prebid_transport_headers(); - assert_eq!(headers.cookie(), Some(&HeaderValue::from_static("first=1"))); + assert_eq!( + headers.cookies(), + [ + HeaderValue::from_static("first=1"), + HeaderValue::from_static("second=2") + ], + "should retain every Cookie field in order" + ); assert_eq!( headers.user_agent().expect("should retain UA").as_bytes(), b"agent-\x80" diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index a7db8624e4..f28ddfb297 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -36,7 +36,7 @@ use crate::auction::types::{AuctionContext, AuctionRequest, MediaType}; use crate::auction::types::{AuctionResponse, Bid as AuctionBid}; use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; use crate::consent_config::ConsentForwardingMode; -use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; +use crate::cookies::{CONSENT_COOKIE_NAMES, TS_IDENTITY_COOKIE_NAMES, strip_cookie_headers}; use crate::error::TrustedServerError; #[cfg(test)] use crate::http_util::RequestInfo; @@ -2026,8 +2026,14 @@ fn copy_request_headers( consent_forwarding: ConsentForwardingMode, client_ip: Option, ) { + let cookies: Vec = from + .headers() + .get_all(header::COOKIE) + .iter() + .cloned() + .collect(); apply_prebid_header_values( - from.headers().get(header::COOKIE), + &cookies, from.headers().get(header::USER_AGENT), from.headers().get(header::REFERER), from.headers().get(header::ACCEPT_LANGUAGE), @@ -2045,7 +2051,7 @@ pub(crate) fn apply_prebid_transport_headers( client_ip: Option, ) { apply_prebid_header_values( - from.cookie(), + from.cookies(), from.user_agent(), from.referer(), from.accept_language(), @@ -2060,7 +2066,7 @@ pub(crate) fn apply_prebid_transport_headers( reason = "the helper preserves four independently optional raw headers plus transport policy" )] fn apply_prebid_header_values( - cookie: Option<&HeaderValue>, + cookies: &[HeaderValue], user_agent: Option<&HeaderValue>, referer: Option<&HeaderValue>, accept_language: Option<&HeaderValue>, @@ -2085,27 +2091,18 @@ fn apply_prebid_header_values( .insert(header::HeaderName::from_static("x-forwarded-for"), value); } - let Some(cookie_value) = cookie else { - return; - }; - if !consent_forwarding.strips_consent_cookies() { - to.headers_mut() - .insert(header::COOKIE, cookie_value.clone()); - return; - } - match cookie_value.to_str() { - Ok(value) => { - let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); - if !stripped.is_empty() - && let Ok(cookie_header) = HeaderValue::from_str(&stripped) - { - to.headers_mut().insert(header::COOKIE, cookie_header); - } - } - Err(_) => { - to.headers_mut() - .insert(header::COOKIE, cookie_value.clone()); - } + // Identity cookies are stripped in every mode, before the consent mode is + // checked, so no mode forwards the raw header. + let mut cookie_headers = http::HeaderMap::new(); + for value in cookies { + cookie_headers.append(header::COOKIE, value.clone()); + } + strip_cookie_headers(&mut cookie_headers, TS_IDENTITY_COOKIE_NAMES); + if consent_forwarding.strips_consent_cookies() { + strip_cookie_headers(&mut cookie_headers, CONSENT_COOKIE_NAMES); + } + if let Some(cookie_header) = cookie_headers.remove(header::COOKIE) { + to.headers_mut().insert(header::COOKIE, cookie_header); } } @@ -9757,7 +9754,7 @@ set = { networkId = 42 } } #[test] - fn planned_transport_preserves_malformed_cookie_bytes_and_attested_xff() { + fn planned_transport_drops_malformed_cookie_bytes_and_uses_attested_xff() { let inbound = http::Request::builder() .uri("https://publisher.example/auction") .header( @@ -9796,13 +9793,9 @@ set = { networkId = 42 } routed.attested_client_ip(), ); - assert_eq!( - outbound - .headers() - .get(header::COOKIE) - .map(HeaderValue::as_bytes), - Some(b"session=fictional;\xffbroken".as_slice()), - "should forward malformed cookie bytes unchanged" + assert!( + outbound.headers().get(header::COOKIE).is_none(), + "should drop a non-UTF-8 Cookie field instead of forwarding it" ); assert_eq!( outbound @@ -9870,6 +9863,115 @@ set = { networkId = 42 } ); } + fn cookie_values(request: &http::Request) -> Vec { + request + .headers() + .get_all(header::COOKIE) + .iter() + .map(|value| value.to_str().expect("should be ASCII").to_owned()) + .collect() + } + + #[test] + fn prebid_headers_strip_identity_cookies_when_consent_cookies_are_forwarded() { + let cookies = [ + HeaderValue::from_static("ts-ec=abc; euconsent-v2=tcf; other=1"), + HeaderValue::from_static("__Host-ts-eids-s=eyJ2IjoxfQ; partner=p"), + HeaderValue::from_static("ts-eids=W10; second=2"), + ]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::Both, + None, + ); + + assert_eq!( + cookie_values(&to), + vec!["euconsent-v2=tcf; other=1; partner=p; second=2".to_owned()], + "should drop identity cookies, keep consent and unrelated cookies, and read every field" + ); + } + + #[test] + fn prebid_headers_strip_identity_and_consent_cookies_in_openrtb_only_mode() { + let cookies = [HeaderValue::from_static( + "ts-ec=abc; euconsent-v2=tcf; other=1", + )]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::OpenrtbOnly, + None, + ); + + assert_eq!( + cookie_values(&to), + vec!["other=1".to_owned()], + "should keep only other=1" + ); + } + + #[test] + fn prebid_headers_drop_non_utf8_cookie_fields_and_empty_results() { + let cookies = [ + HeaderValue::from_bytes(b"x=\xff").expect("should build an opaque header"), + HeaderValue::from_static("ts-ec=abc"), + ]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::Both, + None, + ); + + assert!( + to.headers().get(header::COOKIE).is_none(), + "should send no Cookie header" + ); + } + + #[test] + fn prebid_headers_drop_non_utf8_field_but_strip_identity_from_the_rest() { + let cookies = [ + HeaderValue::from_bytes(b"ts-ec=\xff; leaked=1") + .expect("should build an opaque header"), + HeaderValue::from_static("ts-ec=abc; ts-eids=W10; keep=1"), + ]; + let mut to = http::Request::new(EdgeBody::empty()); + + apply_prebid_header_values( + &cookies, + None, + None, + None, + &mut to, + ConsentForwardingMode::Both, + None, + ); + + assert_eq!( + cookie_values(&to), + vec!["keep=1".to_owned()], + "should strip identity cookies from valid fields and drop the non-UTF-8 field" + ); + } + fn header_value<'a>(headers: &'a [(String, Vec)], name: &str) -> Option<&'a [u8]> { headers .iter() @@ -9904,7 +10006,7 @@ set = { networkId = 42 } ConsentForwardingMode::OpenrtbOnly, HeaderValue::from_bytes(b"session=fictional;\xffbroken") .expect("should build malformed cookie header"), - Some(b"session=fictional;\xffbroken".as_slice()), + None, ), ] { let stub = Arc::new(StubHttpClient::new()); From f9c5ac244bca334d3a4519b0de7a6c1cb63c0691 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:19:32 +0530 Subject: [PATCH 10/89] Strip ID cookies from publisher origin requests The origin keeps ts-ec and every unrelated cookie. ts-eids and __Host-ts-eids-s no longer reach it. An origin that read ts-eids must read x-ts-eids instead. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/publisher.rs | 64 ++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index e6761efdca..efa93a8fcc 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -58,8 +58,10 @@ use crate::cache_policy::{ }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; -use crate::cookies::handle_request_cookies; use crate::cookies::template_cache_policy::{TemplateCookieDecision, evaluate_cookie_policy}; +use crate::cookies::{ + TS_ORIGIN_STRIPPED_COOKIE_NAMES, handle_request_cookies, strip_cookie_headers, +}; use crate::creative_opportunities::{ AdStackGateInput, AssemblyMode, CreativeOpportunitiesConfig, RuntimeAdStackExpected, evaluate_ad_stack_gate, @@ -3136,6 +3138,9 @@ fn rewrite_origin_request( // separate change. Until then we strip it here so the edge signal never // reaches publisher backends. req.headers_mut().remove("fastly-ssl"); + // The publisher origin keeps `ts-ec` but never receives the ID cookies, + // which can push its `Cookie` header past common 8 KB limits (spec 6.2). + strip_cookie_headers(req.headers_mut(), TS_ORIGIN_STRIPPED_COOKIE_NAMES); *req.uri_mut() = target_uri; req.headers_mut().insert( header::HOST, @@ -7299,6 +7304,63 @@ mod tests { } } + #[test] + fn rewrite_origin_request_strips_id_cookies_and_keeps_ts_ec() { + let mut req = Request::builder() + .uri("https://www.example.com/article") + .header(header::COOKIE, "ts-ec=abc; ts-eids=W10; session=1") + .header( + header::COOKIE, + "__Host-ts-eids-s=eyJ2IjoxfQ; ts-ec-pull-complete=v1", + ) + .body(EdgeBody::empty()) + .expect("should build request"); + + rewrite_origin_request( + &mut req, + "https://origin.example.com/article" + .parse() + .expect("should parse uri"), + "origin.example.com", + ) + .expect("should rewrite request"); + + let cookies: Vec<_> = req.headers().get_all(header::COOKIE).iter().collect(); + assert_eq!(cookies.len(), 1, "should send one Cookie header"); + assert_eq!( + cookies[0], "ts-ec=abc; session=1; ts-ec-pull-complete=v1", + "should keep ts-ec, the pull marker and unrelated cookies, and drop the ID cookies" + ); + } + + #[test] + fn rewrite_origin_request_drops_cookie_header_when_only_id_cookies() { + let mut req = Request::builder() + .uri("https://www.example.com/") + .header(header::COOKIE, "ts-eids=W10; __Host-ts-eids-s=eyJ2IjoxfQ") + .body(EdgeBody::empty()) + .expect("should build request"); + + rewrite_origin_request( + &mut req, + "https://origin.example.com/" + .parse() + .expect("should parse uri"), + "origin.example.com", + ) + .expect("should rewrite request"); + + assert!( + req.headers().get(header::COOKIE).is_none(), + "should drop the empty Cookie header" + ); + assert_eq!( + req.headers()[header::HOST], + "origin.example.com", + "should still set Host" + ); + } + #[test] fn template_eligibility_implies_origin_shareability() { for bits in 0u16..256 { From e85225d65a7c1b534ae99501ad94a4474973a64a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:21:18 +0530 Subject: [PATCH 11/89] Refuse the identity mirror cookie as a template cache key Signed-off-by: dhruv8sh --- CHANGELOG.md | 1 + .../src/cookies/template_cache_policy.rs | 24 +++++++++++++++++-- docs/guide/edge-cookies.md | 9 +++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f691c6a005..9b9026e031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking:** The publisher origin no longer receives the `ts-eids` cookie, and Prebid Server no longer receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, and the `__Host-ts-eids-s` mirror that this release's shim writes). The origin still receives `ts-ec`. Origin code that read `ts-eids` should read the `x-ts-eids` response header instead. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. - **Breaking:** `ts prebid bundle` is now `ts prebid client`, alongside the new `ts prebid server` namespace. Update scripts and runbooks to use `ts prebid client` with the same arguments. The old `bundle` spelling is no longer accepted and has no compatibility alias. - The S2S `/_ts/api/v1/batch-sync` endpoint now validates the full batch and calls the CAS-protected update path once per distinct normalized EC ID. The last valid UID wins within a group, and infrastructure failures reject the failing and each unprocessed group, so accepted and `kv_unavailable` input indexes may interleave. diff --git a/crates/trusted-server-core/src/cookies/template_cache_policy.rs b/crates/trusted-server-core/src/cookies/template_cache_policy.rs index 61c3b9139c..7c42a8d6f1 100644 --- a/crates/trusted-server-core/src/cookies/template_cache_policy.rs +++ b/crates/trusted-server-core/src/cookies/template_cache_policy.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS}; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_S}; use crate::platform::TemplateCookieValue; /// Whether the request can share a template and its explicit cookie dimensions. @@ -32,7 +32,13 @@ pub(crate) fn validate_cookie_names( return Err(format!("{field} contains invalid cookie name `{name}`")); } if field == "template_cache_key_cookies" - && [COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_SHAREDID].contains(&name.as_str()) + && [ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_S, + COOKIE_SHAREDID, + ] + .contains(&name.as_str()) { return Err(format!( "{field} must not key on Trusted Server identity cookie `{name}`" @@ -246,6 +252,20 @@ mod tests { } } + #[test] + fn template_cache_key_rejects_the_mirror_cookie() { + let error = validate_cookie_names(&names(&["__Host-ts-eids-s"]), &[]) + .expect_err("should reject the mirror cookie as a cache key"); + assert!( + error.contains("must not key on Trusted Server identity cookie"), + "should explain the rejection, got {error}" + ); + assert!( + validate_cookie_names(&names(&["session"]), &[]).is_ok(), + "should still accept an unrelated cookie" + ); + } + #[test] fn template_cookie_policy_allows_only_independent_unlisted_duplicates() { for fields in [ diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 979e268b70..57121a0b36 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -304,6 +304,15 @@ sets `Path=/`, `Secure`, `HttpOnly`, `SameSite=Lax`, and a `Max-Age`. `Domain` is computed as `.{publisher.domain}`. The separate `cookie_domain` setting applies only to non-EC cookies. +## Cookies Forwarded to the Origin + +`ts-ec` still reaches the publisher origin. `ts-eids` and the +`__Host-ts-eids-s` mirror cookie do not: Trusted Server strips both from +the request before it forwards it. Origin code that read `ts-eids` +should read the `x-ts-eids` response header instead. Prebid Server never +receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, or +`__Host-ts-eids-s`). + ## Operational Notes - Rotate the secret periodically. Rotation produces a new 64-hex base From 2bdf774aaca23756507a75fe615c73664eef3b0d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:27:58 +0530 Subject: [PATCH 12/89] Add the identity module with provider and source ID newtypes Signed-off-by: dhruv8sh --- .../src/ec/identity/descriptor.rs | 41 ++++ .../src/ec/identity/ids.rs | 223 ++++++++++++++++++ .../src/ec/identity/mod.rs | 15 ++ .../src/ec/identity/resolution.rs | 85 +++++++ crates/trusted-server-core/src/ec/mod.rs | 1 + .../src/migration_guards.rs | 10 + 6 files changed, 375 insertions(+) create mode 100644 crates/trusted-server-core/src/ec/identity/descriptor.rs create mode 100644 crates/trusted-server-core/src/ec/identity/ids.rs create mode 100644 crates/trusted-server-core/src/ec/identity/mod.rs create mode 100644 crates/trusted-server-core/src/ec/identity/resolution.rs diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs new file mode 100644 index 0000000000..9b97041e42 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -0,0 +1,41 @@ +//! What a provider is, may write, needs and must respect. + +/// An input a provider can ask for. +/// +/// Consent strings, the provider's stored IDs, the publisher domain, the +/// time and the declared endpoint are always available. Private state is +/// available to providers declared `with_private_state()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display)] +#[non_exhaustive] +pub enum Input { + /// The platform's client address. + #[display("client_ip")] + ClientIp, + /// The request's `User-Agent`. + #[display("user_agent")] + UserAgent, + /// The top-level page URL without query string or fragment. + #[display("page_url")] + PageUrl, + /// Country and subdivision only. + #[display("region")] + Region, + /// A SHA-256 of a normalized email address. Never available in v1. + #[display("hashed_email")] + HashedEmail, +} + +/// Something a provider read without declaring it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display)] +#[non_exhaustive] +pub enum Access { + /// An undeclared [`Input`]. + #[display("{_0}")] + Input(Input), + /// The endpoint, when the descriptor declares no `resolves`. + #[display("endpoint")] + Endpoint, + /// Private state, without `with_private_state()`. + #[display("private_state")] + PrivateState, +} diff --git a/crates/trusted-server-core/src/ec/identity/ids.rs b/crates/trusted-server-core/src/ec/identity/ids.rs new file mode 100644 index 0000000000..21c2516cea --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/ids.rs @@ -0,0 +1,223 @@ +//! Identifiers shared by the identity provider API. + +use core::ops::Add; +use core::str::FromStr; +use std::time::Duration; + +use error_stack::Report; +use serde::{Deserialize, Serialize}; + +use super::resolution::IdentityProviderError; +use crate::ec::partner::normalize_partner_source_domain; + +/// An identity provider's ID, such as `id5` or `pull-ssp-example-com`. +/// +/// Matches `^[a-z][a-z0-9_-]{0,62}$`, so it can be a bare TOML key and can +/// equal an integration ID that uses `_`. Separate from +/// [`crate::auction::ProviderId`]. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, Serialize, Deserialize, +)] +#[serde(try_from = "String", into = "String")] +pub struct IdentityProviderId(String); + +impl IdentityProviderId { + /// The ID as written in config. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for IdentityProviderId { + type Err = Report; + + fn from_str(value: &str) -> Result { + let mut bytes = value.bytes(); + let valid_first = bytes.next().is_some_and(|byte| byte.is_ascii_lowercase()); + let valid_rest = bytes.all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' || byte == b'-' + }); + if valid_first && valid_rest && value.len() <= 63 { + Ok(Self(value.to_owned())) + } else { + Err(IdentityProviderError::config(format!( + "identity provider id `{value}` must match ^[a-z][a-z0-9_-]{{0,62}}$" + ))) + } + } +} + +impl TryFrom for IdentityProviderId { + type Error = Report; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(id: IdentityProviderId) -> Self { + id.0 + } +} + +/// An EID `source` domain, normalized. Also the key of the KV `ids` map. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display, Serialize, Deserialize, +)] +#[serde(try_from = "String", into = "String")] +pub struct SourceDomain(String); + +impl SourceDomain { + /// The normalized domain. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Whether a vendor's spelling of a source names this one, after + /// normalization. + #[must_use] + pub fn matches(&self, name: &str) -> bool { + normalize_partner_source_domain(name).is_ok_and(|normalized| normalized == self.0) + } +} + +impl FromStr for SourceDomain { + type Err = Report; + + fn from_str(value: &str) -> Result { + normalize_partner_source_domain(value) + .map(Self) + .map_err(IdentityProviderError::config) + } +} + +impl TryFrom for SourceDomain { + type Error = Report; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(source: SourceDomain) -> Self { + source.0 + } +} + +/// Unix seconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct UnixTime(u64); + +impl UnixTime { + /// From Unix seconds. + #[must_use] + pub fn from_secs(secs: u64) -> Self { + Self(secs) + } + + /// From Unix milliseconds, for vendors such as Lockr and `UID2`. Floors to + /// whole seconds. + #[must_use] + pub fn from_millis(millis: u64) -> Self { + Self(millis / 1000) + } + + /// As Unix seconds. + #[must_use] + pub fn as_secs(self) -> u64 { + self.0 + } +} + +impl Add for UnixTime { + type Output = UnixTime; + + fn add(self, rhs: Duration) -> UnixTime { + UnixTime(self.0.saturating_add(rhs.as_secs())) + } +} +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provider_id_accepts_the_documented_grammar() { + for id in ["id5", "pull-ssp-example-com", "lockr_aim", "a"] { + assert!( + id.parse::().is_ok(), + "should accept `{id}`" + ); + } + let longest = format!("a{}", "b".repeat(62)); + longest + .parse::() + .expect("should accept 63 characters"); + } + + #[test] + fn provider_id_rejects_everything_else() { + let too_long = format!("a{}", "b".repeat(63)); + for id in ["", "1id", "Id5", "id 5", "id.5", too_long.as_str()] { + let error = id + .parse::() + .expect_err("should reject the id"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::Config { .. } + ), + "should be a Config error for `{id}`" + ); + } + } + + #[test] + fn source_domain_normalizes_and_matches() { + let source: SourceDomain = "ID5-Sync.com.".parse().expect("should parse"); + assert_eq!( + source.as_str(), + "id5-sync.com", + "should lower-case and drop the trailing dot" + ); + assert!( + source.matches("id5-sync.COM"), + "should match another spelling" + ); + assert!( + !source.matches("example.com"), + "should not match another domain" + ); + assert!( + !source.matches("https://id5-sync.com"), + "should not match a URL" + ); + } + + #[test] + fn source_domain_round_trips_through_serde() { + let source: SourceDomain = "example.com".parse().expect("should parse"); + let json = serde_json::to_string(&source).expect("should serialize"); + assert_eq!(json, "\"example.com\"", "should serialize as a bare string"); + let back: SourceDomain = serde_json::from_str(&json).expect("should deserialize"); + assert_eq!(back, source, "should round-trip"); + serde_json::from_str::("\"not a domain\"") + .expect_err("should reject an invalid domain on deserialize"); + } + + #[test] + fn unix_time_converts_millis_and_adds_durations() { + assert_eq!( + UnixTime::from_millis(1_760_000_000_999).as_secs(), + 1_760_000_000, + "should floor millis" + ); + let later = UnixTime::from_secs(100) + Duration::from_secs(50); + assert_eq!(later, UnixTime::from_secs(150), "should add seconds"); + let saturated = UnixTime::from_secs(u64::MAX) + Duration::from_secs(1); + assert_eq!(saturated, UnixTime::from_secs(u64::MAX), "should saturate"); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs new file mode 100644 index 0000000000..e4e64f8451 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -0,0 +1,15 @@ +//! Server-side identity providers. +//! +//! An `IdentityProvider` turns request signals into `OpenRTB` EIDs. +//! Providers only build requests and parse buffered responses. The +//! coordinator owns eligibility, consent, transport limits and KV +//! write-back, so every page load makes at most one browser-driven KV +//! write. See `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. + +pub mod descriptor; +pub mod ids; +pub mod resolution; + +pub use descriptor::{Access, Input}; +pub use ids::{IdentityProviderId, SourceDomain, UnixTime}; +pub use resolution::IdentityProviderError; diff --git a/crates/trusted-server-core/src/ec/identity/resolution.rs b/crates/trusted-server-core/src/ec/identity/resolution.rs new file mode 100644 index 0000000000..b49509fabd --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/resolution.rs @@ -0,0 +1,85 @@ +//! Provider results and errors. + +use error_stack::Report; +use http::StatusCode; + +use super::descriptor::{Access, Input}; + +/// Marker message of [`IdentityProviderError::not_implemented`]. The harness +/// matches on it to tell a missing override from a real failure. +pub(crate) const NOT_IMPLEMENTED_MARKER: &str = "capability declared but not implemented: "; + +/// A provider failure. Built through the constructors below. +#[derive(Debug, derive_more::Display)] +#[non_exhaustive] +pub enum IdentityProviderError { + /// Configuration or provider code makes the call impossible. + #[display("identity provider configuration error: {message}")] + Config { + /// What is wrong. Never contains a secret, UID or URL query. + message: String, + }, + /// The provider read something its descriptor did not declare. + #[display("`{access}` was read but not declared")] + UndeclaredInput { + /// What was read. + access: Access, + }, + /// A declared input is not available for this request. + #[display("input `{input}` is unavailable for this request")] + InputUnavailable { + /// The missing input. + input: Input, + }, + /// The provider refused the request. + #[display("provider rejected the request with status {status}")] + Rejected { + /// The response status. + status: StatusCode, + }, + /// The provider's response could not be parsed. + #[display("provider response could not be parsed")] + Malformed, + /// The stored private state does not decode as the provider's type. + #[display("stored provider state could not be decoded")] + InvalidState, + /// Set by the coordinator only. Providers never see transport failures. + #[display("provider request failed in transport")] + Transport, +} + +impl core::error::Error for IdentityProviderError {} + +impl IdentityProviderError { + /// A [`Self::Config`] error. + pub fn config(message: impl Into) -> Report { + Report::new(Self::Config { + message: message.into(), + }) + } + + /// A [`Self::Rejected`] error. + pub fn rejected(status: StatusCode) -> Report { + Report::new(Self::Rejected { status }) + } + + /// A [`Self::Malformed`] error. + pub fn malformed() -> Report { + Report::new(Self::Malformed) + } + + /// The default body of a capability method: a `Config` error with a + /// fixed marker message, so the harness can tell a missing override from + /// a real failure. + pub fn not_implemented(method: &'static str) -> Report { + Self::config(format!("{NOT_IMPLEMENTED_MARKER}{method}")) + } + + /// Whether this is the [`Self::not_implemented`] marker. + // Temporary: first used by the provider test harness in a later task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn is_not_implemented(&self) -> bool { + matches!(self, Self::Config { message } if message.starts_with(NOT_IMPLEMENTED_MARKER)) + } +} diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index cc6705eaf8..ce599c0c36 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -41,6 +41,7 @@ pub mod eids; pub mod finalize; pub mod generation; pub mod identify; +pub mod identity; pub mod kv; pub mod kv_backend; pub mod kv_types; diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 683fbbc138..d22b47c1b8 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -83,6 +83,16 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/finalize.rs", include_str!("ec/finalize.rs")), ("ec/generation.rs", include_str!("ec/generation.rs")), ("ec/identify.rs", include_str!("ec/identify.rs")), + ( + "ec/identity/descriptor.rs", + include_str!("ec/identity/descriptor.rs"), + ), + ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), + ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), + ( + "ec/identity/resolution.rs", + include_str!("ec/identity/resolution.rs"), + ), ("ec/kv.rs", include_str!("ec/kv.rs")), ("ec/kv_types.rs", include_str!("ec/kv_types.rs")), ("ec/mod.rs", include_str!("ec/mod.rs")), From e661413cd29f7aef36694f369113ee63a902e351 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:37:29 +0530 Subject: [PATCH 13/89] Move the KV schema to version 2 with one record shape Every record stores its source's entry set in eids, with optional writer, revision and timing fields, and the row gains per-provider state. Version 1 rows read as one-entry records and are written as version 2. A build without this change cannot read a version 2 row, so rolling back past it is unsupported. A record that breaks a limit is dropped on read and logged with its source only, so one bad record never makes a row unreadable; only the write path rejects it. Blank UIDs are now rejected on validation, so a version 1 record with a blank uid is dropped on read. KvPartnerId and KvEntry derive Default, and literals move to constructors. The admin lookup redacts provider state. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 11 +- crates/trusted-server-core/src/ec/admin.rs | 122 ++- crates/trusted-server-core/src/ec/eids.rs | 14 +- crates/trusted-server-core/src/ec/finalize.rs | 16 +- crates/trusted-server-core/src/ec/identify.rs | 10 +- crates/trusted-server-core/src/ec/kv.rs | 197 ++-- crates/trusted-server-core/src/ec/kv_types.rs | 844 +++++++++++++++++- .../trusted-server-core/src/ec/pull_sync.rs | 26 +- .../src/ec/pull_sync_marker.rs | 8 +- crates/trusted-server-core/src/publisher.rs | 9 +- 10 files changed, 1085 insertions(+), 172 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index c21caa1b0a..15eab65773 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -711,7 +711,7 @@ mod tests { .expect("should read store") .expect("row should exist"); assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), Some("shared-cookie-id"), "the sharedId update must still be ingested from the shared snapshot" ); @@ -752,7 +752,12 @@ mod tests { stored .ids .iter() - .map(|(source, id)| (source.clone(), id.uid.clone())) + .map(|(source, id)| { + ( + source.clone(), + id.first_uid().unwrap_or_default().to_owned(), + ) + }) .collect() } @@ -981,7 +986,7 @@ mod tests { .expect("should read store") .expect("row should exist"); assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), + stored.ids.get("id5-sync.com").and_then(|id| id.first_uid()), Some("ID5_from_body"), "the body's EID must be ingested into KV without a ts-eids cookie" ); diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 18b5563307..04dc13e507 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -368,6 +368,19 @@ fn requested_ec_id(req: &Request) -> Result.st` with a marker. Provider state is never +/// returned by the admin lookup (spec 6.1, 9.3). +fn redact_provider_state(entry: &mut JsonValue) { + let Some(states) = entry.get_mut("ps").and_then(JsonValue::as_object_mut) else { + return; + }; + for state in states.values_mut() { + if let Some(st) = state.get_mut("st") { + *st = JsonValue::String("[redacted]".to_owned()); + } + } +} + /// Builds the success payload from a raw KV lookup. /// /// Parse failures are reported in the payload rather than propagated, so @@ -393,14 +406,26 @@ fn build_lookup_response( match serde_json::from_slice::(&lookup.body) { Ok(mut entry_json) => { + redact_provider_state(&mut entry_json); add_iso_timestamp_companions(&mut entry_json); payload.entry = Some(entry_json); match serde_json::from_slice::(&lookup.body) { - Ok(entry) => { + Ok(mut entry) => { payload.tombstone = Some(!entry.consent.ok); - match entry.validate() { - Ok(()) => payload.auction = Some(build_auction_view(registry, &entry)), + // Upgrade first so the auction view matches what the read + // path serves, with invalid records dropped. + let upgraded = entry + .upgrade_for_read() + .and_then(|dropped| entry.validate().map(|()| dropped)); + match upgraded { + Ok(dropped) => { + payload.auction = Some(build_auction_view(registry, &entry)); + if !dropped.is_empty() { + payload.entry_error = + Some(format!("dropped on read: {}", dropped.join("; "))); + } + } Err(message) => { payload.entry_error = Some(format!( "entry failed validation (auction reads fail closed \ @@ -487,7 +512,7 @@ fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEid let mut skipped = Vec::new(); for (source_domain, partner_uid) in &entry.ids { - let reason = if partner_uid.uid.is_empty() { + let reason = if partner_uid.first_uid().is_none_or(str::is_empty) { "empty_uid" } else { match registry.get(source_domain) { @@ -684,7 +709,7 @@ mod tests { use super::*; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvPartnerId, KvProviderState}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -778,15 +803,11 @@ mod tests { let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); entry.ids.insert( "disabled.example".to_owned(), - KvPartnerId { - uid: "uid-disabled".to_owned(), - }, + KvPartnerId::new("uid-disabled"), ); entry.ids.insert( "unknown.example".to_owned(), - KvPartnerId { - uid: "uid-unknown".to_owned(), - }, + KvPartnerId::new("uid-unknown"), ); entry } @@ -971,6 +992,83 @@ mod tests { } } + fn admin_lookup_body_for_test(entry: &KvEntry) -> serde_json::Value { + let ec_id = test_ec_id(); + let kv = kv_with_entry(&ec_id, entry); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"); + response_json(response) + } + + #[test] + fn admin_lookup_redacts_provider_state() { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1_760_000_000); + entry.ps.insert( + "id5".to_owned(), + KvProviderState { + n: Some(5), + st: Some(serde_json::json!({ "signature": "secret-state" })), + ..KvProviderState::default() + }, + ); + let body = admin_lookup_body_for_test(&entry); + + assert_eq!( + body["entry"]["ps"]["id5"]["st"], + serde_json::json!("[redacted]"), + "should redact the state" + ); + assert_eq!( + body["entry"]["ps"]["id5"]["n"], + serde_json::json!(5), + "should keep the rest of ps" + ); + assert_eq!( + body["entry"]["ids"]["example.com"]["eids"][0]["uids"][0]["id"], + serde_json::json!("example-uid"), + "should keep the ids" + ); + assert!( + !body.to_string().contains("secret-state"), + "should never return the provider state" + ); + } + + #[test] + fn admin_lookup_reports_records_dropped_on_read() { + let ec_id = test_ec_id(); + let body = serde_json::json!({ + "v": 2, + "created": 1_741_824_000_u64, + "consent": { "ok": true, "updated": 1_741_824_000_u64 }, + "geo": { "country": "US" }, + "ids": { + "bidstream.example": { "eids": [{ "uids": [{ "id": "uid-live" }] }] }, + "blank.example": { "eids": [{ "uids": [{ "id": " " }] }] } + } + }) + .to_string(); + let kv = kv_with_raw_body(&ec_id, &body); + let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); + + let json = response_json( + handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + .expect("should handle lookup"), + ); + + let error = json["entry_error"].as_str().expect("should report a drop"); + assert!( + error.starts_with("dropped on read: ") && error.contains("blank.example"), + "should name the dropped source, got {error}" + ); + assert_eq!( + json["auction"]["eids"][0]["source"], "bidstream.example", + "should build the auction view from the surviving record" + ); + } + #[test] fn returns_entry_with_auction_view() { let ec_id = test_ec_id(); @@ -995,7 +1093,7 @@ mod tests { assert_eq!(json["store"], "test-store"); assert_eq!(json["tombstone"], false); assert_eq!( - json["entry"]["ids"]["bidstream.example"]["uid"], "uid-live", + json["entry"]["ids"]["bidstream.example"]["eids"][0]["uids"][0]["id"], "uid-live", "should echo the stored entry verbatim" ); assert_eq!( diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 0493f5dec1..837ad581c8 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -37,9 +37,9 @@ pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec Vec>(), vec![("ids.example.com", "cookie-uid")], "a new EC with no sync source should ingest the cookie" @@ -1390,7 +1390,7 @@ mod tests { .expect("should read store") .expect("row should remain"); assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), Some("generated-cookie-id") ); } @@ -1536,7 +1536,7 @@ mod tests { .expect("should read store") .expect("row should remain"); assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), + stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), Some("shared-cookie-id"), "a stale endpoint miss must not suppress EID persistence" ); @@ -1614,12 +1614,12 @@ mod tests { .expect("should read store") .expect("row should exist after ingestion"); assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), + stored.ids.get("id5-sync.com").and_then(|id| id.first_uid()), Some("ID5_from_body"), "the request body's EID should win over the cookie's stale value" ); assert_eq!( - stored.ids.get("liveramp.com").map(|id| id.uid.as_str()), + stored.ids.get("liveramp.com").and_then(|id| id.first_uid()), Some("LR_from_body"), "a partner the cookie trimmed must still be ingested from the body" ); @@ -2376,9 +2376,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "partner-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("partner-uid"), ); ec_context.set_kv_snapshot(EcKvSnapshot::Present { ec_id, diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index c22a592461..e320b817f3 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -17,6 +17,7 @@ use crate::settings::Settings; use super::EcContext; use super::kv::KvIdentityGraph; +use super::kv_types::KvPartnerId; use super::log_id; use super::registry::PartnerRegistry; @@ -96,10 +97,13 @@ pub fn handle_identify( log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { // Extract only this partner's UID. - if let Some(partner_uid) = entry.ids.get(&partner.source_domain) - && !partner_uid.uid.is_empty() + if let Some(partner_uid) = entry + .ids + .get(&partner.source_domain) + .and_then(KvPartnerId::first_uid) + .filter(|partner_uid| !partner_uid.is_empty()) { - uid = Some(partner_uid.uid.clone()); + uid = Some(partner_uid.to_owned()); } // Evaluate cluster size lazily for identify responses. Existing diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0eb83d3fa..de7df2c73f 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -144,7 +144,7 @@ fn apply_cookie_partner_id_updates( let mut deferred = false; for (partner_id, uid) in latest_updates { match entry.ids.get(partner_id) { - Some(existing) if existing.uid == uid => continue, + Some(existing) if existing.first_uid() == Some(uid) => continue, // Browser EID cookies carry no value-owned sequence or timestamp. // A different value therefore has unknown freshness and must not // replace the stored value. @@ -157,9 +157,7 @@ fn apply_cookie_partner_id_updates( entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::new(uid), ); changed = true; } @@ -183,7 +181,7 @@ fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> boo entry .ids .get(partner_id) - .is_some_and(|existing| existing.uid == uid) + .is_some_and(|existing| existing.first_uid() == Some(uid)) }) } @@ -198,16 +196,14 @@ pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerId if entry .ids .get(partner_id) - .is_some_and(|existing| existing.uid == uid) + .is_some_and(|existing| existing.first_uid() == Some(uid)) { continue; } entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::new(uid), ); changed = true; } @@ -357,12 +353,28 @@ impl KvIdentityGraph { ec_id: &str, body_bytes: &[u8], ) -> Result> { - let entry: KvEntry = + let mut entry: KvEntry = serde_json::from_slice(body_bytes).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: format!("Failed to deserialize entry for key '{}'", log_id(ec_id)), })?; + let dropped = entry.upgrade_for_read().map_err(|message| { + Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Loaded invalid entry for key '{}': {message}", + log_id(ec_id) + ), + }) + })?; + for message in dropped { + log::warn!( + "Dropping an invalid record on read for key '{}': {message}", + log_id(ec_id) + ); + } + entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -913,16 +925,14 @@ impl KvIdentityGraph { if entry .ids .get(partner_id) - .is_some_and(|existing| existing.uid == uid) + .is_some_and(|existing| existing.first_uid() == Some(uid)) { return Ok(()); } entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::new(uid), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -985,16 +995,14 @@ impl KvIdentityGraph { if entry .ids .get(partner_id) - .is_some_and(|existing| existing.uid == uid) + .is_some_and(|existing| existing.first_uid() == Some(uid)) { return Ok(UpsertResult::Unchanged); } entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::new(uid), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -1733,22 +1741,74 @@ mod tests { } #[test] - fn deserialize_entry_rejects_invalid_legacy_values() { + fn deserialize_entry_drops_an_invalid_record() { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::new( + "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + ), + ); + entry.ids.insert( + "ssp_y".to_owned(), + crate::ec::kv_types::KvPartnerId::new("valid-uid"), ); let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); - let err = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) - .expect_err("should reject invalid legacy entry values"); - let err_text = format!("{err}"); + let read = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) + .expect("should read a row whose one record is invalid"); + assert!( - err_text.contains("Loaded invalid entry"), - "should report validation failure for loaded entries" + !read.ids.contains_key("ssp_x"), + "should drop the invalid record" + ); + assert_eq!( + read.ids["ssp_y"].first_uid(), + Some("valid-uid"), + "should keep the valid record" + ); + + let malformed = serde_json::json!({ + "v": 2, + "created": 1000, + "consent": { "ok": "yes", "updated": 1000 }, + "geo": { "country": "US" } + }); + let err = KvIdentityGraph::deserialize_entry( + "test-store", + "ec-id", + &serde_json::to_vec(&malformed).expect("should serialize"), + ) + .expect_err("should still fail a row-level fault as a whole"); + assert!( + format!("{err}").contains("Failed to deserialize entry"), + "should report the row-level fault" + ); + } + + #[test] + fn deserialize_entry_reads_a_version_1_body() { + let body = serde_json::to_vec(&serde_json::json!({ + "v": 1, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { "id5-sync.com": { "uid": "ID5*example" } } + })) + .expect("should serialize a row from main"); + + let entry = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) + .expect("should read a version 1 row"); + + assert_eq!( + entry.v, + crate::ec::kv_types::SCHEMA_VERSION, + "should upgrade the version" + ); + assert_eq!( + entry.ids["id5-sync.com"], + crate::ec::kv_types::KvPartnerId::new("ID5*example"), + "should read the uid as one entry" ); } @@ -1772,9 +1832,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::new( + "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), + ), ); let err = KvIdentityGraph::serialize_entry(&entry, "test-store") @@ -1796,9 +1856,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "concurrent.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("concurrent-uid"), ); entry } @@ -2131,16 +2189,14 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-1".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("uid-1"), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; let changed = apply_partner_id_updates(&mut entry, &updates); assert!(!changed, "should not change when UID already matches"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-1"); + assert_eq!(entry.ids["ssp_x"].first_uid(), Some("uid-1")); } #[test] @@ -2151,7 +2207,7 @@ mod tests { let changed = apply_partner_id_updates(&mut entry, &updates); assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-1"); + assert_eq!(entry.ids["ssp_x"].first_uid(), Some("uid-1")); } #[test] @@ -2159,16 +2215,14 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "old-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("old-uid"), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; let changed = apply_partner_id_updates(&mut entry, &updates); assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "new-uid"); + assert_eq!(entry.ids["ssp_x"].first_uid(), Some("new-uid")); } #[test] @@ -2182,8 +2236,8 @@ mod tests { let changed = apply_partner_id_updates(&mut entry, &updates); assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-x"); - assert_eq!(entry.ids["ssp_y"].uid, "uid-y"); + assert_eq!(entry.ids["ssp_x"].first_uid(), Some("uid-x")); + assert_eq!(entry.ids["ssp_y"].first_uid(), Some("uid-y")); } #[test] @@ -2191,9 +2245,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "original".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("original"), ); let updates = vec![ PartnerIdUpdate::new("ssp_x", "intermediate"), @@ -2206,7 +2258,7 @@ mod tests { !changed, "should not write when the final duplicate value matches existing state" ); - assert_eq!(entry.ids["ssp_x"].uid, "original"); + assert_eq!(entry.ids["ssp_x"].first_uid(), Some("original")); } #[test] @@ -2215,9 +2267,7 @@ mod tests { let mut concurrent = live_entry(); concurrent.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "desired-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("desired-uid"), ); let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -2241,7 +2291,7 @@ mod tests { snapshot .entry_for(&ec_id) .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), + .and_then(|id| id.first_uid()), Some("desired-uid"), "the authoritative follow-up snapshot should replace stale request state" ); @@ -2333,9 +2383,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "latest-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("latest-uid"), ); let updates = [ PartnerIdUpdate::new("ssp_x", "older-uid"), @@ -2354,9 +2402,7 @@ mod tests { let mut concurrent = live_entry(); concurrent.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("concurrent-uid"), ); let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -2390,7 +2436,7 @@ mod tests { .get(&ec_id) .expect("should read concurrent row") .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "concurrent-uid"); + assert_eq!(stored.ids["ssp_x"].first_uid(), Some("concurrent-uid")); } #[test] @@ -2439,9 +2485,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("protected-uid"), ); graph.create(&ec_id, &entry).expect("should seed entry"); @@ -2456,7 +2500,7 @@ mod tests { .get(&ec_id) .expect("should read protected row") .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); + assert_eq!(stored.ids["ssp_x"].first_uid(), Some("protected-uid")); } #[test] @@ -2466,9 +2510,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("protected-uid"), ); graph.create(&ec_id, &entry).expect("should seed entry"); @@ -2486,8 +2528,8 @@ mod tests { .get(&ec_id) .expect("should read updated row") .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - assert_eq!(stored.ids["ssp_y"].uid, "new-uid"); + assert_eq!(stored.ids["ssp_x"].first_uid(), Some("protected-uid")); + assert_eq!(stored.ids["ssp_y"].first_uid(), Some("new-uid")); } #[test] @@ -3139,7 +3181,7 @@ mod tests { .expect("should read persisted entry") .expect("should retain entry after conflict"); assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + entry.ids.get("ssp_x").and_then(|id| id.first_uid()), Some("uid-1"), "should persist requested UID after retry" ); @@ -3176,7 +3218,7 @@ mod tests { .entry_for(&ec_id) .expect("should retain persisted entry"); assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + entry.ids.get("ssp_x").and_then(|id| id.first_uid()), Some("uid-1") ); assert_eq!( @@ -3607,7 +3649,7 @@ mod tests { outcome .entry_for(&ec_id) .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), + .and_then(|id| id.first_uid()), Some("uid-1") ); assert_eq!( @@ -3695,7 +3737,7 @@ mod tests { outcome .entry_for(&ec_id) .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), + .and_then(|id| id.first_uid()), Some("uid-1"), "the fifth CAS attempt must still succeed after a refresh and four conflicts" ); @@ -3715,7 +3757,7 @@ mod tests { .entry_for(&ec_id) .expect("should persist re-merged entry"); assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + entry.ids.get("ssp_x").and_then(|id| id.first_uid()), Some("uid-1"), "conflict must re-merge our update onto the concurrently revived row" ); @@ -3742,9 +3784,10 @@ mod tests { ); assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x").map(|id| id.uid.clone())), + outcome.entry_for(&ec_id).and_then(|entry| entry + .ids + .get("ssp_x") + .and_then(|id| id.first_uid().map(str::to_owned))), Some("uid-1".to_owned()), "a stale miss must be revalidated before the updates are dropped" ); @@ -3753,7 +3796,7 @@ mod tests { .expect("should read store") .expect("row should remain"); assert_eq!( - stored.ids.get("ssp_x").map(|id| id.uid.as_str()), + stored.ids.get("ssp_x").and_then(|id| id.first_uid()), Some("uid-1"), "the revalidated update must reach the store" ); diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 589f280c11..c75c51848c 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -4,7 +4,8 @@ //! EC identity graph. Each EC ID (`{64hex}.{6alnum}`) maps to a [`KvEntry`] //! containing consent state, geo location, and accumulated partner IDs. //! -//! The schema is versioned (`v: 1`) to allow future migrations. +//! The schema is versioned (`v: 2`). Version 1 rows, which hold one `uid` per +//! source, still read and are written back as version 2. use std::collections::{BTreeMap, BTreeSet}; @@ -14,10 +15,11 @@ use crate::consent::ConsentContext; use crate::geo::GeoInfo; /// Current schema version for KV entries. -pub const SCHEMA_VERSION: u8 = 1; +pub const SCHEMA_VERSION: u8 = 2; -// Unsupported schema versions fail closed on read. Future schema bumps must -// add an explicit lazy migration or backfill path before changing this value. +// Versions 1 and 2 read (see [`KvEntry::upgrade_for_read`]). Any other +// version fails closed on read. Future schema bumps must add an explicit lazy +// migration or backfill path before changing this value. /// Maximum number of publisher-domain entries accepted in /// [`KvPubProperties::seen_domains`]. @@ -34,11 +36,37 @@ pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// validation. pub const MAX_UID_LENGTH: usize = 512; +/// Maximum serialized bytes of a stored UID-level or EID-level `ext`. +pub const MAX_PARTNER_EXT_BYTES: usize = 2048; +/// Maximum bytes of a stored `inserter` or `matcher`. +pub const MAX_EID_FIELD_BYTES: usize = 255; +/// Maximum number of EID entries stored for one source. +pub const MAX_ENTRIES_PER_SOURCE: usize = 8; +/// Maximum number of UIDs stored for one source, across its entries. +pub const MAX_UIDS_PER_SOURCE: usize = 32; +/// Maximum bytes of one source's stored entry set, as measured by +/// [`entry_set_bytes`]. +pub const MAX_ENTRY_SET_BYTES: usize = 4096; +/// Maximum serialized bytes of one provider's private state (`ps.st`). +pub const MAX_PROVIDER_STATE_BYTES: usize = 1024; +/// Maximum number of providers with a `ps` entry in one row. +pub const MAX_PROVIDER_STATES: usize = 32; +/// Maximum number of UID hashes kept in each of a provider's `ps.iv` and +/// `ps.ov` lists, enough for four whole 32-UID sets. +pub const MAX_INVALIDATED_UIDS: usize = 128; +/// Length of a record revision (`rv`) in lowercase hex digits. +pub const RECORD_REVISION_HEX_LEN: usize = 16; +/// Maximum length of an opt-out fingerprint (`ps.o.fp`) in lowercase hex +/// digits. +const OPT_OUT_FINGERPRINT_MAX_HEX_LEN: usize = 16; + /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). -/// **KV value:** JSON-serialized `KvEntry` (max ~5KB). -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// **KV value:** JSON-serialized `KvEntry`. Typically a few hundred bytes to +/// 2KB; with every cap of spec 6.1 reached (32 sources of 4KB entry sets, plus +/// provider state) it can approach 150KB, which is far below the store limit. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct KvEntry { /// Schema version — always [`SCHEMA_VERSION`]. pub v: u8, @@ -62,10 +90,14 @@ pub struct KvEntry { /// Populated by pixel sync, batch sync, and pull sync operations. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub ids: BTreeMap, + /// Per-provider scheduling, opt-out, removal and private state, keyed by + /// identity provider ID. Never sent to bidders. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub ps: BTreeMap, } /// Consent state within a KV entry. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct KvConsent { /// Raw TCF v2 consent string. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -80,7 +112,7 @@ pub struct KvConsent { } /// Geo location within a KV entry. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct KvGeo { /// ISO 3166-1 alpha-2 country code (e.g. `"US"`). pub country: String, @@ -97,11 +129,225 @@ pub struct KvGeo { pub dma: Option, } -/// A partner user ID within a KV entry. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// A source's record within a KV entry (spec 6.1). +/// +/// Every record stores its source's entry set in `eids`. A version 1 record, +/// which holds one `uid`, reads as one entry holding that UID. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(from = "KvPartnerIdWire")] pub struct KvPartnerId { - /// The partner's user identifier. - pub uid: String, + /// Writer: an identity provider ID. Absent for browser ingest and + /// version 1 records. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub w: Option, + /// Record revision: 16 lowercase hex digits, random on every write. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rv: Option, + /// Unix seconds when the ID was acquired or last refreshed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at: Option, + /// Unix seconds at or after which the ID must not be sent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exp: Option, + /// Unix seconds after which the ID is due for refresh. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ra: Option, + /// Unix seconds after which the ID can no longer be refreshed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ru: Option, + /// The source's entry set, in order. Never empty in a valid record. + pub eids: Vec, +} + +/// The stored shape of [`KvPartnerId`], which also reads version 1's `uid`. +#[derive(Deserialize)] +struct KvPartnerIdWire { + #[serde(default)] + uid: Option, + #[serde(default)] + w: Option, + #[serde(default)] + rv: Option, + #[serde(default)] + at: Option, + #[serde(default)] + exp: Option, + #[serde(default)] + ra: Option, + #[serde(default)] + ru: Option, + #[serde(default)] + eids: Vec, +} + +impl From for KvPartnerId { + fn from(wire: KvPartnerIdWire) -> Self { + let eids = match (wire.eids.is_empty(), wire.uid) { + (true, Some(uid)) => vec![KvEid::one_uid(uid)], + (_, _) => wire.eids, + }; + Self { + w: wire.w, + rv: wire.rv, + at: wire.at, + exp: wire.exp, + ra: wire.ra, + ru: wire.ru, + eids, + } + } +} + +impl KvPartnerId { + /// A record holding one entry with one UID, and no metadata. + #[must_use] + pub fn new(uid: impl Into) -> Self { + Self { + eids: vec![KvEid::one_uid(uid.into())], + ..Self::default() + } + } + + /// A browser-ingested record: no writer, with its acquisition time. + #[must_use] + pub fn browser(uid: impl Into, acquired_at: Option) -> Self { + Self { + at: acquired_at, + ..Self::new(uid) + } + } + + /// The first UID of the first entry, which `/identify` returns and + /// today's one-UID readers use. + #[must_use] + pub fn first_uid(&self) -> Option<&str> { + self.eids + .first() + .and_then(|entry| entry.uids.first()) + .map(|uid| uid.id.as_str()) + } +} + +/// One EID entry of a record (spec 6.1). +/// +/// The `OpenRTB` EID object without `source`, which the `ids` key holds. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvEid { + /// The entry's UIDs, in order. Never empty in a valid record. + pub uids: Vec, + /// `eid.inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// `eid.matcher`. Absent means `source`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// `eid.mm`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// EID-level `ext`, at most [`MAX_PARTNER_EXT_BYTES`] serialized bytes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +impl KvEid { + /// One entry holding one UID and nothing else: the shape browser + /// ingest, generic pull and batch sync write. + fn one_uid(id: String) -> Self { + Self { + uids: vec![KvUid { + id, + atype: None, + ext: None, + }], + ..Self::default() + } + } +} + +/// One UID of a stored entry. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + /// The user identifier. + pub id: String, + /// `uid.atype`. Absent: use the partner's `openrtb_atype`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub atype: Option, + /// UID-level `ext`, at most [`MAX_PARTNER_EXT_BYTES`] serialized bytes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +/// A provider's sticky opt-out (spec 8). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct KvOptOut { + /// Unix seconds when the opt-out was reported. + pub at: u64, + /// The consent bits the provider's policy read then, as a lowercase hex + /// bitmask of at most 16 digits. Plain, not keyed (spec 8). + pub fp: String, +} + +/// Per-provider state in a KV row (`ps.`). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct KvProviderState { + /// Unix seconds before which the provider must not be called. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub n: Option, + /// Consecutive failures, for backoff. + #[serde(default, skip_serializing_if = "is_zero")] + pub f: u32, + /// Sticky opt-out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub o: Option, + /// When the provider's opt-out was last cleared. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oc: Option, + /// Unkeyed hashes of UIDs the owner invalidated, oldest first, at most + /// [`MAX_INVALIDATED_UIDS`]. Never emptied (spec 5.4). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub iv: Vec, + /// Unkeyed hashes of UIDs the provider's opt-out removed, oldest first, + /// at most [`MAX_INVALIDATED_UIDS`]. Emptied when the opt-out clears. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ov: Vec, + /// The provider's private state as embedded JSON. KV only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub st: Option, +} + +fn is_zero(value: &u32) -> bool { + *value == 0 +} + +/// Bytes of a stored entry set as compact JSON: the array of entries without +/// `source`, as `eids` stores it. Both `validate` and the coordinator measure +/// [`MAX_ENTRY_SET_BYTES`] with this function. +pub(crate) fn entry_set_bytes(set: &[KvEid]) -> usize { + serde_json::to_vec(set).map_or(usize::MAX, |bytes| bytes.len()) +} + +/// Whether `value` is non-empty lowercase hex of at most `max_len` digits. +/// +/// Shared by the `rv` and `ps.o.fp` checks. +pub(crate) fn is_lowercase_hex(value: &str, max_len: usize) -> bool { + !value.is_empty() + && value.len() <= max_len + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Whether an `inserter` or `matcher` is stored-valid: not blank and at most +/// [`MAX_EID_FIELD_BYTES`] bytes. +pub(crate) fn is_valid_eid_field(value: &str) -> bool { + !value.trim().is_empty() && value.len() <= MAX_EID_FIELD_BYTES +} + +/// Whether a UID-level or EID-level `ext` is stored-valid: a JSON object of at +/// most [`MAX_PARTNER_EXT_BYTES`] bytes as compact JSON. +pub(crate) fn is_valid_partner_ext(ext: &serde_json::Value) -> bool { + ext.is_object() + && serde_json::to_vec(ext).is_ok_and(|bytes| bytes.len() <= MAX_PARTNER_EXT_BYTES) } /// Publisher property metadata captured when an EC entry is created. @@ -293,6 +539,7 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + ps: BTreeMap::new(), } } @@ -303,12 +550,7 @@ impl KvEntry { #[must_use] pub fn minimal(partner_id: &str, uid: &str, now: u64) -> Self { let mut ids = BTreeMap::new(); - ids.insert( - partner_id.to_owned(), - KvPartnerId { - uid: uid.to_owned(), - }, - ); + ids.insert(partner_id.to_owned(), KvPartnerId::new(uid)); Self { v: SCHEMA_VERSION, created: now, @@ -328,6 +570,7 @@ impl KvEntry { device: None, network: None, ids, + ps: BTreeMap::new(), } } @@ -360,6 +603,7 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + ps: BTreeMap::new(), } } @@ -380,11 +624,35 @@ impl KvEntry { )); } - for (partner_id, partner_uid) in &self.ids { - if partner_uid.uid.len() > MAX_UID_LENGTH { + for (partner_id, record) in &self.ids { + validate_record(partner_id, record)?; + } + + if self.ps.len() > MAX_PROVIDER_STATES { + return Err(format!( + "ps exceeds MAX_PROVIDER_STATES ({})", + self.ps.len() + )); + } + for (provider, state) in &self.ps { + for (name, hashes) in [("iv", &state.iv), ("ov", &state.ov)] { + if hashes.len() > MAX_INVALIDATED_UIDS { + return Err(format!( + "ps '{provider}' {name} exceeds {MAX_INVALIDATED_UIDS} hashes" + )); + } + } + if let Some(opt_out) = &state.o + && !is_lowercase_hex(&opt_out.fp, OPT_OUT_FINGERPRINT_MAX_HEX_LEN) + { + return Err(format!("ps '{provider}' o.fp is not a hex bitmask")); + } + if let Some(st) = &state.st + && serde_json::to_vec(st) + .map_or(true, |bytes| bytes.len() > MAX_PROVIDER_STATE_BYTES) + { return Err(format!( - "partner ID '{partner_id}' exceeds MAX_UID_LENGTH ({})", - partner_uid.uid.len() + "ps '{provider}' st exceeds {MAX_PROVIDER_STATE_BYTES} bytes" )); } } @@ -417,6 +685,117 @@ impl KvEntry { Ok(()) } + + /// Brings a row read from KV to the current schema (spec 6.1). + /// + /// Accepts schema versions 1 and 2 and sets the version to 2. Drops each + /// record that breaks a rule of spec 4.5 or 6.1, and returns one message + /// per dropped record, naming the source and the rule, never a UID. + /// Called on read only, before [`Self::validate`], so one bad record + /// never makes the row unreadable. + /// + /// # Errors + /// + /// Returns an error naming the version when the row's version is + /// neither 1 nor 2. + pub(crate) fn upgrade_for_read(&mut self) -> Result, String> { + if !(1..=SCHEMA_VERSION).contains(&self.v) { + return Err(format!( + "unsupported KV entry schema version {} (expected 1 or {})", + self.v, SCHEMA_VERSION + )); + } + self.v = SCHEMA_VERSION; + let mut messages = Vec::new(); + self.ids.retain( + |partner_id, record| match validate_record(partner_id, record) { + Ok(()) => true, + Err(message) => { + messages.push(message); + false + } + }, + ); + Ok(messages) + } +} + +/// Checks one record against the limits of spec 4.5 and 6.1. +/// +/// Messages name the partner ID and the rule and never a UID, so they are safe +/// to log. +fn validate_record(partner_id: &str, record: &KvPartnerId) -> Result<(), String> { + if record.eids.is_empty() { + return Err(format!("partner ID '{partner_id}' has an empty eids")); + } + if let Some(rv) = &record.rv + && (rv.len() != RECORD_REVISION_HEX_LEN || !is_lowercase_hex(rv, RECORD_REVISION_HEX_LEN)) + { + return Err(format!( + "partner ID '{partner_id}' rv is not {RECORD_REVISION_HEX_LEN} hex digits" + )); + } + if record.eids.len() > MAX_ENTRIES_PER_SOURCE { + return Err(format!( + "partner ID '{partner_id}' has more than {MAX_ENTRIES_PER_SOURCE} entries" + )); + } + let mut uid_count = 0; + for entry in &record.eids { + if entry.uids.is_empty() { + return Err(format!( + "partner ID '{partner_id}' has an entry without a uid" + )); + } + for (name, value) in [("inserter", &entry.inserter), ("matcher", &entry.matcher)] { + if value + .as_deref() + .is_some_and(|value| !is_valid_eid_field(value)) + { + return Err(format!( + "partner ID '{partner_id}' {name} is blank or over {MAX_EID_FIELD_BYTES} bytes" + )); + } + } + if entry.mm.is_some_and(|mm| mm < 0) { + return Err(format!("partner ID '{partner_id}' mm is negative")); + } + validate_ext(partner_id, entry.ext.as_ref())?; + for uid in &entry.uids { + if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + return Err(format!( + "partner ID '{partner_id}' has a blank uid or one over MAX_UID_LENGTH ({MAX_UID_LENGTH})" + )); + } + if uid.atype.is_some_and(|atype| atype < 0) { + return Err(format!("partner ID '{partner_id}' atype is negative")); + } + validate_ext(partner_id, uid.ext.as_ref())?; + } + uid_count += entry.uids.len(); + } + if uid_count > MAX_UIDS_PER_SOURCE { + return Err(format!( + "partner ID '{partner_id}' has more than {MAX_UIDS_PER_SOURCE} uids" + )); + } + if entry_set_bytes(&record.eids) > MAX_ENTRY_SET_BYTES { + return Err(format!( + "partner ID '{partner_id}' entry set exceeds {MAX_ENTRY_SET_BYTES} bytes" + )); + } + Ok(()) +} + +/// Checks an optional UID-level or EID-level `ext` with +/// [`is_valid_partner_ext`]. +fn validate_ext(partner_id: &str, ext: Option<&serde_json::Value>) -> Result<(), String> { + if ext.is_some_and(|ext| !is_valid_partner_ext(ext)) { + return Err(format!( + "partner ID '{partner_id}' ext is not an object of at most {MAX_PARTNER_EXT_BYTES} bytes" + )); + } + Ok(()) } impl KvMetadata { @@ -463,6 +842,7 @@ impl KvGeo { #[cfg(test)] mod tests { use super::*; + use serde_json::json; fn sample_consent_context() -> ConsentContext { ConsentContext { @@ -490,12 +870,9 @@ mod tests { let geo = sample_geo_info(); let consent = sample_consent_context(); let mut entry = KvEntry::new(&consent, Some(&geo), 1_741_824_000, "example.com"); - entry.ids.insert( - "liveramp".to_owned(), - KvPartnerId { - uid: "LR_xyz".to_owned(), - }, - ); + entry + .ids + .insert("liveramp".to_owned(), KvPartnerId::new("LR_xyz")); let json = serde_json::to_string(&entry).expect("should serialize KvEntry"); let deserialized: KvEntry = @@ -514,7 +891,7 @@ mod tests { assert_eq!(deserialized.geo.asn, Some(7922)); assert_eq!(deserialized.geo.dma, Some(807)); assert_eq!( - deserialized.ids.get("liveramp").map(|p| p.uid.as_str()), + deserialized.ids.get("liveramp").and_then(|p| p.first_uid()), Some("LR_xyz"), ); assert!( @@ -560,7 +937,7 @@ mod tests { entry .ids .get("liveramp") - .map(|partner| partner.uid.as_str()), + .and_then(|partner| partner.first_uid()), Some("abc") ); assert!( @@ -737,9 +1114,7 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - KvPartnerId { - uid: "x".repeat(MAX_UID_LENGTH + 1), - }, + KvPartnerId::new("x".repeat(MAX_UID_LENGTH + 1)), ); let err = entry @@ -853,7 +1228,7 @@ mod tests { ); assert_eq!(entry.ids.len(), 1); let partner = entry.ids.get("ssp_x").expect("should have ssp_x entry"); - assert_eq!(partner.uid, "abc123"); + assert_eq!(partner.first_uid(), Some("abc123")); } #[test] @@ -1206,4 +1581,405 @@ mod tests { "known_browser should default to None" ); } + + fn kv_uid(id: &str, atype: Option) -> KvUid { + KvUid { + id: id.to_owned(), + atype, + ext: None, + } + } + + fn kv_eid(uids: Vec) -> KvEid { + KvEid { + uids, + inserter: None, + matcher: None, + mm: None, + ext: None, + } + } + + /// The set of spec 6.1's `example-ids.example.com` record. + fn example_full_set() -> Vec { + vec![ + KvEid { + uids: vec![ + kv_uid("example-a", Some(1)), + KvUid { + id: "example-b".to_owned(), + atype: Some(3), + ext: Some(json!({ "stype": "ppuid" })), + }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + KvEid { + uids: vec![kv_uid("example-c", Some(1))], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ] + } + + fn with_set(set: Vec) -> KvEntry { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ids.get_mut("example.com").expect("should exist").eids = set; + entry + } + + #[test] + fn golden_version_1_row_reads_as_one_entry_records_and_writes_version_2() { + let main_row = json!({ + "v": 1, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { + "id5-sync.com": { "uid": "ID5*example" }, + "sharedid.org": { "uid": "example-shared-id" } + } + }); + + let mut entry: KvEntry = + serde_json::from_value(main_row).expect("should read a row from main"); + let messages = entry.upgrade_for_read().expect("should accept version 1"); + entry.validate().expect("should be valid once upgraded"); + + assert!( + messages.is_empty(), + "should drop nothing from a valid version 1 row" + ); + assert_eq!( + entry.ids["id5-sync.com"], + KvPartnerId::new("ID5*example"), + "should read uid as one entry" + ); + assert_eq!( + serde_json::to_value(&entry).expect("should write"), + json!({ + "v": 2, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { + "id5-sync.com": { "eids": [{ "uids": [{ "id": "ID5*example" }] }] }, + "sharedid.org": { "eids": [{ "uids": [{ "id": "example-shared-id" }] }] } + } + }), + "should write version 2, with no uid key and no ps" + ); + } + + #[test] + fn version_2_row_and_provider_state_round_trip() { + let stored = json!({ + "v": 2, + "created": 1_760_000_000, + "consent": { "ok": true, "updated": 1_760_000_000 }, + "geo": { "country": "US" }, + "ids": { + "id5-sync.com": { + "w": "id5", + "rv": "9c1d04b7e6a83f2a", + "at": 1_760_000_000, + "exp": 1_767_776_000, + "ra": 1_760_086_400, + "ru": 1_770_000_000, + "eids": [{ "uids": [{ "id": "ID5*example", "atype": 1, "ext": { "linkType": 2 } }] }] + }, + "example-ids.example.com": { + "w": "exampleid", + "rv": "4be71f0c2d9a6e35", + "at": 1_760_000_000, + "eids": [ + { + "uids": [ + { "id": "example-a", "atype": 1 }, + { "id": "example-b", "atype": 3, "ext": { "stype": "ppuid" } } + ], + "inserter": "example.com", + "mm": 3 + }, + { + "uids": [{ "id": "example-c", "atype": 1 }], + "matcher": "partner.example.com" + } + ] + } + }, + "ps": { + "id5": { + "n": 1_760_003_600, + "f": 1, + "o": { "at": 1_759_000_000, "fp": "1a" }, + "oc": 1_759_500_000, + "iv": ["aaaaaaaaaaaaaaaa"], + "ov": ["bbbbbbbbbbbbbbbb"], + "st": { "signature": "example-opaque-state" } + } + } + }); + + let entry: KvEntry = serde_json::from_value(stored.clone()).expect("should read"); + entry.validate().expect("should be valid"); + assert_eq!( + serde_json::to_value(&entry).expect("should write"), + stored, + "should round-trip exactly" + ); + assert_eq!( + entry.ids["example-ids.example.com"].eids, + example_full_set(), + "should read the set whole and in order" + ); + } + + #[test] + fn a_record_reads_its_eids_and_its_version_1_uid_only_without_them() { + let both: KvPartnerId = serde_json::from_value(json!({ + "uid": "stale-copy", + "eids": [{ "uids": [{ "id": "example-a" }] }] + })) + .expect("should read a record with both keys"); + assert_eq!( + both.first_uid(), + Some("example-a"), + "should take eids over a uid copy" + ); + + let neither: KvPartnerId = + serde_json::from_value(json!({ "w": "exampleid" })).expect("should read"); + assert!( + neither.eids.is_empty(), + "should read no entry from a record with neither key" + ); + assert_eq!(neither.first_uid(), None, "should have no first UID"); + with_set(Vec::new()) + .validate() + .expect_err("should reject a record with an empty eids on write"); + } + + #[test] + fn upgrade_for_read_drops_bad_records_and_names_the_source_only() { + let mut set = example_full_set(); + set[1].matcher = Some(" ".to_owned()); + let mut entry = with_set(set); + entry.ids.insert( + "sharedid.org".to_owned(), + KvPartnerId::new("example-shared-id"), + ); + entry.v = 1; + + let messages = entry.upgrade_for_read().expect("should accept the row"); + + assert_eq!(entry.v, SCHEMA_VERSION, "should upgrade the version"); + assert_eq!(messages.len(), 1, "should report one dropped record"); + assert!( + messages[0].contains("example.com"), + "should name the source" + ); + assert!( + !messages[0].contains("example-a"), + "should never name a UID" + ); + assert!( + !entry.ids.contains_key("example.com"), + "should drop the bad record" + ); + assert_eq!( + entry.ids["sharedid.org"], + KvPartnerId::new("example-shared-id"), + "should leave a valid record alone" + ); + entry.validate().expect("should validate once upgraded"); + + let mut future = KvEntry::minimal("example.com", "example-uid", 1); + future.v = SCHEMA_VERSION + 1; + future + .upgrade_for_read() + .expect_err("should reject a version it does not know"); + } + + #[test] + fn entry_set_bytes_measures_the_stored_array() { + let set = example_full_set(); + let bytes = serde_json::to_vec(&set).expect("should serialize the set"); + + assert_eq!( + entry_set_bytes(&set), + bytes.len(), + "should measure the stored array" + ); + assert!( + !String::from_utf8(bytes) + .expect("should be UTF-8") + .contains("\"source\""), + "should measure entries without source" + ); + } + + #[test] + fn validate_rejects_entry_set_limits() { + let rejects = |entry: &KvEntry, field: &str| { + let error = entry + .validate() + .expect_err(&format!("should reject {field}")); + assert!(error.contains(field), "should name {field}, got {error}"); + }; + + rejects(&with_set(Vec::new()), "eids"); + + let mut set = example_full_set(); + set[0].ext = Some(json!({ "pad": "x".repeat(MAX_PARTNER_EXT_BYTES) })); + rejects(&with_set(set), "ext"); + + let mut set = example_full_set(); + set[0].uids[0].ext = Some(json!("not an object")); + rejects(&with_set(set), "ext"); + + let mut set = example_full_set(); + set[0].inserter = Some(" ".to_owned()); + rejects(&with_set(set), "inserter"); + + let mut set = example_full_set(); + set[1].matcher = Some("m".repeat(MAX_EID_FIELD_BYTES + 1)); + rejects(&with_set(set), "matcher"); + + let mut set = example_full_set(); + set[0].mm = Some(-1); + rejects(&with_set(set), "mm"); + + let mut set = example_full_set(); + set[0].uids[1].atype = Some(-1); + rejects(&with_set(set), "atype"); + + let mut set = example_full_set(); + set[1].uids[0].id = " ".to_owned(); + rejects(&with_set(set), "uid"); + + let mut set = example_full_set(); + set[1].uids.clear(); + rejects(&with_set(set), "uid"); + + let nine_entries = (0..=MAX_ENTRIES_PER_SOURCE) + .map(|index| kv_eid(vec![kv_uid(&format!("example-{index}"), None)])) + .collect(); + rejects(&with_set(nine_entries), "entries"); + + let thirty_three_uids = (0..=MAX_UIDS_PER_SOURCE) + .map(|index| kv_uid(&format!("example-{index}"), None)) + .collect(); + rejects(&with_set(vec![kv_eid(thirty_three_uids)]), "uids"); + + let oversized = (0..4) + .map(|index| KvEid { + ext: Some(json!({ "pad": "x".repeat(1_100) })), + ..kv_eid(vec![kv_uid(&format!("example-{index}"), None)]) + }) + .collect(); + rejects(&with_set(oversized), "entry set"); + + let mut old = with_set(example_full_set()); + old.v = 1; + rejects(&old, "version"); + + with_set(example_full_set()) + .validate() + .expect("should accept the control set"); + } + + #[test] + fn validate_rejects_oversized_state_and_too_many_providers() { + let with_state = |state: KvProviderState| { + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ps.insert("id5".to_owned(), state); + entry + }; + let hashes = |count: usize| vec!["a".repeat(16); count]; + + let error = with_state(KvProviderState { + st: Some(json!("x".repeat(MAX_PROVIDER_STATE_BYTES))), + ..KvProviderState::default() + }) + .validate() + .expect_err("should reject st"); + assert!(error.contains("st"), "should name st"); + + for (name, state) in [ + ( + "iv", + KvProviderState { + iv: hashes(MAX_INVALIDATED_UIDS + 1), + ..KvProviderState::default() + }, + ), + ( + "ov", + KvProviderState { + ov: hashes(MAX_INVALIDATED_UIDS + 1), + ..KvProviderState::default() + }, + ), + ( + "fp", + KvProviderState { + o: Some(KvOptOut { + at: 1, + fp: "XYZ".to_owned(), + }), + ..KvProviderState::default() + }, + ), + ] { + let error = with_state(state) + .validate() + .expect_err(&format!("should reject {name}")); + assert!(error.contains(name), "should name {name}, got {error}"); + } + with_state(KvProviderState { + iv: hashes(MAX_INVALIDATED_UIDS), + ov: hashes(MAX_INVALIDATED_UIDS), + o: Some(KvOptOut { + at: 1, + fp: "1a".to_owned(), + }), + ..KvProviderState::default() + }) + .validate() + .expect("should accept 128 hashes in each list and a hex fingerprint"); + + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + for index in 0..=MAX_PROVIDER_STATES { + entry + .ps + .insert(format!("p{index}"), KvProviderState::default()); + } + assert!( + entry + .validate() + .expect_err("should reject ps size") + .contains("ps"), + "should name ps" + ); + + let mut entry = KvEntry::minimal("example.com", "example-uid", 1); + entry.ids.get_mut("example.com").expect("should exist").rv = Some("not-hex".to_owned()); + assert!( + entry + .validate() + .expect_err("should reject rv") + .contains("rv"), + "should name rv" + ); + + KvEntry::minimal("example.com", "example-uid", 1) + .validate() + .expect("should accept the control entry"); + } } diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 191e6c6d24..e01d662da7 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -652,9 +652,7 @@ mod tests { if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("uid"), ); } ec_context.set_kv_snapshot(snapshot); @@ -702,9 +700,7 @@ mod tests { entry.ids.insert( "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-b".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("uid-b"), ); assert!( @@ -982,12 +978,18 @@ mod tests { .expect("should read store") .expect("entry should exist"); assert_eq!( - entry.ids.get("alpha.example.com").map(|id| id.uid.as_str()), + entry + .ids + .get("alpha.example.com") + .and_then(|id| id.first_uid()), Some("synced-uid"), "first partner UID should persist" ); assert_eq!( - entry.ids.get("beta.example.com").map(|id| id.uid.as_str()), + entry + .ids + .get("beta.example.com") + .and_then(|id| id.first_uid()), Some("synced-uid"), "second partner UID should persist" ); @@ -1045,7 +1047,7 @@ mod tests { entry .ids .get("alpha.example.com") - .map(|partner_id| partner_id.uid.as_str()), + .and_then(|partner_id| partner_id.first_uid()), Some("existing-alpha"), "existing UID should remain unchanged" ); @@ -1053,7 +1055,7 @@ mod tests { entry .ids .get("beta.example.com") - .map(|partner_id| partner_id.uid.as_str()), + .and_then(|partner_id| partner_id.first_uid()), Some("new-beta"), "missing UID should persist" ); @@ -1213,9 +1215,7 @@ mod tests { entry.consent.ok = true; entry.ids.insert( "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "already-known".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("already-known"), ); graph .create(&ec_id, &entry) diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index f2a2fe6013..8a3510b284 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -375,9 +375,7 @@ mod tests { for domain in domains { entry.ids.insert( (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: format!("uid-{domain}"), - }, + crate::ec::kv_types::KvPartnerId::new(format!("uid-{domain}")), ); } EcKvSnapshot::Present { @@ -510,9 +508,7 @@ mod tests { let mut tombstone = KvEntry::tombstone(1_000); tombstone.ids.insert( "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "stale".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::new("stale"), ); let snapshots = [ live_snapshot(EC_ID, &[]), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index efa93a8fcc..86f6bdbbab 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -7979,12 +7979,9 @@ mod tests { ]) .expect("should build partner registry"); let mut entry = KvEntry::minimal("ids.example.com", "kv-stale", 1000); - entry.ids.insert( - "other.example.com".to_owned(), - KvPartnerId { - uid: "kv-other".to_owned(), - }, - ); + entry + .ids + .insert("other.example.com".to_owned(), KvPartnerId::new("kv-other")); let snapshot = EcKvSnapshot::Present { ec_id: ec_id.clone(), entry: Box::new(entry), From 38fbf52360b70e800c528e6bb04a51b4946ec74e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:39:48 +0530 Subject: [PATCH 14/89] Restate spec 6.1 size figures in the KvEntry doc comment The estimate cites spec 6.1 and uses its numbers. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/kv_types.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index c75c51848c..a576f05830 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -63,9 +63,13 @@ const OPT_OUT_FINGERPRINT_MAX_HEX_LEN: usize = 16; /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). -/// **KV value:** JSON-serialized `KvEntry`. Typically a few hundred bytes to -/// 2KB; with every cap of spec 6.1 reached (32 sources of 4KB entry sets, plus -/// provider state) it can approach 150KB, which is far below the store limit. +/// **KV value:** JSON-serialized `KvEntry`. This is a documented estimate, not +/// an enforced limit (spec 6.1): a one-UID record is about 120 bytes, and a +/// record at every cap is about 4.2 KB. With every cap reached, eight owned +/// sources come to about 34 KB, plus 1 KB for each provider with state, and +/// `ps.iv` and `ps.ov` add up to 2.4 KB each per provider. A row with two full +/// sets and two stateful providers at every cap is about 15 KB, and typical +/// sets are far smaller. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct KvEntry { /// Schema version — always [`SCHEMA_VERSION`]. From e15ce97080467cb0b01122869051f740ad224674 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:43:27 +0530 Subject: [PATCH 15/89] Record when browser ingest acquired each ID Ingest still stores one UID per registered source, by today's rule: the first valid UID of a repeated source's last entry, with the sharedId cookie applied last. A golden test pins that extra UIDs and EID-level fields from the browser are not stored. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 18 +++- crates/trusted-server-core/src/ec/kv.rs | 53 ++++++++--- .../trusted-server-core/src/ec/prebid_eids.rs | 89 ++++++++++++++++--- 3 files changed, 135 insertions(+), 25 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 0ffc6d047b..ec19ac35ea 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -24,7 +24,7 @@ use super::kv_types::KvEntry; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, EidSyncSource, current_timestamp, log_id}; +use super::{EcKvSnapshot, EidSyncSource, checked_current_timestamp, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -84,7 +84,13 @@ pub fn ec_finalize_response( let collected = source .map(|_| { let (cookie_input, client_input) = browser_eid_input(ec_context, eids_cookie); - collect_eid_updates(cookie_input, sharedid_cookie, client_input, registry) + collect_eid_updates( + cookie_input, + sharedid_cookie, + client_input, + registry, + checked_current_timestamp(), + ) }) .unwrap_or_default(); let had_eid_updates = !collected.is_empty(); @@ -138,7 +144,13 @@ pub fn ec_finalize_response( let (cookie_input, client_input) = browser_eid_input(ec_context, eids_cookie); let updates = gate_eid_updates_by_consent( - collect_eid_updates(cookie_input, sharedid_cookie, client_input, registry), + collect_eid_updates( + cookie_input, + sharedid_cookie, + client_input, + registry, + checked_current_timestamp(), + ), ec_context.consent(), ); sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index de7df2c73f..926797892c 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -77,16 +77,26 @@ pub(crate) struct PartnerIdUpdate { pub(crate) partner_id: String, /// Partner-scoped user ID value. pub(crate) uid: String, + /// Unix seconds this request saw the ID. Stored as the record's `at`. + pub(crate) acquired_at: Option, } impl PartnerIdUpdate { - /// Creates a partner UID update. + /// Creates a partner UID update with no acquisition time. pub(crate) fn new(partner_id: impl Into, uid: impl Into) -> Self { Self { partner_id: partner_id.into(), uid: uid.into(), + acquired_at: None, } } + + /// Records when this request saw the ID. + #[must_use] + pub(crate) fn with_acquired_at(mut self, at: u64) -> Self { + self.acquired_at = Some(at); + self + } } /// Terminal result of one browser EID-cookie persistence attempt. @@ -137,12 +147,13 @@ fn apply_cookie_partner_id_updates( ) -> CookieUpdateApplication { let mut latest_updates = BTreeMap::new(); for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); + latest_updates.insert(update.partner_id.as_str(), update); } let mut changed = false; let mut deferred = false; - for (partner_id, uid) in latest_updates { + for (partner_id, update) in latest_updates { + let uid = update.uid.as_str(); match entry.ids.get(partner_id) { Some(existing) if existing.first_uid() == Some(uid) => continue, // Browser EID cookies carry no value-owned sequence or timestamp. @@ -157,7 +168,7 @@ fn apply_cookie_partner_id_updates( entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId::new(uid), + super::kv_types::KvPartnerId::browser(uid, update.acquired_at), ); changed = true; } @@ -174,25 +185,26 @@ fn apply_cookie_partner_id_updates( fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); + latest_updates.insert(update.partner_id.as_str(), update); } - latest_updates.into_iter().all(|(partner_id, uid)| { + latest_updates.into_iter().all(|(partner_id, update)| { entry .ids .get(partner_id) - .is_some_and(|existing| existing.first_uid() == Some(uid)) + .is_some_and(|existing| existing.first_uid() == Some(update.uid.as_str())) }) } pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); + latest_updates.insert(update.partner_id.as_str(), update); } let mut changed = false; - for (partner_id, uid) in latest_updates { + for (partner_id, update) in latest_updates { + let uid = update.uid.as_str(); if entry .ids .get(partner_id) @@ -203,7 +215,7 @@ pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerId entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId::new(uid), + super::kv_types::KvPartnerId::browser(uid, update.acquired_at), ); changed = true; } @@ -2199,6 +2211,27 @@ mod tests { assert_eq!(entry.ids["ssp_x"].first_uid(), Some("uid-1")); } + #[test] + fn browser_ingest_records_acquisition_time() { + let mut entry = KvEntry::minimal("sharedid.org", "existing", 1); + let updates = + [PartnerIdUpdate::new("example.com", "new-uid").with_acquired_at(1_760_000_000)]; + + let changed = apply_partner_id_updates(&mut entry, &updates); + + assert!(changed, "should report a change"); + assert_eq!( + entry.ids["example.com"], + crate::ec::kv_types::KvPartnerId::browser("new-uid", Some(1_760_000_000)), + "should store the UID with at and no writer" + ); + assert_eq!( + entry.ids["sharedid.org"], + crate::ec::kv_types::KvPartnerId::new("existing"), + "should leave other sources alone" + ); + } + #[test] fn apply_partner_id_updates_inserts_new_partner_uid() { let mut entry = live_entry(); diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 9d2b587806..54acc102fa 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -148,11 +148,14 @@ fn raw_diagnostic_sources(raw: &RawValue) -> Vec { /// route the cookie. The `sharedId` cookie is applied last, so it wins over a /// Prebid EID for the same partner. With repeated sources, the later entry's /// first valid UID is the candidate for that partner. +/// +/// When `now` is `Some`, each update carries it as its acquisition time. pub(crate) fn collect_eid_updates( eids_cookie: Option<&str>, sharedid_cookie: Option<&str>, body_eids: Option<&[Eid]>, registry: &PartnerRegistry, + now: Option, ) -> Vec { if registry.is_empty() { return Vec::new(); @@ -170,6 +173,12 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } + if let Some(now) = now { + updates = updates + .into_iter() + .map(|update| update.with_acquired_at(now)) + .collect(); + } dedupe_partner_updates(updates) } @@ -233,13 +242,10 @@ fn collect_prebid_eid_updates_from_eids( pub(crate) fn dedupe_partner_updates(updates: Vec) -> Vec { let mut latest = std::collections::BTreeMap::new(); for update in updates { - latest.insert(update.partner_id, update.uid); + latest.insert(update.partner_id.clone(), update); } - latest - .into_iter() - .map(|(partner_id, uid)| PartnerIdUpdate::new(partner_id, uid)) - .collect() + latest.into_values().collect() } fn first_valid_uid(uids: &[Uid]) -> Option<&Uid> { @@ -313,6 +319,8 @@ mod tests { use base64::engine::general_purpose::STANDARD as BASE64; use serde_json::json; + use crate::ec::kv::apply_partner_id_updates; + use crate::ec::kv_types::KvEntry; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -689,7 +697,13 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_abc", "atype": 1}]} ])); - let updates = collect_eid_updates(Some(&eids_cookie), Some(" shared-1 "), None, ®istry); + let updates = collect_eid_updates( + Some(&eids_cookie), + Some(" shared-1 "), + None, + ®istry, + None, + ); assert_eq!( updates.len(), @@ -729,7 +743,8 @@ mod tests { ), ]; - let updates = collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry); + let updates = + collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry, None); assert_eq!( updates, @@ -750,7 +765,8 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_abc", "atype": 1}]} ])); - let updates = collect_eid_updates(Some(&eids_cookie), Some("shared-1"), None, ®istry); + let updates = + collect_eid_updates(Some(&eids_cookie), Some("shared-1"), None, ®istry, None); assert!( updates.is_empty(), @@ -791,7 +807,13 @@ mod tests { {"source": "liveramp.com", "uids": [{"id": "LR_xyz", "atype": 3}]} ])); - let updates = collect_eid_updates(Some(&cookie), Some("shared-cookie-id"), None, ®istry); + let updates = collect_eid_updates( + Some(&cookie), + Some("shared-cookie-id"), + None, + ®istry, + None, + ); assert_eq!( updates, @@ -814,7 +836,7 @@ mod tests { } ])); - let updates = collect_eid_updates(Some(&cookie), None, None, ®istry); + let updates = collect_eid_updates(Some(&cookie), None, None, ®istry, None); assert_eq!( updates, @@ -830,7 +852,8 @@ mod tests { {"source": "sharedid.org", "uids": [{"id": "prebid-shared", "atype": 3}]} ])); - let updates = collect_eid_updates(Some(&cookie), Some("cookie-shared"), None, ®istry); + let updates = + collect_eid_updates(Some(&cookie), Some("cookie-shared"), None, ®istry, None); assert_eq!( updates, @@ -846,11 +869,53 @@ mod tests { {"source": "unknown.example", "uids": [{"id": "unknown", "atype": 1}]} ])); - let updates = collect_eid_updates(Some(&cookie), None, None, ®istry); + let updates = collect_eid_updates(Some(&cookie), None, None, ®istry, None); assert!( updates.is_empty(), "should ignore unmatched partner updates" ); } + + #[test] + fn golden_browser_ingest_stores_one_uid_per_registered_source() { + let registry = make_registry(vec![("id5", "id5-sync.com"), ("sharedid", "sharedid.org")]); + let eids_cookie = encode_json(&json!([ + { "source": "id5-sync.com", "uids": [{ "id": "ID5_earlier_entry", "atype": 1 }] }, + { + "source": "id5-sync.com", + "uids": [ + { "id": " ", "atype": 1 }, + { "id": "ID5_first", "atype": 1, "ext": { "linkType": 2 } }, + { "id": "ID5_second", "atype": 1 } + ], + "inserter": "example.com", + "matcher": "partner.example.com", + "mm": 3, + "ext": { "segments": ["example-segment"] } + }, + { "source": "sharedid.org", "uids": [{ "id": "prebid-shared-id", "atype": 1 }] }, + { "source": "unregistered.example.com", "uids": [{ "id": "other", "atype": 1 }] } + ])); + let updates = collect_eid_updates( + Some(&eids_cookie), + Some(" cookie-shared-id "), + None, + ®istry, + Some(1_760_000_000), + ); + let mut entry = KvEntry::minimal("example.com", "existing", 1); + apply_partner_id_updates(&mut entry, &updates); + let stored = serde_json::to_value(&entry.ids).expect("should serialize ids"); + + assert_eq!( + stored, + json!({ + "example.com": { "eids": [{ "uids": [{ "id": "existing" }] }] }, + "id5-sync.com": { "at": 1_760_000_000, "eids": [{ "uids": [{ "id": "ID5_first" }] }] }, + "sharedid.org": { "at": 1_760_000_000, "eids": [{ "uids": [{ "id": "cookie-shared-id" }] }] } + }), + "should store one UID per registered source by today's rule, with no EID-level field, atype or ext" + ); + } } From 27a2c915720dd75ce20f4e5d2c23c37905a050e9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:48:23 +0530 Subject: [PATCH 16/89] Add the identity provider descriptor and its builder Signed-off-by: dhruv8sh --- .../src/ec/identity/descriptor.rs | 989 ++++++++++++++++++ .../src/ec/identity/mod.rs | 5 +- 2 files changed, 993 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index 9b97041e42..d3ed46634a 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -1,5 +1,697 @@ //! What a provider is, may write, needs and must respect. +use std::collections::{BTreeMap, BTreeSet}; + +use error_stack::Report; +use http::Method; +use url::Url; + +use super::ids::{IdentityProviderId, SourceDomain}; +use super::resolution::IdentityProviderError; +use crate::ec::kv_types::{MAX_EID_FIELD_BYTES, is_valid_eid_field}; +use crate::ec::partner::normalize_partner_source_domain; + +/// What a provider is, may write, needs and must respect. +/// +/// Built from config with no secrets, so `ts config validate` and the running +/// coordinator read the same value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderDescriptor { + id: IdentityProviderId, + sources: Vec, + resolves: Option, + endpoint: Option, + captures: Option, + consent_policy: ConsentPolicy, + provenance: BTreeMap, + expiring: BTreeSet, +} + +impl ProviderDescriptor { + /// Starts a descriptor for `id`. + #[must_use] + pub fn builder(id: IdentityProviderId) -> ProviderDescriptorBuilder { + ProviderDescriptorBuilder { + id, + sources: Vec::new(), + resolves: None, + captures: None, + consent_policy: ConsentPolicy::strict(), + provenance: BTreeMap::new(), + expiring_all: false, + expiring: BTreeSet::new(), + } + } + + /// The provider's ID. + #[must_use] + pub fn id(&self) -> &IdentityProviderId { + &self.id + } + + /// The sources this provider may write, in declaration order. + #[must_use] + pub fn sources(&self) -> &[SourceDomain] { + &self.sources + } + + /// The first source added. Every built descriptor has one. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + &self.sources[0] + } + + /// Finds a declared source by a vendor's spelling of it, normalized. + #[must_use] + pub fn source_named(&self, name: &str) -> Option<&SourceDomain> { + self.sources.iter().find(|source| source.matches(name)) + } + + /// Active resolution, when declared. + #[must_use] + pub fn resolves(&self) -> Option<&Resolves> { + self.resolves.as_ref() + } + + /// The endpoint declared with [`Resolves::with_endpoint`], if any. + #[must_use] + pub fn endpoint(&self) -> Option<&Url> { + self.endpoint.as_ref() + } + + /// Passive capture, when declared. + #[must_use] + pub fn captures(&self) -> Option<&Captures> { + self.captures.as_ref() + } + + /// The consent this provider requires. + #[must_use] + pub fn consent_policy(&self) -> &ConsentPolicy { + &self.consent_policy + } + + /// Default EID-level fields for `source`, if declared. + #[must_use] + pub fn provenance(&self, source: &SourceDomain) -> Option<&EidProvenance> { + self.provenance.get(source) + } + + /// Whether IDs of `source` are declared as expiring. + #[must_use] + pub fn is_expiring(&self, source: &SourceDomain) -> bool { + self.expiring.contains(source) + } +} + +/// Builds a [`ProviderDescriptor`]. +#[derive(Debug, Clone)] +pub struct ProviderDescriptorBuilder { + id: IdentityProviderId, + sources: Vec, + resolves: Option, + captures: Option, + consent_policy: ConsentPolicy, + provenance: BTreeMap, + expiring_all: bool, + expiring: BTreeSet, +} + +impl ProviderDescriptorBuilder { + /// Adds a source this provider may write. Repeatable. A repeat is ignored. + #[must_use] + pub fn with_source(mut self, source: SourceDomain) -> Self { + if !self.sources.contains(&source) { + self.sources.push(source); + } + self + } + + /// Declares active resolution. + #[must_use] + pub fn with_resolves(mut self, resolves: Resolves) -> Self { + self.resolves = Some(resolves); + self + } + + /// Declares passive capture. + #[must_use] + pub fn with_captures(mut self, captures: Captures) -> Self { + self.captures = Some(captures); + self + } + + /// Replaces the default policy, [`ConsentPolicy::strict`]. + #[must_use] + pub fn with_consent_policy(mut self, policy: ConsentPolicy) -> Self { + self.consent_policy = policy; + self + } + + /// Marks the IDs of every declared source as expiring (spec 6.1). + #[must_use] + pub fn with_expiring_ids(mut self) -> Self { + self.expiring_all = true; + self + } + + /// Marks the IDs of one declared source as expiring. + #[must_use] + pub fn with_expiring_source(mut self, source: &SourceDomain) -> Self { + self.expiring.insert(source.clone()); + self + } + + /// Default EID-level fields for a declared source. At output, the + /// coordinator applies them to a stored entry of that source, written by + /// this provider, only when the entry sets none of `inserter`, `matcher` + /// and `mm`. An entry that sets any of them is the vendor's complete + /// statement, so vendor and config provenance never mix in one entry. + /// The defaults are not stored, so a config change applies at once. + #[must_use] + pub fn with_provenance(mut self, source: &SourceDomain, provenance: EidProvenance) -> Self { + self.provenance.insert(source.clone(), provenance); + self + } + + /// Validates and builds the descriptor. + /// + /// # Errors + /// + /// Returns `Config` when the descriptor has no source, no capability, a + /// resolver with no endpoint or host, an endpoint that is not an `https` + /// URL, a host that is not a bare hostname, provenance or an expiring + /// mark for a source it does not declare, a provenance `inserter` or + /// `matcher` that is blank or over 255 bytes, or a provenance `mm` below 0. + pub fn build(self) -> Result> { + let id = self.id; + let fail = |what: &str| IdentityProviderError::config(format!("provider `{id}`: {what}")); + + if self.sources.is_empty() { + return Err(fail("declares no source")); + } + if self.resolves.is_none() && self.captures.is_none() { + return Err(fail("declares neither resolves nor captures")); + } + for source in self.provenance.keys().chain(self.expiring.iter()) { + if !self.sources.contains(source) { + return Err(fail(&format!("names undeclared source `{source}`"))); + } + } + for (source, provenance) in &self.provenance { + let fields = [ + ("inserter", provenance.inserter()), + ("matcher", provenance.matcher()), + ]; + for (name, value) in fields { + if let Some(value) = value + && !is_valid_eid_field(value) + { + return Err(fail(&format!( + "provenance `{name}` for `{source}` is blank or over {MAX_EID_FIELD_BYTES} bytes" + ))); + } + } + if provenance.mm().is_some_and(|mm| mm < 0) { + return Err(fail(&format!("provenance `mm` for `{source}` is below 0"))); + } + } + + let mut endpoint = None; + let mut resolves = self.resolves; + if let Some(resolves) = resolves.as_mut() { + if let Some(raw) = resolves.raw_endpoint.as_deref() { + let url = Url::parse(raw).map_err(|_| fail("endpoint is not a URL"))?; + if url.scheme() != "https" { + return Err(fail("endpoint must use https")); + } + let host = url + .host_str() + .ok_or_else(|| fail("endpoint has no host"))? + .to_owned(); + if !resolves.hosts.contains(&host) { + resolves.hosts.insert(0, host); + } + endpoint = Some(url); + } + if resolves.hosts.is_empty() { + return Err(fail("resolves declares no endpoint or host")); + } + for host in &resolves.hosts { + if normalize_partner_source_domain(host).ok().as_deref() != Some(host.as_str()) { + return Err(fail(&format!( + "host `{host}` is not a bare lowercase hostname" + ))); + } + } + } + + let expiring = if self.expiring_all { + self.sources.iter().cloned().collect() + } else { + self.expiring + }; + + Ok(ProviderDescriptor { + id, + sources: self.sources, + resolves, + endpoint, + captures: self.captures, + consent_policy: self.consent_policy, + provenance: self.provenance, + expiring, + }) + } +} + +/// The kind of resolution one source needs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display)] +#[non_exhaustive] +pub enum ResolveKind { + /// No stored record for the source. + #[display("initial")] + Initial, + /// A stored record exists, valid or expired, and is due. + #[display("refresh")] + Refresh, +} + +/// Active resolution: the provider calls an endpoint. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Resolves { + raw_endpoint: Option, + hosts: Vec, + needs: BTreeSet, + needs_for: BTreeMap>, + wants: BTreeSet, + kinds: Vec, + private_state: bool, + ec_id: bool, +} + +impl Default for Resolves { + fn default() -> Self { + Self::new() + } +} + +impl Resolves { + /// A resolver with no inputs, handling `Initial` and `Refresh`. + #[must_use] + pub fn new() -> Self { + Self { + raw_endpoint: None, + hosts: Vec::new(), + needs: BTreeSet::new(), + needs_for: BTreeMap::new(), + wants: BTreeSet::new(), + kinds: vec![ResolveKind::Initial, ResolveKind::Refresh], + private_state: false, + ec_id: false, + } + } + + /// The endpoint the provider calls, as config spells it. `build()` parses + /// it and requires `https`. Its host becomes an allowed host. + #[must_use] + pub fn with_endpoint(mut self, endpoint: &str) -> Self { + self.raw_endpoint = Some(endpoint.to_owned()); + self + } + + /// Adds another host the provider may call. Repeatable. + #[must_use] + pub fn with_host(mut self, host: &str) -> Self { + if !self.hosts.iter().any(|known| known == host) { + self.hosts.push(host.to_owned()); + } + self + } + + /// Inputs required for every kind. + #[must_use] + pub fn with_needs(mut self, inputs: impl IntoIterator) -> Self { + self.needs.extend(inputs); + self + } + + /// Inputs required for one kind only. + #[must_use] + pub fn with_needs_for( + mut self, + kind: ResolveKind, + inputs: impl IntoIterator, + ) -> Self { + self.needs_for.entry(kind).or_default().extend(inputs); + self + } + + /// Inputs passed when available, never required. + #[must_use] + pub fn with_wants(mut self, inputs: impl IntoIterator) -> Self { + self.wants.extend(inputs); + self + } + + /// The kinds this provider handles. Default: `Initial` and `Refresh`. + #[must_use] + pub fn with_kinds(mut self, kinds: impl IntoIterator) -> Self { + self.kinds = kinds.into_iter().collect(); + self + } + + /// This provider keeps private state in KV (`ps.st`, spec 6.1). It can + /// then read the stored state and return a state change. + #[must_use] + pub fn with_private_state(mut self) -> Self { + self.private_state = true; + self + } + + /// Gives the provider the full EC ID. Only generic pull may use this. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(in crate::ec::identity) fn with_ec_id(mut self) -> Self { + self.ec_id = true; + self + } + + /// Hosts the provider may call. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn hosts(&self) -> &[String] { + &self.hosts + } + + /// Inputs required for `kind`: the common needs plus that kind's. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn needs(&self, kind: ResolveKind) -> BTreeSet { + let mut needs = self.needs.clone(); + if let Some(extra) = self.needs_for.get(&kind) { + needs.extend(extra.iter().copied()); + } + needs + } + + /// Every input declared for any kind, needed or wanted. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn declared_inputs(&self) -> BTreeSet { + let mut all = self.needs.clone(); + all.extend(self.wants.iter().copied()); + for extra in self.needs_for.values() { + all.extend(extra.iter().copied()); + } + all + } + + /// Inputs passed when available. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn wants(&self) -> &BTreeSet { + &self.wants + } + + /// The kinds this provider handles. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn kinds(&self) -> &[ResolveKind] { + &self.kinds + } + + /// Whether the provider keeps private state. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn has_private_state(&self) -> bool { + self.private_state + } + + /// Whether the provider receives the full EC ID. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn gets_ec_id(&self) -> bool { + self.ec_id + } +} + +/// Revoke-only capture from responses TS proxies for this provider: it +/// can remove IDs, opt out or clear an opt-out, and never stores a positive +/// ID (spec 4.7). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Captures { + paths: Vec<(Method, String)>, + prefixes: Vec, +} + +impl Captures { + /// No capture rules yet. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Offers proxied responses to exactly this method and path. Repeatable. + #[must_use] + pub fn with_path(mut self, method: Method, path: &str) -> Self { + self.paths.push((method, path.to_owned())); + self + } + + /// Offers proxied responses whose path starts with `prefix`, for any + /// method. Repeatable. + #[must_use] + pub fn with_path_prefix(mut self, prefix: &str) -> Self { + self.prefixes.push(prefix.to_owned()); + self + } + + /// Whether a proxied request matches a rule. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn matches(&self, method: &Method, path: &str) -> bool { + self.paths + .iter() + .any(|(rule_method, rule_path)| rule_method == method && rule_path == path) + || self + .prefixes + .iter() + .any(|prefix| path.starts_with(prefix.as_str())) + } + + /// Exact path rules. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn paths(&self) -> &[(Method, String)] { + &self.paths + } + + /// Prefix rules. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn prefixes(&self) -> &[String] { + &self.prefixes + } +} + +/// Default EID-level fields (`OpenRTB` 2.6 §3.2.27) that bidders read, such +/// as who inserted an ID and how it was matched. They apply only to an entry +/// that sets none of the three. A provider whose vendor returns these per +/// response sets them on each `ResolvedEid` instead (spec 4.5). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct EidProvenance { + inserter: Option, + matcher: Option, + mm: Option, +} + +impl EidProvenance { + /// No fields set. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Sets `inserter`. + #[must_use] + pub fn with_inserter(mut self, inserter: &str) -> Self { + self.inserter = Some(inserter.to_owned()); + self + } + + /// Sets `matcher`. + #[must_use] + pub fn with_matcher(mut self, matcher: &str) -> Self { + self.matcher = Some(matcher.to_owned()); + self + } + + /// Sets `mm`. + #[must_use] + pub fn with_mm(mut self, mm: i32) -> Self { + self.mm = Some(mm); + self + } + + /// `inserter`, if set. + #[must_use] + pub fn inserter(&self) -> Option<&str> { + self.inserter.as_deref() + } + + /// `matcher`, if set. + #[must_use] + pub fn matcher(&self) -> Option<&str> { + self.matcher.as_deref() + } + + /// `mm`, if set. + #[must_use] + pub fn mm(&self) -> Option { + self.mm + } +} + +/// A US opt-out a policy can relax. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, derive_more::Display)] +#[non_exhaustive] +pub enum UsOptOut { + /// Sale of personal information. + #[display("sale")] + Sale, + /// Sharing for cross-context behavioural advertising. + #[display("sharing")] + Sharing, + /// Targeted advertising. + #[display("targeted_advertising")] + TargetedAdvertising, +} + +/// The consent a provider requires. Evaluated by the coordinator (Task 12). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConsentPolicy { + tcf_purposes: BTreeSet, + tcf_vendor: Option, + allow_dnt: bool, + allow_gpc: bool, + allowed_us_opt_outs: BTreeSet, + eid_persistence_only: bool, +} + +impl ConsentPolicy { + /// The default. Blocked when any consent signal present cannot be + /// decoded. Where TCF applies, requires Purposes 1 and 4. Blocked by + /// every US opt-out TS decodes, by `Sec-GPC: 1` and by `DNT: 1`. + #[must_use] + pub fn strict() -> Self { + Self { + tcf_purposes: [1, 4].into_iter().collect(), + tcf_vendor: None, + allow_dnt: false, + allow_gpc: false, + allowed_us_opt_outs: BTreeSet::new(), + eid_persistence_only: false, + } + } + + /// Also requires these TCF purposes, where TCF applies. + #[must_use] + pub fn with_tcf_purposes(mut self, purposes: impl IntoIterator) -> Self { + self.tcf_purposes.extend(purposes); + self + } + + /// Requires vendor consent for this IAB Global Vendor List ID. + #[must_use] + pub fn with_tcf_vendor(mut self, vendor_id: u16) -> Self { + self.tcf_vendor = Some(vendor_id); + self + } + + /// Does not block on `DNT: 1`. Needs a documented legal basis. + #[must_use] + pub fn allowing_dnt(mut self) -> Self { + self.allow_dnt = true; + self + } + + /// Does not block on GPC. Needs a documented legal basis. + #[must_use] + pub fn allowing_gpc(mut self) -> Self { + self.allow_gpc = true; + self + } + + /// Does not block on one US opt-out. Needs a documented legal basis. + #[must_use] + pub fn allowing_us_opt_out(mut self, opt_out: UsOptOut) -> Self { + self.allowed_us_opt_outs.insert(opt_out); + self + } + + /// Blocks exactly what pull sync blocks today: only the global EC and + /// EID-persistence gates. For generic pull's parity (spec 10.4). + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(in crate::ec::identity) fn eid_persistence_only() -> Self { + Self { + eid_persistence_only: true, + ..Self::strict() + } + } + + /// Required TCF purposes. + #[must_use] + pub fn tcf_purposes(&self) -> &BTreeSet { + &self.tcf_purposes + } + + /// Required GVL vendor, if any. + #[must_use] + pub fn tcf_vendor(&self) -> Option { + self.tcf_vendor + } + + /// Whether `DNT: 1` is allowed. + #[must_use] + pub fn allows_dnt(&self) -> bool { + self.allow_dnt + } + + /// Whether GPC is allowed. + #[must_use] + pub fn allows_gpc(&self) -> bool { + self.allow_gpc + } + + /// Whether a US opt-out is allowed. + #[must_use] + pub fn allows_us_opt_out(&self, opt_out: UsOptOut) -> bool { + self.allowed_us_opt_outs.contains(&opt_out) + } + + /// Whether this is generic pull's parity policy. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn is_eid_persistence_only(&self) -> bool { + self.eid_persistence_only + } +} + /// An input a provider can ask for. /// /// Consent strings, the provider's stored IDs, the publisher domain, the @@ -39,3 +731,300 @@ pub enum Access { #[display("private_state")] PrivateState, } + +#[cfg(test)] +mod tests { + use http::Method; + + use super::*; + use crate::ec::kv_types::MAX_EID_FIELD_BYTES; + + fn source(name: &str) -> SourceDomain { + name.parse().expect("should parse source") + } + + fn id(name: &str) -> IdentityProviderId { + name.parse().expect("should parse id") + } + + #[test] + fn builds_the_documented_example() { + let descriptor = ProviderDescriptor::builder(id("exampleid")) + .with_source(source("exampleid.example.com")) + .with_resolves( + Resolves::new() + .with_endpoint("https://api.exampleid.example.com/v1/fetch") + .with_needs([Input::ClientIp, Input::UserAgent]) + .with_wants([Input::PageUrl]), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(9999)) + .build() + .expect("should build"); + + assert_eq!(descriptor.id().as_str(), "exampleid", "should keep the id"); + assert_eq!( + descriptor.primary_source(), + &source("exampleid.example.com"), + "should keep the source" + ); + assert_eq!( + descriptor.endpoint().map(Url::as_str), + Some("https://api.exampleid.example.com/v1/fetch"), + "should parse the endpoint" + ); + let resolves = descriptor.resolves().expect("should resolve"); + assert_eq!( + resolves.hosts(), + &["api.exampleid.example.com".to_owned()], + "should allow the endpoint host" + ); + assert_eq!( + resolves.kinds(), + &[ResolveKind::Initial, ResolveKind::Refresh], + "should default to Initial and Refresh" + ); + assert!(descriptor.captures().is_none(), "should not capture"); + assert!( + !descriptor.is_expiring(&source("exampleid.example.com")), + "should not expire by default" + ); + assert_eq!( + descriptor.consent_policy().tcf_vendor(), + Some(9999), + "should keep the vendor" + ); + assert_eq!( + descriptor + .consent_policy() + .tcf_purposes() + .iter() + .copied() + .collect::>(), + vec![1, 4], + "should require Purposes 1 and 4 under strict" + ); + } + + #[test] + fn build_rejects_invalid_descriptors() { + let cases: Vec<(&str, ProviderDescriptorBuilder)> = vec![ + ( + "no source", + ProviderDescriptor::builder(id("p")) + .with_resolves(Resolves::new().with_endpoint("https://a.example.com")), + ), + ( + "no capability", + ProviderDescriptor::builder(id("p")).with_source(source("example.com")), + ), + ( + "no endpoint or host", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves(Resolves::new()), + ), + ( + "http endpoint", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves(Resolves::new().with_endpoint("http://a.example.com")), + ), + ( + "bad host", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_resolves(Resolves::new().with_host("a.example.com/path")), + ), + ( + "provenance for undeclared source", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("other.example.com"), + EidProvenance::new().with_mm(2), + ), + ), + ( + "expiring undeclared source", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_expiring_source(&source("other.example.com")), + ), + ( + "blank provenance inserter", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("example.com"), + EidProvenance::new().with_inserter(" "), + ), + ), + ( + "blank provenance matcher", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("example.com"), + EidProvenance::new().with_matcher(""), + ), + ), + ( + "oversized provenance matcher", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("example.com"), + EidProvenance::new().with_matcher(&"m".repeat(MAX_EID_FIELD_BYTES + 1)), + ), + ), + ( + "negative provenance mm", + ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance(&source("example.com"), EidProvenance::new().with_mm(-1)), + ), + ]; + for (name, builder) in cases { + let error = builder + .build() + .expect_err(&format!("should reject a descriptor with {name}")); + assert!( + matches!( + error.current_context(), + IdentityProviderError::Config { .. } + ), + "should be a Config error: {name}" + ); + } + } + + #[test] + fn expiring_and_provenance_are_per_source() { + let a = source("a.example.com"); + let b = source("b.example.com"); + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(a.clone()) + .with_source(b.clone()) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_expiring_source(&b) + .with_provenance( + &a, + EidProvenance::new() + .with_inserter("example-inserter") + .with_mm(2), + ) + .build() + .expect("should build"); + + assert!(!descriptor.is_expiring(&a), "should not mark a as expiring"); + assert!(descriptor.is_expiring(&b), "should mark b as expiring"); + let provenance = descriptor + .provenance(&a) + .expect("should have provenance for a"); + assert_eq!( + provenance.inserter(), + Some("example-inserter"), + "should keep the inserter" + ); + assert_eq!(provenance.matcher(), None, "should leave the matcher unset"); + assert_eq!(provenance.mm(), Some(2), "should keep mm"); + assert!( + descriptor.provenance(&b).is_none(), + "should have no provenance for b" + ); + assert_eq!( + descriptor.source_named("A.EXAMPLE.COM"), + Some(&a), + "should find a by any spelling" + ); + } + + #[test] + fn expiring_ids_marks_every_source() { + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(source("a.example.com")) + .with_source(source("b.example.com")) + .with_expiring_ids() + .with_captures(Captures::new().with_path_prefix("/integrations/p/")) + .build() + .expect("should build"); + assert!( + descriptor + .sources() + .iter() + .all(|s| descriptor.is_expiring(s)), + "should mark every source" + ); + } + + #[test] + fn captures_match_exact_paths_and_prefixes() { + let captures = Captures::new() + .with_path(Method::POST, "/integrations/p/api/token") + .with_path_prefix("/integrations/p/stream/"); + assert!( + captures.matches(&Method::POST, "/integrations/p/api/token"), + "should match the exact path" + ); + assert!( + !captures.matches(&Method::GET, "/integrations/p/api/token"), + "should require the method" + ); + assert!( + !captures.matches(&Method::POST, "/integrations/p/api/token/x"), + "should not treat an exact path as a prefix" + ); + assert!( + captures.matches(&Method::GET, "/integrations/p/stream/1"), + "should match a prefix for any method" + ); + assert!( + !captures.matches(&Method::GET, "/integrations/p/settings"), + "should not match others" + ); + } + + #[test] + fn needs_differ_by_kind() { + let resolves = Resolves::new() + .with_endpoint("https://a.example.com") + .with_needs([Input::ClientIp]) + .with_needs_for(ResolveKind::Initial, [Input::HashedEmail]); + assert_eq!( + resolves.needs(ResolveKind::Initial), + [Input::ClientIp, Input::HashedEmail].into_iter().collect(), + "should need both inputs for Initial" + ); + assert_eq!( + resolves.needs(ResolveKind::Refresh), + [Input::ClientIp].into_iter().collect(), + "should need only the IP for Refresh" + ); + } + + #[test] + fn valid_provenance_builds() { + let descriptor = ProviderDescriptor::builder(id("p")) + .with_source(source("example.com")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .with_provenance( + &source("example.com"), + EidProvenance::new() + .with_inserter("example-inserter") + .with_matcher("example-matcher") + .with_mm(0), + ) + .build() + .expect("should accept non-blank provenance and mm 0"); + assert!( + descriptor.provenance(&source("example.com")).is_some(), + "should keep the provenance" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index e4e64f8451..2b7614a374 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -10,6 +10,9 @@ pub mod descriptor; pub mod ids; pub mod resolution; -pub use descriptor::{Access, Input}; +pub use descriptor::{ + Access, Captures, ConsentPolicy, EidProvenance, Input, ProviderDescriptor, + ProviderDescriptorBuilder, ResolveKind, Resolves, UsOptOut, +}; pub use ids::{IdentityProviderId, SourceDomain, UnixTime}; pub use resolution::IdentityProviderError; From 81aad20aa35ae2ff37f77565c981ab58f7c91dc7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:53:21 +0530 Subject: [PATCH 17/89] Add declared-input enforcement for identity providers Signed-off-by: dhruv8sh --- .../src/ec/identity/descriptor.rs | 3 + .../src/ec/identity/input.rs | 1033 +++++++++++++++++ .../src/ec/identity/mod.rs | 2 + .../src/migration_guards.rs | 1 + 4 files changed, 1039 insertions(+) create mode 100644 crates/trusted-server-core/src/ec/identity/input.rs diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index d3ed46634a..9476b2ff86 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -730,6 +730,9 @@ pub enum Access { /// Private state, without `with_private_state()`. #[display("private_state")] PrivateState, + /// The full EC ID, without `with_ec_id()`. Only generic pull declares it. + #[display("ec_id")] + EcId, } #[cfg(test)] diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs new file mode 100644 index 0000000000..63a833711d --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -0,0 +1,1033 @@ +//! What a provider may read for one call. + +// Temporary: the coordinator, the first non-test caller, arrives in a later +// task. Remove this when it lands. +#![allow(dead_code)] + +use core::cell::RefCell; +use core::fmt; +use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use error_stack::Report; +use rand::RngCore as _; +use serde::de::DeserializeOwned; +use url::Url; + +use super::descriptor::{Access, Input, ProviderDescriptor, ResolveKind, Resolves}; +use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::resolution::IdentityProviderError; +use crate::consent::ConsentContext; +use crate::ec::kv_types::KvPartnerId; +use crate::openrtb::{Eid, Uid}; + +/// The raw consent strings a provider forwards, and `gdpr_applies`. +/// +/// Providers never decide consent. The coordinator has already applied the +/// descriptor's policy. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct ProviderConsent { + gdpr_applies: bool, + tc_string: Option, + gpp_string: Option, + gpp_section_ids: Option>, + us_privacy: Option, +} + +impl fmt::Debug for ProviderConsent { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderConsent") + .field("gdpr_applies", &self.gdpr_applies) + .field("tc_string", &self.tc_string.as_ref().map(|_| "[redacted]")) + .field( + "gpp_string", + &self.gpp_string.as_ref().map(|_| "[redacted]"), + ) + .field("gpp_section_ids", &self.gpp_section_ids) + .field( + "us_privacy", + &self.us_privacy.as_ref().map(|_| "[redacted]"), + ) + .finish() + } +} + +impl ProviderConsent { + /// The view of a request's consent context. + #[must_use] + pub(crate) fn from_context(context: &ConsentContext) -> Self { + Self { + gdpr_applies: context.gdpr_applies, + tc_string: context.raw_tc_string.clone(), + gpp_string: context.raw_gpp_string.clone(), + gpp_section_ids: context.gpp_section_ids.clone(), + us_privacy: context.raw_us_privacy.clone(), + } + } + + /// Whether GDPR applies to this request. + #[must_use] + pub fn gdpr_applies(&self) -> bool { + self.gdpr_applies + } + + /// The raw TCF string. + #[must_use] + pub fn raw_tc_string(&self) -> Option<&str> { + self.tc_string.as_deref() + } + + /// The raw GPP string. + #[must_use] + pub fn raw_gpp_string(&self) -> Option<&str> { + self.gpp_string.as_deref() + } + + /// The applicable GPP section IDs. + #[must_use] + pub fn gpp_section_ids(&self) -> Option<&[u16]> { + self.gpp_section_ids.as_deref() + } + + /// The raw US Privacy string. + #[must_use] + pub fn raw_us_privacy(&self) -> Option<&str> { + self.us_privacy.as_deref() + } +} + +/// Country and subdivision, upper case. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Region { + country: String, + subdivision: Option, +} + +impl Region { + /// Normalizes platform geo: upper case, and an ISO 3166-2 `US-CA` style + /// region loses its country prefix. `None` without a two-letter country. + #[must_use] + pub(crate) fn from_geo(country: &str, region: Option<&str>) -> Option { + let country = country.trim().to_ascii_uppercase(); + if country.len() != 2 || !country.bytes().all(|byte| byte.is_ascii_alphabetic()) { + return None; + } + let subdivision = region + .map(|value| value.trim().to_ascii_uppercase()) + .map(|value| match value.split_once('-') { + Some((prefix, rest)) if prefix == country => rest.to_owned(), + _ => value, + }) + .filter(|value| { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_alphanumeric()) + }); + Some(Self { + country, + subdivision, + }) + } + + /// ISO 3166-1 alpha-2, upper case, for example `"US"`. + #[must_use] + pub fn country(&self) -> &str { + &self.country + } + + /// The ISO 3166-2 subdivision without the country prefix, for example + /// `"CA"`. `None` when the platform does not know it. + #[must_use] + pub fn subdivision(&self) -> Option<&str> { + self.subdivision.as_deref() + } + + /// Whether this is `country` and `subdivision`. + #[must_use] + pub fn is(&self, country: &str, subdivision: &str) -> bool { + self.country == country && self.subdivision.as_deref() == Some(subdivision) + } +} + +/// The 32 raw bytes of a SHA-256 of a normalized email. Personal data. +#[derive(Clone, PartialEq, Eq)] +pub struct HashedEmail([u8; 32]); + +impl fmt::Debug for HashedEmail { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HashedEmail([redacted])") + } +} + +impl HashedEmail { + /// From digest bytes. Test support only: v1 has no source of hashed + /// emails, so nothing else builds one. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// The digest. + #[must_use] + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + /// Lowercase hex, as `LiveRamp` takes it. + #[must_use] + pub fn to_hex(&self) -> String { + hex::encode(self.0) + } + + /// Standard base64, as `UID2` takes it. + #[must_use] + pub fn to_base64(&self) -> String { + STANDARD.encode(self.0) + } +} + +/// A stored record for one of this provider's sources, valid or expired. +/// +/// It holds the source's whole entry set, which is never empty. +#[derive(Clone, PartialEq)] +pub struct StoredId { + eids: Vec, + uid: String, + writer: Option, + acquired_at: Option, + expires_at: Option, + refresh_at: Option, +} + +impl fmt::Debug for StoredId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("StoredId") + .field("entries", &self.eids.len()) + .field("uid", &"[redacted]") + .field("writer", &self.writer) + .field("acquired_at", &self.acquired_at) + .field("expires_at", &self.expires_at) + .field("refresh_at", &self.refresh_at) + .finish_non_exhaustive() + } +} + +impl StoredId { + /// From the KV record of `source`. A writer that does not parse is + /// treated as none. `None` for a record with no entry, which + /// `KvEntry::validate` never lets through. + #[must_use] + pub(crate) fn from_record(source: &SourceDomain, record: &KvPartnerId) -> Option { + let uid = record.first_uid()?.to_owned(); + let eids = record + .eids + .iter() + .map(|entry| Eid { + source: source.as_str().to_owned(), + uids: entry + .uids + .iter() + .map(|uid| Uid { + id: uid.id.clone(), + atype: uid.atype, + ext: uid.ext.clone(), + }) + .collect(), + inserter: entry.inserter.clone(), + matcher: entry.matcher.clone(), + mm: entry.mm, + ext: entry.ext.clone(), + }) + .collect(); + Some(Self { + eids, + uid, + writer: record.w.as_deref().and_then(|writer| writer.parse().ok()), + acquired_at: record.at.map(UnixTime::from_secs), + expires_at: record.exp.map(UnixTime::from_secs), + refresh_at: record.ra.map(UnixTime::from_secs), + }) + } + + fn first_uid(&self) -> Option<&Uid> { + self.eids.first().and_then(|eid| eid.uids.first()) + } + + /// Every stored entry for the source, in stored order, with its UIDs and + /// EID-level fields. A record that browser ingest, generic pull, batch + /// sync or a version 1 row wrote is one entry with one UID. + #[must_use] + pub fn eids(&self) -> &[Eid] { + &self.eids + } + + /// The first UID of the first entry. What a single-UID refresh, such as + /// `LiveRamp`'s, sends back. + #[must_use] + pub fn uid(&self) -> &str { + &self.uid + } + + /// The `atype` of that first UID. + #[must_use] + pub fn atype(&self) -> Option { + self.first_uid().and_then(|uid| uid.atype) + } + + /// The UID-level ext of that first UID, as bidders read it in + /// `uids[].ext`. + #[must_use] + pub fn ext(&self) -> Option<&serde_json::Value> { + self.first_uid().and_then(|uid| uid.ext.as_ref()) + } + + /// The provider that wrote it. `None` for browser ingest and legacy rows. + #[must_use] + pub fn writer(&self) -> Option<&IdentityProviderId> { + self.writer.as_ref() + } + + /// When it was acquired or last refreshed. + #[must_use] + pub fn acquired_at(&self) -> Option { + self.acquired_at + } + + /// When it must stop being sent. + #[must_use] + pub fn expires_at(&self) -> Option { + self.expires_at + } + + /// When it is due for refresh. + #[must_use] + pub fn refresh_at(&self) -> Option { + self.refresh_at + } + + /// Whether `now` is at or past its expiry. + #[must_use] + pub fn is_expired(&self, now: UnixTime) -> bool { + self.expires_at.is_some_and(|expires| now >= expires) + } +} + +/// A source of random bytes. +pub(crate) trait RandomSource: Send + Sync { + /// Fills `bytes`. + fn fill(&self, bytes: &mut [u8]); +} + +/// The platform generator EC generation uses (`rand::thread_rng`). +pub(crate) struct ThreadRandom; + +impl RandomSource for ThreadRandom { + fn fill(&self, bytes: &mut [u8]) { + rand::thread_rng().fill_bytes(bytes); + } +} + +/// Deterministic bytes for tests: xorshift64* from a seed. +#[cfg(any(test, feature = "test-utils"))] +#[derive(Debug)] +pub struct SeededRandom(std::sync::Mutex); + +#[cfg(any(test, feature = "test-utils"))] +impl SeededRandom { + /// A generator that yields the same bytes for the same seed. A zero + /// seed is treated as one, since xorshift never leaves zero. + #[must_use] + pub fn new(seed: u64) -> Self { + Self(std::sync::Mutex::new(seed.max(1))) + } +} + +#[cfg(any(test, feature = "test-utils"))] +impl RandomSource for SeededRandom { + fn fill(&self, bytes: &mut [u8]) { + let mut state = self.0.lock().expect("should lock the seed"); + for byte in bytes { + *state ^= *state >> 12; + *state ^= *state << 25; + *state ^= *state >> 27; + *byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D).to_be_bytes()[0]; + } + } +} + +/// Every signal the coordinator holds for one request. Crate-private: a +/// provider only ever sees a [`ResolveInput`] filtered by its descriptor. +pub(crate) struct ResolveSignals<'a> { + pub(crate) client_ip: Option, + pub(crate) user_agent: Option<&'a str>, + pub(crate) page_url: Option, + pub(crate) region: Option, + pub(crate) hashed_email: Option, + pub(crate) consent: ProviderConsent, + pub(crate) publisher_domain: &'a str, + pub(crate) now: UnixTime, + pub(crate) random: &'a dyn RandomSource, + pub(crate) ec_id: Option, + pub(crate) stored: BTreeMap, + pub(crate) state_json: Option, +} + +/// What one provider may read for one call. +pub struct ResolveInput<'a> { + descriptor: &'a ProviderDescriptor, + kinds: BTreeMap, + signals: &'a ResolveSignals<'a>, + declared: BTreeSet, + available: BTreeSet, + undeclared: RefCell>, + state_failed: RefCell, +} + +impl fmt::Debug for ResolveInput<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ResolveInput") + .field("provider", self.descriptor.id()) + .field("kinds", &self.kinds) + .finish_non_exhaustive() + } +} + +impl<'a> ResolveInput<'a> { + /// The input for one call. `kinds` holds only the sources to resolve. + /// The inputs of every kind present are available, plus the wanted ones. + /// An input declared only for a kind this call does not hold is + /// declared, so reading it is not an error of the provider, but it is + /// unavailable (spec 4.4). + #[must_use] + pub(crate) fn new( + descriptor: &'a ProviderDescriptor, + kinds: BTreeMap, + signals: &'a ResolveSignals<'a>, + ) -> Self { + let (declared, available) = + descriptor + .resolves() + .map_or_else(Default::default, |resolves| { + let mut available = resolves.wants().clone(); + for kind in kinds.values().copied().collect::>() { + available.extend(resolves.needs(kind)); + } + (resolves.declared_inputs(), available) + }); + Self { + descriptor, + kinds, + signals, + declared, + available, + undeclared: RefCell::new(Vec::new()), + state_failed: RefCell::new(false), + } + } + + fn record(&self, access: Access) -> Report { + self.undeclared.borrow_mut().push(access); + Report::new(IdentityProviderError::UndeclaredInput { access }) + } + + fn declared( + &self, + input: Input, + value: Option, + ) -> Result> { + if !self.declared.contains(&input) { + return Err(self.record(Access::Input(input))); + } + if !self.available.contains(&input) { + return Err(Report::new(IdentityProviderError::InputUnavailable { + input, + })); + } + value.ok_or_else(|| Report::new(IdentityProviderError::InputUnavailable { input })) + } + + /// Undeclared reads recorded so far. The coordinator logs them as + /// `Config`, and the harness fails the test. + #[must_use] + pub(crate) fn recorded_errors(&self) -> Vec { + self.undeclared.borrow().clone() + } + + /// Whether a `state()` call failed to decode. + #[must_use] + pub(crate) fn state_decode_failed(&self) -> bool { + *self.state_failed.borrow() + } + + /// The sources to resolve this time, each with its own kind. + pub fn sources(&self) -> impl Iterator + '_ { + self.kinds.iter().map(|(source, kind)| (source, *kind)) + } + + /// Whether `source` is being resolved for the first time. + #[must_use] + pub fn is_initial(&self, source: &SourceDomain) -> bool { + self.kinds.get(source) == Some(&ResolveKind::Initial) + } + + /// Whether `source` is being refreshed. + #[must_use] + pub fn is_refresh(&self, source: &SourceDomain) -> bool { + self.kinds.get(source) == Some(&ResolveKind::Refresh) + } + + /// The descriptor's primary source. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + self.descriptor.primary_source() + } + + /// The stored record for one of this provider's sources. + #[must_use] + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId> { + self.signals.stored.get(source) + } + + /// This provider's private state, decoded. `Ok(None)` when there is none. + /// + /// # Errors + /// + /// `InvalidState` when the stored state does not decode as `T`. The + /// failure is recorded, and the coordinator clears the state whatever + /// the provider returns. `UndeclaredInput` without `with_private_state()`. + pub fn state(&self) -> Result, Report> { + if !self + .descriptor + .resolves() + .is_some_and(Resolves::has_private_state) + { + return Err(self.record(Access::PrivateState)); + } + let Some(raw) = self.signals.state_json.as_deref() else { + return Ok(None); + }; + serde_json::from_str(raw).map(Some).map_err(|_| { + *self.state_failed.borrow_mut() = true; + Report::new(IdentityProviderError::InvalidState) + }) + } + + /// The endpoint declared with `Resolves::with_endpoint`. + /// + /// # Errors + /// + /// `UndeclaredInput { access: Endpoint }` when none was declared. + pub fn endpoint(&self) -> Result<&Url, Report> { + self.descriptor + .endpoint() + .ok_or_else(|| self.record(Access::Endpoint)) + } + + /// The consent strings to forward. + #[must_use] + pub fn consent(&self) -> &ProviderConsent { + &self.signals.consent + } + + /// The publisher's domain. + #[must_use] + pub fn publisher_domain(&self) -> &str { + self.signals.publisher_domain + } + + /// The request's time. + #[must_use] + pub fn now(&self) -> UnixTime { + self.signals.now + } + + /// Random bytes from the platform generator. Fixed by the harness. + #[must_use] + pub fn random_bytes(&self) -> [u8; N] { + let mut bytes = [0u8; N]; + self.signals.random.fill(&mut bytes); + bytes + } + + /// The platform's client address. + /// + /// # Errors + /// + /// `UndeclaredInput` when not declared, `InputUnavailable` when missing. + pub fn client_ip(&self) -> Result> { + self.declared(Input::ClientIp, self.signals.client_ip) + } + + /// The `User-Agent`. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn user_agent(&self) -> Result<&str, Report> { + self.declared(Input::UserAgent, self.signals.user_agent) + } + + /// The page URL, without query string or fragment. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn page_url(&self) -> Result<&Url, Report> { + self.declared(Input::PageUrl, self.signals.page_url.as_ref()) + } + + /// Country and subdivision. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn region(&self) -> Result<&Region, Report> { + self.declared(Input::Region, self.signals.region.as_ref()) + } + + /// A hashed email. Never available in v1. + /// + /// # Errors + /// + /// As [`Self::client_ip`]. + pub fn hashed_email(&self) -> Result<&HashedEmail, Report> { + self.declared(Input::HashedEmail, self.signals.hashed_email.as_ref()) + } + + /// The full EC ID. Only for a resolver declared `with_ec_id()`. + /// + /// # Errors + /// + /// `UndeclaredInput { access: EcId }` otherwise. `Config` when the + /// request has no EC ID, which the coordinator never allows: it does not + /// dispatch generic pull without one. + pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report> { + if !self.descriptor.resolves().is_some_and(Resolves::gets_ec_id) { + return Err(self.record(Access::EcId)); + } + self.signals.ec_id.as_deref().ok_or_else(|| { + IdentityProviderError::config("generic pull was called without an EC ID") + }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::net::IpAddr; + + use super::*; + use crate::ec::identity::descriptor::{ProviderDescriptor, Resolves}; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + + fn descriptor(resolves: Resolves) -> ProviderDescriptor { + ProviderDescriptor::builder("p".parse().expect("should parse the provider id")) + .with_source("example.com".parse().expect("should parse the source")) + .with_resolves(resolves) + .build() + .expect("should build") + } + + fn signals() -> ResolveSignals<'static> { + ResolveSignals { + client_ip: Some("192.0.2.10".parse().expect("should parse ip")), + user_agent: Some("ExampleBrowser/1.0"), + page_url: None, + region: Region::from_geo("us", Some("us-ca")), + hashed_email: None, + consent: ProviderConsent::default(), + publisher_domain: "www.example.com", + now: UnixTime::from_secs(1_760_000_000), + random: &ThreadRandom, + ec_id: Some("a".repeat(64) + ".ABC123"), + stored: BTreeMap::new(), + state_json: None, + } + } + + #[test] + fn declared_inputs_are_available_and_wanted_missing_ones_are_unavailable() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_needs([Input::ClientIp, Input::Region]) + .with_wants([Input::PageUrl]), + ); + let signals = signals(); + let input = ResolveInput::new( + &descriptor, + kinds(&descriptor, ResolveKind::Initial), + &signals, + ); + + assert_eq!( + input.client_ip().expect("should read the ip"), + "192.0.2.10".parse::().expect("should parse the ip"), + "should return the client ip" + ); + let region = input.region().expect("should read region"); + assert_eq!( + (region.country(), region.subdivision()), + ("US", Some("CA")), + "should normalize the region" + ); + assert!(region.is("US", "CA"), "should test country and subdivision"); + let error = input.page_url().expect_err("should be unavailable"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::InputUnavailable { + input: Input::PageUrl + } + ), + "should report a declared but missing input as InputUnavailable" + ); + assert!( + input.recorded_errors().is_empty(), + "should record nothing for declared reads" + ); + } + + #[test] + fn input_declared_only_for_an_absent_kind_is_unavailable() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_needs([Input::ClientIp]) + .with_needs_for(ResolveKind::Refresh, [Input::UserAgent]), + ); + let signals = signals(); + + let initial = ResolveInput::new( + &descriptor, + kinds(&descriptor, ResolveKind::Initial), + &signals, + ); + let refresh = ResolveInput::new( + &descriptor, + kinds(&descriptor, ResolveKind::Refresh), + &signals, + ); + + let error = initial + .user_agent() + .expect_err("should hide a Refresh-only input on an Initial call"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::InputUnavailable { + input: Input::UserAgent + } + ), + "should report InputUnavailable, not an undeclared read" + ); + assert!( + initial.recorded_errors().is_empty(), + "should record no undeclared read" + ); + assert_eq!( + refresh + .user_agent() + .expect("should read the user agent on a Refresh call"), + "ExampleBrowser/1.0", + "should return the user agent for the kind that declares it" + ); + } + + #[test] + fn undeclared_reads_are_recorded_even_when_ignored() { + let descriptor = descriptor(Resolves::new().with_endpoint("https://a.example.com")); + let signals = signals(); + let input = ResolveInput::new( + &descriptor, + kinds(&descriptor, ResolveKind::Initial), + &signals, + ); + + let ignored = input.user_agent().ok(); + let state = input.state::(); + + assert!(ignored.is_none(), "should not return an undeclared input"); + assert!(state.is_err(), "should need with_private_state for state"); + assert_eq!( + input.recorded_errors(), + vec![Access::Input(Input::UserAgent), Access::PrivateState], + "should record both undeclared reads" + ); + input + .endpoint() + .expect("should always return the declared endpoint"); + } + + #[test] + fn state_decodes_and_a_bad_state_is_recorded() { + let descriptor = descriptor( + Resolves::new() + .with_endpoint("https://a.example.com") + .with_private_state(), + ); + let mut good = signals(); + good.state_json = Some(r#"{"signature":"s"}"#.to_owned()); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &good); + let state: serde_json::Value = input + .state() + .expect("should decode the state") + .expect("should have a state"); + assert_eq!(state["signature"], "s", "should decode the state"); + assert!( + !input.state_decode_failed(), + "should not record a good state as a failure" + ); + + let mut bad = signals(); + bad.state_json = Some(r#""a bare string""#.to_owned()); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &bad); + let error = input + .state::() + .expect_err("should not decode a bare string"); + assert!( + matches!(error.current_context(), IdentityProviderError::InvalidState), + "should report InvalidState" + ); + assert!(input.state_decode_failed(), "should record the failure"); + } + + #[test] + fn ec_id_is_only_for_providers_declared_with_ec_id() { + let generic = descriptor(Resolves::new().with_host("a.example.com").with_ec_id()); + let other = descriptor(Resolves::new().with_host("a.example.com")); + let with_ec = signals(); + ResolveInput::new(&generic, kinds(&generic, ResolveKind::Initial), &with_ec) + .ec_id() + .expect("should let generic pull read the EC ID"); + let input = ResolveInput::new(&other, kinds(&other, ResolveKind::Initial), &with_ec); + assert!( + input.ec_id().is_err(), + "should refuse the EC ID to other providers" + ); + assert_eq!( + input.recorded_errors(), + vec![Access::EcId], + "should record an EC ID read" + ); + + let mut no_ec = signals(); + no_ec.ec_id = None; + let error = ResolveInput::new(&generic, kinds(&generic, ResolveKind::Initial), &no_ec) + .ec_id() + .expect_err("should fail without an EC ID"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::Config { .. } + ), + "should report a missing EC ID as Config, not as a missing input" + ); + } + + #[test] + fn stored_ids_and_expiry() { + let record = KvPartnerId { + w: Some("p".to_owned()), + rv: Some("0123456789abcdef".to_owned()), + at: Some(10), + exp: Some(20), + ra: Some(15), + ru: None, + eids: vec![KvEid { + uids: vec![KvUid { + id: "u".to_owned(), + atype: Some(3), + ext: None, + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + }], + }; + let source: SourceDomain = "example.com".parse().expect("should parse the source"); + let stored = + StoredId::from_record(&source, &record).expect("should read a record with an entry"); + assert_eq!( + stored.writer().map(IdentityProviderId::as_str), + Some("p"), + "should parse the writer" + ); + assert!( + !stored.is_expired(UnixTime::from_secs(19)), + "should not be expired before exp" + ); + assert!( + stored.is_expired(UnixTime::from_secs(20)), + "should be expired at exp" + ); + assert_eq!( + stored.refresh_at(), + Some(UnixTime::from_secs(15)), + "should read ra" + ); + assert_eq!( + stored.eids().len(), + 1, + "should expose a one-UID record as one entry" + ); + let eid = &stored.eids()[0]; + assert_eq!(eid.source, "example.com", "should fill in the source"); + assert_eq!(eid.uids.len(), 1, "should expose its one UID"); + assert_eq!(eid.uids[0].id, "u", "should keep the UID"); + assert_eq!(eid.uids[0].atype, Some(3), "should keep the atype"); + assert_eq!( + ( + eid.inserter.as_deref(), + eid.matcher.as_deref(), + eid.mm, + eid.ext.as_ref() + ), + (None, None, None, None), + "should expose no EID-level fields when none are stored" + ); + assert_eq!( + (stored.uid(), stored.atype()), + ("u", Some(3)), + "should describe the first UID" + ); + + let empty = KvPartnerId { + eids: Vec::new(), + ..record + }; + assert!( + StoredId::from_record(&source, &empty).is_none(), + "should not read a record with no entry" + ); + } + + #[test] + fn a_full_record_exposes_every_entry_and_its_first_uid() { + let record = KvPartnerId { + w: Some("p".to_owned()), + rv: Some("0123456789abcdef".to_owned()), + at: Some(10), + exp: None, + ra: None, + ru: None, + eids: vec![ + KvEid { + uids: vec![ + KvUid { + id: "first".to_owned(), + atype: Some(1), + ext: Some(serde_json::json!({ "rtiPartner": "TDID" })), + }, + KvUid { + id: "second".to_owned(), + atype: Some(3), + ext: None, + }, + ], + inserter: Some("inserter.example.com".to_owned()), + matcher: None, + mm: Some(2), + ext: Some(serde_json::json!({ "segments": ["a"] })), + }, + KvEid { + uids: vec![KvUid { + id: "third".to_owned(), + atype: None, + ext: None, + }], + inserter: None, + matcher: Some("matcher.example.com".to_owned()), + mm: None, + ext: None, + }, + ], + }; + let source: SourceDomain = "example.com".parse().expect("should parse the source"); + + let stored = StoredId::from_record(&source, &record).expect("should read the record"); + + let ids: Vec> = stored + .eids() + .iter() + .map(|eid| eid.uids.iter().map(|uid| uid.id.as_str()).collect()) + .collect(); + assert_eq!( + ids, + vec![vec!["first", "second"], vec!["third"]], + "should keep every entry in stored order" + ); + assert!( + stored.eids().iter().all(|eid| eid.source == "example.com"), + "should fill in the source" + ); + assert_eq!( + stored.eids()[0].inserter.as_deref(), + Some("inserter.example.com"), + "should keep the inserter" + ); + assert_eq!(stored.eids()[0].mm, Some(2), "should keep mm"); + assert_eq!( + stored.eids()[0].ext, + Some(serde_json::json!({ "segments": ["a"] })), + "should keep the EID-level ext" + ); + assert_eq!( + stored.eids()[1].matcher.as_deref(), + Some("matcher.example.com"), + "should keep the matcher" + ); + assert_eq!( + stored.uid(), + "first", + "should return the first UID of the first entry" + ); + assert_eq!(stored.atype(), Some(1), "should return that UID's atype"); + assert_eq!( + stored.ext(), + Some(&serde_json::json!({ "rtiPartner": "TDID" })), + "should return that UID's ext" + ); + assert!( + !format!("{stored:?}").contains("second"), + "should redact every UID in Debug" + ); + } + + #[test] + fn hashed_email_debug_is_redacted() { + let email = HashedEmail::from_bytes([0xab; 32]); + assert_eq!( + email.to_hex(), + "ab".repeat(32), + "should encode lowercase hex" + ); + assert_eq!( + format!("{email:?}"), + "HashedEmail([redacted])", + "should redact Debug" + ); + } + + fn kinds( + descriptor: &ProviderDescriptor, + kind: ResolveKind, + ) -> BTreeMap { + descriptor + .sources() + .iter() + .map(|source| (source.clone(), kind)) + .collect() + } + + #[derive(Debug, serde::Deserialize)] + struct TypedState { + #[allow(dead_code)] + signature: String, + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 2b7614a374..77656c68a3 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -8,6 +8,7 @@ pub mod descriptor; pub mod ids; +pub mod input; pub mod resolution; pub use descriptor::{ @@ -15,4 +16,5 @@ pub use descriptor::{ ProviderDescriptorBuilder, ResolveKind, Resolves, UsOptOut, }; pub use ids::{IdentityProviderId, SourceDomain, UnixTime}; +pub use input::{HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; pub use resolution::IdentityProviderError; diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index d22b47c1b8..2c4c14698c 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -88,6 +88,7 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { include_str!("ec/identity/descriptor.rs"), ), ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), + ("ec/identity/input.rs", include_str!("ec/identity/input.rs")), ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), ( "ec/identity/resolution.rs", From 6f1c9b1c797ab235d13fe4839b6b62e5f6ca09bf Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 19:58:35 +0530 Subject: [PATCH 18/89] Add provider requests, responses and resolutions Signed-off-by: dhruv8sh --- .../src/ec/identity/exchange.rs | 449 ++++++ .../src/ec/identity/input.rs | 130 +- .../src/ec/identity/mod.rs | 8 +- .../src/ec/identity/resolution.rs | 1207 +++++++++++++++++ .../src/migration_guards.rs | 4 + 5 files changed, 1795 insertions(+), 3 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/exchange.rs diff --git a/crates/trusted-server-core/src/ec/identity/exchange.rs b/crates/trusted-server-core/src/ec/identity/exchange.rs new file mode 100644 index 0000000000..ae493fe0de --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/exchange.rs @@ -0,0 +1,449 @@ +//! What a provider sends and receives. The coordinator does all I/O. + +use core::fmt; + +use error_stack::Report; +use http::header::{AUTHORIZATION, CONTENT_TYPE, COOKIE}; +use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use url::Url; + +use super::resolution::IdentityProviderError; +use crate::redacted::Redacted; + +/// Maximum JSON bytes of a call context. +pub const MAX_CALL_CONTEXT_BYTES: usize = 1024; + +/// One call a provider wants made. +#[derive(Clone)] +pub struct ProviderRequest { + method: Method, + url: Url, + headers: HeaderMap, + body: Vec, + call_context: Option, +} + +impl fmt::Debug for ProviderRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderRequest") + .field("method", &self.method) + .field("host", &self.url.host_str()) + .field("path", &self.url.path()) + .field("headers", &self.headers.keys().collect::>()) + .field("body_len", &self.body.len()) + .finish_non_exhaustive() + } +} + +impl ProviderRequest { + /// Any method, with no body. + #[must_use] + pub fn new(method: Method, url: &Url) -> Self { + Self { + method, + url: url.clone(), + headers: HeaderMap::new(), + body: Vec::new(), + call_context: None, + } + } + + /// A `GET`. + #[must_use] + pub fn get(url: &Url) -> Self { + Self::new(Method::GET, url) + } + + /// A `POST` with a JSON body. + /// + /// # Errors + /// + /// `Config` when `body` does not serialize. + pub fn post_json( + url: &Url, + body: &impl Serialize, + ) -> Result> { + let bytes = serde_json::to_vec(body) + .map_err(|_| IdentityProviderError::config("request body does not serialize"))?; + Self::new(Method::POST, url).with_body("application/json", bytes) + } + + /// A `POST` with a `text/plain` body. + #[must_use] + pub fn post_text(url: &Url, body: &str) -> Self { + let mut request = Self::new(Method::POST, url); + request + .headers + .insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + request.body = body.as_bytes().to_vec(); + request + } + + /// Sets the body and its `Content-Type`. + /// + /// # Errors + /// + /// `Config` when `content_type` is not a valid header value. + pub fn with_body( + mut self, + content_type: &str, + body: Vec, + ) -> Result> { + let value = HeaderValue::from_str(content_type) + .map_err(|_| IdentityProviderError::config("invalid content type"))?; + self.headers.insert(CONTENT_TYPE, value); + self.body = body; + Ok(self) + } + + /// Appends a query parameter, URL-encoded. + #[must_use] + pub fn with_query(mut self, name: &str, value: &str) -> Self { + self.url.query_pairs_mut().append_pair(name, value); + self + } + + /// Sets a header. A `Cookie` header makes the coordinator refuse the call. + #[must_use] + pub fn with_header(mut self, name: HeaderName, value: HeaderValue) -> Self { + self.headers.insert(name, value); + self + } + + /// Sets a header from strings. + /// + /// # Errors + /// + /// `Config` when the name or value is invalid. + pub fn try_with_header( + self, + name: &str, + value: &str, + ) -> Result> { + let name = HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| IdentityProviderError::config("invalid header name"))?; + let value = HeaderValue::from_str(value) + .map_err(|_| IdentityProviderError::config("invalid header value"))?; + Ok(self.with_header(name, value)) + } + + /// Adds `Authorization: Bearer `. + #[must_use] + pub fn with_bearer(mut self, token: &Redacted) -> Self { + if let Ok(mut value) = HeaderValue::from_str(&format!("Bearer {}", token.expose())) { + value.set_sensitive(true); + self.headers.insert(AUTHORIZATION, value); + } + self + } + + /// Data `response` needs from this call. Kept in memory, never sent. + /// + /// # Errors + /// + /// `Config` when it does not serialize or is over 1,024 bytes of JSON. + pub fn try_with_call_context( + mut self, + context: &impl Serialize, + ) -> Result> { + let json = serde_json::to_string(context) + .map_err(|_| IdentityProviderError::config("call context does not serialize"))?; + if json.len() > MAX_CALL_CONTEXT_BYTES { + return Err(IdentityProviderError::config( + "call context exceeds 1024 bytes", + )); + } + self.call_context = Some(json); + Ok(self) + } + + /// The method. + #[must_use] + pub fn method(&self) -> &Method { + &self.method + } + + /// The URL. + #[must_use] + pub fn url(&self) -> &Url { + &self.url + } + + /// The headers. + #[must_use] + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// The body. + #[must_use] + pub fn body(&self) -> &[u8] { + &self.body + } + + /// The call context JSON. + // Temporary: the coordinator, the first non-test caller, arrives in a later task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn call_context_json(&self) -> Option<&str> { + self.call_context.as_deref() + } + + /// Whether the provider set a `Cookie` header. + // Temporary: the coordinator, the first non-test caller, arrives in a later task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn has_cookie_header(&self) -> bool { + self.headers.contains_key(COOKIE) + } +} + +/// A buffered, decoded provider response. +#[derive(Clone)] +pub struct ProviderResponse { + status: StatusCode, + headers: HeaderMap, + body: Vec, + call_context: Option, +} + +impl fmt::Debug for ProviderResponse { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProviderResponse") + .field("status", &self.status) + .field("body_len", &self.body.len()) + .finish_non_exhaustive() + } +} + +impl ProviderResponse { + /// Built by the coordinator after buffering and decoding. + // Temporary: the coordinator, the first non-test caller, arrives in a later task. + #[allow(dead_code)] + #[must_use] + pub(crate) fn new( + status: StatusCode, + headers: HeaderMap, + body: Vec, + call_context: Option, + ) -> Self { + Self { + status, + headers, + body, + call_context, + } + } + + /// A response for tests. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn for_test(status: StatusCode, body: &[u8]) -> Self { + Self::new(status, HeaderMap::new(), body.to_vec(), None) + } + + /// The status. + #[must_use] + pub fn status(&self) -> StatusCode { + self.status + } + + /// A header as a string, if present and visible ASCII. + #[must_use] + pub fn header(&self, name: &str) -> Option<&str> { + self.headers.get(name).and_then(|value| value.to_str().ok()) + } + + /// The decoded body. + #[must_use] + pub fn body(&self) -> &[u8] { + &self.body + } + + /// Parses the body as JSON, whatever the status. + /// + /// # Errors + /// + /// `Malformed` when it does not parse. + pub fn json(&self) -> Result> { + serde_json::from_slice(&self.body).map_err(|_| IdentityProviderError::malformed()) + } + + /// Requires 2xx, then parses JSON. + /// + /// # Errors + /// + /// `Rejected` for a non-2xx status, `Malformed` for a bad body. + pub fn success_json(&self) -> Result> { + self.require_success()?; + self.json() + } + + /// Requires 2xx and a UTF-8 body. + /// + /// # Errors + /// + /// `Rejected` for a non-2xx status, `Malformed` for non-UTF-8. + pub fn success_text(&self) -> Result<&str, Report> { + self.require_success()?; + core::str::from_utf8(&self.body).map_err(|_| IdentityProviderError::malformed()) + } + + /// The context the request carried, decoded. + /// + /// # Errors + /// + /// `Config` when it does not decode as `T`. + pub fn call_context( + &self, + ) -> Result, Report> { + self.call_context + .as_deref() + .map(|json| { + serde_json::from_str(json) + .map_err(|_| IdentityProviderError::config("call context does not decode")) + }) + .transpose() + } + + fn require_success(&self) -> Result<(), Report> { + if self.status.is_success() { + Ok(()) + } else { + Err(IdentityProviderError::rejected(self.status)) + } + } +} + +#[cfg(test)] +mod tests { + use http::{HeaderName, HeaderValue, Method, StatusCode}; + use serde_json::json; + + use super::*; + + fn url() -> Url { + Url::parse("https://api.example.com/v1/fetch").expect("should parse") + } + + #[test] + fn post_json_sets_body_and_content_type() { + let request = ProviderRequest::post_json(&url(), &json!({ "a": 1 })) + .expect("should build a JSON request"); + assert_eq!(request.method(), Method::POST, "should use POST"); + assert_eq!( + request.headers()["content-type"], + "application/json", + "should set the JSON content type" + ); + assert_eq!(request.body(), br#"{"a":1}"#, "should serialize the body"); + } + + #[test] + fn query_bearer_and_headers() { + let request = ProviderRequest::get(&url()) + .with_query("pid", "1 2") + .with_bearer(&Redacted::new("token".to_owned())) + .try_with_header("x-example", "v") + .expect("should accept a valid header"); + assert_eq!( + request.url().as_str(), + "https://api.example.com/v1/fetch?pid=1+2", + "should encode the query" + ); + assert_eq!( + request.headers()["authorization"], + "Bearer token", + "should set the bearer token" + ); + assert_eq!(request.headers()["x-example"], "v", "should set the header"); + let _ = ProviderRequest::get(&url()) + .try_with_header("bad header", "v") + .expect_err("should reject a bad name"); + let _ = ProviderRequest::get(&url()) + .try_with_header("x", "bad\nvalue") + .expect_err("should reject a bad value"); + } + + #[test] + fn cookie_headers_are_flagged_from_either_helper() { + let typed = ProviderRequest::get(&url()).with_header( + HeaderName::from_static("cookie"), + HeaderValue::from_static("a=1"), + ); + let dynamic = ProviderRequest::get(&url()) + .try_with_header("Cookie", "a=1") + .expect("should accept a Cookie header from strings"); + assert!( + typed.has_cookie_header(), + "should flag a Cookie header from the typed helper" + ); + assert!( + dynamic.has_cookie_header(), + "should flag a Cookie header from the string helper" + ); + assert!( + !ProviderRequest::get(&url()).has_cookie_header(), + "should not flag a request without one" + ); + } + + #[test] + fn call_context_round_trips_and_is_capped() { + let request = ProviderRequest::get(&url()) + .try_with_call_context(&json!({ "nonce": [1, 2] })) + .expect("should accept a small call context"); + let response = ProviderResponse::new( + StatusCode::OK, + http::HeaderMap::new(), + Vec::new(), + request.call_context_json().map(str::to_owned), + ); + let context: serde_json::Value = response + .call_context() + .expect("should decode the call context") + .expect("should carry the call context"); + assert_eq!(context, json!({ "nonce": [1, 2] }), "should round-trip"); + + let error = ProviderRequest::get(&url()) + .try_with_call_context(&"x".repeat(MAX_CALL_CONTEXT_BYTES)) + .expect_err("should reject an oversized call context"); + assert!( + matches!( + error.current_context(), + IdentityProviderError::Config { .. } + ), + "should report an oversized call context as Config" + ); + } + + #[test] + fn success_helpers_check_status_and_encoding() { + let ok = ProviderResponse::for_test(StatusCode::OK, br#"{"uid":"u"}"#); + let value: serde_json::Value = ok.success_json().expect("should parse a 2xx JSON body"); + assert_eq!(value["uid"], "u", "should read the uid"); + + let rejected = ProviderResponse::for_test(StatusCode::FORBIDDEN, br#"{}"#); + let error = rejected + .success_json::() + .expect_err("should reject a non-2xx status"); + assert!( + matches!(error.current_context(), IdentityProviderError::Rejected { status } if *status == StatusCode::FORBIDDEN), + "should report Rejected with the status" + ); + rejected + .json::() + .expect("should parse JSON whatever the status"); + + let binary = ProviderResponse::for_test(StatusCode::OK, &[0xff, 0xfe]); + let error = binary.success_text().expect_err("should reject non-UTF-8"); + assert!( + matches!(error.current_context(), IdentityProviderError::Malformed), + "should report non-UTF-8 as Malformed" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs index 63a833711d..79f944d7dc 100644 --- a/crates/trusted-server-core/src/ec/identity/input.rs +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -17,6 +17,7 @@ use serde::de::DeserializeOwned; use url::Url; use super::descriptor::{Access, Input, ProviderDescriptor, ResolveKind, Resolves}; +use super::exchange::ProviderResponse; use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; use super::resolution::IdentityProviderError; use crate::consent::ConsentContext; @@ -615,13 +616,108 @@ impl<'a> ResolveInput<'a> { } } +/// What a capture may read from one proxied response. +pub struct CaptureInput<'a> { + descriptor: &'a ProviderDescriptor, + path: &'a str, + response: &'a ProviderResponse, + stored: &'a BTreeMap, + consent: &'a ProviderConsent, + publisher_domain: &'a str, + now: UnixTime, +} + +impl fmt::Debug for CaptureInput<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CaptureInput") + .field("provider", self.descriptor.id()) + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl<'a> CaptureInput<'a> { + /// The input for one capture. + #[must_use] + pub(crate) fn new( + descriptor: &'a ProviderDescriptor, + path: &'a str, + response: &'a ProviderResponse, + stored: &'a BTreeMap, + consent: &'a ProviderConsent, + publisher_domain: &'a str, + now: UnixTime, + ) -> Self { + Self { + descriptor, + path, + response, + stored, + consent, + publisher_domain, + now, + } + } + + /// The proxied request path, without query string. + #[must_use] + pub fn path(&self) -> &str { + self.path + } + + /// The proxied response, decoded. + #[must_use] + pub fn response(&self) -> &ProviderResponse { + self.response + } + + /// The descriptor's primary source. + #[must_use] + pub fn primary_source(&self) -> &SourceDomain { + self.descriptor.primary_source() + } + + /// Finds a declared source by a vendor's spelling. + #[must_use] + pub fn source_named(&self, name: &str) -> Option<&SourceDomain> { + self.descriptor.source_named(name) + } + + /// The stored record for a source. + #[must_use] + pub fn stored(&self, source: &SourceDomain) -> Option<&StoredId> { + self.stored.get(source) + } + + /// The request's consent, from its cookies and headers. The request + /// body is never read (spec 4.7). + #[must_use] + pub fn consent(&self) -> &ProviderConsent { + self.consent + } + + /// The publisher's domain. + #[must_use] + pub fn publisher_domain(&self) -> &str { + self.publisher_domain + } + + /// The request's time. + #[must_use] + pub fn now(&self) -> UnixTime { + self.now + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::net::IpAddr; use super::*; - use crate::ec::identity::descriptor::{ProviderDescriptor, Resolves}; + use crate::ec::identity::descriptor::{Captures, ProviderDescriptor, Resolves}; + use crate::ec::identity::exchange::ProviderResponse; use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; fn descriptor(resolves: Resolves) -> ProviderDescriptor { @@ -1014,6 +1110,38 @@ mod tests { ); } + #[test] + fn capture_input_exposes_path_and_never_the_request_body() { + let descriptor = + ProviderDescriptor::builder("c".parse().expect("should parse the provider id")) + .with_source("example.com".parse().expect("should parse the source")) + .with_captures(Captures::new().with_path(http::Method::POST, "/integrations/c/api")) + .build() + .expect("should build"); + let response = ProviderResponse::for_test(http::StatusCode::OK, b"{}"); + let consent = ProviderConsent::default(); + let stored = BTreeMap::new(); + let input = CaptureInput::new( + &descriptor, + "/integrations/c/api", + &response, + &stored, + &consent, + "www.example.com", + UnixTime::from_secs(1), + ); + assert_eq!( + input.path(), + "/integrations/c/api", + "should expose the proxied path" + ); + assert_eq!( + input.primary_source().as_str(), + "example.com", + "should expose the primary source" + ); + } + fn kinds( descriptor: &ProviderDescriptor, kind: ResolveKind, diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 77656c68a3..0d79d61239 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -7,6 +7,7 @@ //! write. See `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. pub mod descriptor; +pub mod exchange; pub mod ids; pub mod input; pub mod resolution; @@ -15,6 +16,9 @@ pub use descriptor::{ Access, Captures, ConsentPolicy, EidProvenance, Input, ProviderDescriptor, ProviderDescriptorBuilder, ResolveKind, Resolves, UsOptOut, }; +pub use exchange::{ProviderRequest, ProviderResponse}; pub use ids::{IdentityProviderId, SourceDomain, UnixTime}; -pub use input::{HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; -pub use resolution::IdentityProviderError; +pub use input::{CaptureInput, HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; +pub use resolution::{ + IdentityProviderError, Resolution, ResolvedEid, ResolvedUid, StateChange, When, +}; diff --git a/crates/trusted-server-core/src/ec/identity/resolution.rs b/crates/trusted-server-core/src/ec/identity/resolution.rs index b49509fabd..d5fa6d89dc 100644 --- a/crates/trusted-server-core/src/ec/identity/resolution.rs +++ b/crates/trusted-server-core/src/ec/identity/resolution.rs @@ -1,9 +1,24 @@ //! Provider results and errors. +// Temporary: the coordinator, the first non-test caller, arrives in a later +// task. Remove this when it lands. +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::iter; +use std::time::Duration; + use error_stack::Report; use http::StatusCode; +use serde::Serialize; use super::descriptor::{Access, Input}; +use super::ids::{SourceDomain, UnixTime}; +use crate::ec::kv_types::{ + KvEid, KvUid, MAX_ENTRIES_PER_SOURCE, MAX_ENTRY_SET_BYTES, MAX_PARTNER_EXT_BYTES, + MAX_PROVIDER_STATE_BYTES, MAX_UID_LENGTH, MAX_UIDS_PER_SOURCE, entry_set_bytes, + is_valid_eid_field, is_valid_partner_ext, +}; /// Marker message of [`IdentityProviderError::not_implemented`]. The harness /// matches on it to tell a missing override from a real failure. @@ -83,3 +98,1195 @@ impl IdentityProviderError { matches!(self, Self::Config { message } if message.starts_with(NOT_IMPLEMENTED_MARKER)) } } + +/// A time as a provider gave it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum When { + /// Relative to the request's `now`. + In(Duration), + /// Absolute. + At(UnixTime), +} + +impl When { + /// Resolves against `now`. + #[must_use] + // Temporary: the coordinator, the first non-test caller, arrives in a later task. + #[allow(dead_code)] + pub(crate) fn resolve(self, now: UnixTime) -> UnixTime { + match self { + Self::In(delay) => now + delay, + Self::At(at) => at, + } + } +} + +fn ext_fits(ext: &serde_json::Value) -> bool { + serde_json::to_vec(ext).is_ok_and(|bytes| bytes.len() <= MAX_PARTNER_EXT_BYTES) +} + +/// One UID within a [`ResolvedEid`]. +#[derive(Clone, PartialEq)] +pub struct ResolvedUid { + id: String, + atype: Option, + ext: Option, +} + +impl core::fmt::Debug for ResolvedUid { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("ResolvedUid") + .field("id", &"[redacted]") + .field("atype", &self.atype) + .finish_non_exhaustive() + } +} + +impl ResolvedUid { + /// A UID with no `atype` or ext. + #[must_use] + pub fn new(id: impl Into) -> Self { + Self { + id: id.into(), + atype: None, + ext: None, + } + } + + /// Sets `atype`. + #[must_use] + pub fn with_atype(mut self, atype: i32) -> Self { + self.atype = Some(atype); + self + } + + /// UID-level ext. Over 2,048 bytes as compact JSON it is dropped and the + /// UID kept. + #[must_use] + pub fn with_ext(mut self, ext: serde_json::Value) -> Self { + let fits = ext_fits(&ext); + if !fits { + log::debug!("Dropping an oversized UID-level ext"); + } + self.ext = fits.then_some(ext); + self + } + + /// The UID. + #[must_use] + pub fn id(&self) -> &str { + &self.id + } + + /// `atype`. + #[must_use] + pub fn atype(&self) -> Option { + self.atype + } + + /// UID-level ext. + #[must_use] + pub fn ext(&self) -> Option<&serde_json::Value> { + self.ext.as_ref() + } +} + +/// One `OpenRTB` EID entry a provider returns: a source, its UIDs and its +/// EID-level fields. +#[derive(Clone, PartialEq)] +pub struct ResolvedEid { + source: SourceDomain, + uids: Vec, + inserter: Option, + matcher: Option, + mm: Option, + ext: Option, + expiry: Option, + refresh: Option, + refresh_until: Option, +} + +impl core::fmt::Debug for ResolvedEid { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("ResolvedEid") + .field("source", &self.source) + .field("uids", &self.uids) + .field("mm", &self.mm) + .field("expiry", &self.expiry) + .finish_non_exhaustive() + } +} + +impl ResolvedEid { + /// An entry with one UID, the common case. + #[must_use] + pub fn new(source: &SourceDomain, uid: ResolvedUid) -> Self { + Self { + source: source.clone(), + uids: vec![uid], + inserter: None, + matcher: None, + mm: None, + ext: None, + expiry: None, + refresh: None, + refresh_until: None, + } + } + + /// Adds another UID to this entry. Repeatable. + #[must_use] + pub fn with_uid(mut self, uid: ResolvedUid) -> Self { + self.uids.push(uid); + self + } + + /// Sets `inserter`. An entry that sets any of `inserter`, `matcher` and + /// `mm` states its EID-level fields completely, so the descriptor's + /// `with_provenance` default no longer applies to it (spec 4.3). + #[must_use] + pub fn with_inserter(mut self, inserter: &str) -> Self { + self.inserter = Some(inserter.to_owned()); + self + } + + /// Sets `matcher`. As [`Self::with_inserter`], it turns off the + /// descriptor's default for this entry. + #[must_use] + pub fn with_matcher(mut self, matcher: &str) -> Self { + self.matcher = Some(matcher.to_owned()); + self + } + + /// Sets `mm`. As [`Self::with_inserter`], it turns off the descriptor's + /// default for this entry. + #[must_use] + pub fn with_mm(mut self, mm: i32) -> Self { + self.mm = Some(mm); + self + } + + /// EID-level ext. Over 2,048 bytes as compact JSON it is dropped and the + /// entry kept. + #[must_use] + pub fn with_ext(mut self, ext: serde_json::Value) -> Self { + let fits = ext_fits(&ext); + if !fits { + log::debug!( + "Dropping an oversized EID-level ext for source {}", + self.source + ); + } + self.ext = fits.then_some(ext); + self + } + + /// Expires after `delay`. Timing applies to the source's whole entry + /// set, and the earliest time across its entries wins. + #[must_use] + pub fn with_expires_in(mut self, delay: Duration) -> Self { + self.expiry = Some(When::In(delay)); + self + } + + /// Expires at `at`. + #[must_use] + pub fn with_expires_at(mut self, at: UnixTime) -> Self { + self.expiry = Some(When::At(at)); + self + } + + /// Due for refresh after `delay`. + #[must_use] + pub fn with_refresh_in(mut self, delay: Duration) -> Self { + self.refresh = Some(When::In(delay)); + self + } + + /// Due for refresh at `at`. + #[must_use] + pub fn with_refresh_at(mut self, at: UnixTime) -> Self { + self.refresh = Some(When::At(at)); + self + } + + /// When this ID stops being refreshable. Only for a provider declared + /// `with_private_state()`; the coordinator ignores it otherwise. + #[must_use] + pub fn with_refresh_until(mut self, at: UnixTime) -> Self { + self.refresh_until = Some(at); + self + } + + /// The source. + #[must_use] + pub fn source(&self) -> &SourceDomain { + &self.source + } + + /// The UIDs, in order. + #[must_use] + pub fn uids(&self) -> &[ResolvedUid] { + &self.uids + } + + /// `inserter`, if set. + #[must_use] + pub fn inserter(&self) -> Option<&str> { + self.inserter.as_deref() + } + + /// `matcher`, if set. + #[must_use] + pub fn matcher(&self) -> Option<&str> { + self.matcher.as_deref() + } + + /// `mm`, if set. + #[must_use] + pub fn mm(&self) -> Option { + self.mm + } + + /// EID-level ext. + #[must_use] + pub fn ext(&self) -> Option<&serde_json::Value> { + self.ext.as_ref() + } + + /// The expiry as given. + #[must_use] + pub fn expiry(&self) -> Option { + self.expiry + } + + /// The refresh hint as given. + #[must_use] + pub fn refresh(&self) -> Option { + self.refresh + } + + /// The refresh limit. + #[must_use] + pub fn refresh_until(&self) -> Option { + self.refresh_until + } +} + +/// How a result changes the provider's private state. +#[derive(Debug, PartialEq, Eq)] +pub enum StateChange<'a> { + /// Leave it. + Keep, + /// Replace it with this JSON. + Replace(&'a str), + /// Remove it. + Clear, +} + +#[derive(Clone, PartialEq)] +enum StateOp { + Keep, + Replace(String), + Clear, + Rejected, +} + +impl core::fmt::Debug for StateOp { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Keep => formatter.write_str("Keep"), + Self::Replace(_) => formatter.write_str("Replace([redacted])"), + Self::Clear => formatter.write_str("Clear"), + Self::Rejected => formatter.write_str("Rejected"), + } + } +} + +/// What one provider call produced. Its `Debug` prints entries through +/// [`ResolvedEid`]'s redacting `Debug` and never prints provider state. +#[derive(Debug, Clone, PartialEq)] +pub struct Resolution { + eids: Vec, + invalidated: Vec, + opted_out: bool, + retry_in: Option, + state: StateOp, +} + +impl Resolution { + fn empty() -> Self { + Self { + eids: Vec::new(), + invalidated: Vec::new(), + opted_out: false, + retry_in: None, + state: StateOp::Keep, + } + } + + /// One or more EID entries. A source may appear in several entries, in + /// the order bidders should see them. The entries of one source form its + /// new entry set, which replaces the stored set whole for an owned + /// source. For an unowned source the set only fills an absent or expired + /// record (spec 7.2). A source the result does not name, or whose set + /// the limits discard, is left as it is. An empty set is `no_match`. + #[must_use] + pub fn found(eids: impl IntoIterator) -> Self { + Self { + eids: eids.into_iter().collect(), + ..Self::empty() + } + } + + /// Nothing recognized, or a capture held nothing. + #[must_use] + pub fn no_match() -> Self { + Self::empty() + } + + /// The stored ID for `source` is no longer valid. + #[must_use] + pub fn invalidate(source: &SourceDomain) -> Self { + Self { + invalidated: vec![source.clone()], + ..Self::empty() + } + } + + /// The provider reports the user opted out. + #[must_use] + pub fn opted_out() -> Self { + Self { + opted_out: true, + ..Self::empty() + } + } + + /// Also invalidates `source`. + #[must_use] + pub fn also_invalidate(mut self, source: &SourceDomain) -> Self { + if !self.invalidated.contains(source) { + self.invalidated.push(source.clone()); + } + self + } + + /// The earliest the coordinator may call again. + #[must_use] + pub fn with_retry_in(mut self, delay: Duration) -> Self { + self.retry_in = Some(delay); + self + } + + /// Replaces the provider's private state. Over 1,024 bytes of JSON, or + /// not serializable, the positive part of the result is rejected whole. + #[must_use] + pub fn replacing_state(mut self, state: &impl Serialize) -> Self { + self.state = match serde_json::to_string(state) { + Ok(json) if json.len() <= MAX_PROVIDER_STATE_BYTES => StateOp::Replace(json), + _ => StateOp::Rejected, + }; + self + } + + /// Clears the provider's private state. + #[must_use] + pub fn clearing_state(mut self) -> Self { + self.state = StateOp::Clear; + self + } + + /// The returned entries, as given. + #[must_use] + pub fn eids(&self) -> &[ResolvedEid] { + &self.eids + } + + /// Invalidated sources. + #[must_use] + pub fn invalidated(&self) -> &[SourceDomain] { + &self.invalidated + } + + /// Whether nothing was found, invalidated or opted out. + #[must_use] + pub fn is_no_match(&self) -> bool { + self.eids.is_empty() && self.invalidated.is_empty() && !self.opted_out + } + + /// Whether the provider reported an opt-out. + #[must_use] + pub fn is_opted_out(&self) -> bool { + self.opted_out + } + + /// The retry delay. + #[must_use] + pub fn retry_in(&self) -> Option { + self.retry_in + } + + /// The state change. + #[must_use] + pub fn state_change(&self) -> StateChange<'_> { + match &self.state { + StateOp::Keep | StateOp::Rejected => StateChange::Keep, + StateOp::Replace(json) => StateChange::Replace(json), + StateOp::Clear => StateChange::Clear, + } + } + + /// Whether `replacing_state` was given an oversized or unserializable + /// value. The coordinator then rejects the positive part (spec 4.5, 8). + #[must_use] + pub(crate) fn state_rejected(&self) -> bool { + self.state == StateOp::Rejected + } + + /// Whether this result changes state at all. + #[must_use] + pub(crate) fn changes_state(&self) -> bool { + self.state != StateOp::Keep + } +} + +/// The part of a result that a spec 4.5 limit dropped. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub(crate) enum DroppedPart { + /// A UID that is empty, blank or over `MAX_UID_LENGTH`. + #[display("uid")] + Uid, + /// A negative `atype`. + #[display("atype")] + Atype, + /// A UID-level ext that is not an object or is oversized. + #[display("uid ext")] + UidExt, + /// A blank or oversized `inserter`. + #[display("inserter")] + Inserter, + /// A blank or oversized `matcher`. + #[display("matcher")] + Matcher, + /// A negative `mm`. + #[display("mm")] + Mm, + /// An EID-level ext that is not an object or is oversized. + #[display("eid ext")] + EidExt, + /// An entry left with no UID. + #[display("entry without uid")] + EmptyEntry, + /// An entry equal to an earlier kept entry of its source. + #[display("duplicate entry")] + DuplicateEntry, + /// An entry past the source's first 8, or with no room left for a UID. + #[display("entry over the limit")] + ExtraEntry, + /// A UID past the source's first 32. + #[display("uid over the limit")] + ExtraUid, +} + +/// One dropped part, for the coordinator's log line (provider, source and +/// part only, never a value). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct EntryDrop { + pub(crate) source: SourceDomain, + pub(crate) part: DroppedPart, +} + +/// One source's new entry set, as it is stored, and its timing resolved +/// against `now`. +#[derive(Clone, PartialEq)] +pub(crate) struct EntrySet { + pub(crate) entries: Vec, + pub(crate) expires_at: Option, + pub(crate) refresh_at: Option, + pub(crate) refresh_until: Option, +} + +impl core::fmt::Debug for EntrySet { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("EntrySet") + .field("entries", &self.entries.len()) + .field("expires_at", &self.expires_at) + .field("refresh_at", &self.refresh_at) + .field("refresh_until", &self.refresh_until) + .finish() + } +} + +/// A result's entries, grouped by source and checked against spec 4.5. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct EntrySets { + /// Each source with at least one kept entry, under every limit. + pub(crate) sets: BTreeMap, + /// What the limits dropped, in order. + pub(crate) drops: Vec, + /// Sources whose set was over `MAX_ENTRY_SET_BYTES` and was discarded + /// whole. A stateful provider's positive part is then rejected. + pub(crate) oversized: Vec, +} + +impl EntrySets { + /// Whether the result named at least one source and every set was + /// discarded. Every entry that is not kept records a drop, so a result + /// with entries and no kept set always has a drop or an oversized set. + /// The coordinator handles such a result as `Malformed` (spec 4.5). + #[must_use] + pub(crate) fn all_discarded(&self) -> bool { + self.sets.is_empty() && (!self.drops.is_empty() || !self.oversized.is_empty()) + } +} + +fn earliest(current: Option, candidate: Option) -> Option { + match (current, candidate) { + (Some(current), Some(candidate)) => Some(current.min(candidate)), + (current, candidate) => current.or(candidate), + } +} + +fn clean_text( + value: Option<&str>, + part: DroppedPart, + dropped: &mut Vec, +) -> Option { + let value = value?; + if !is_valid_eid_field(value) { + dropped.push(part); + return None; + } + Some(value.to_owned()) +} + +fn clean_number( + value: Option, + part: DroppedPart, + dropped: &mut Vec, +) -> Option { + let value = value?; + if value < 0 { + dropped.push(part); + return None; + } + Some(value) +} + +fn clean_ext( + value: Option<&serde_json::Value>, + part: DroppedPart, + dropped: &mut Vec, +) -> Option { + let value = value?; + if !is_valid_partner_ext(value) { + dropped.push(part); + return None; + } + Some(value.clone()) +} + +fn clean_uid(uid: &ResolvedUid, dropped: &mut Vec) -> Option { + if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + dropped.push(DroppedPart::Uid); + return None; + } + Some(KvUid { + id: uid.id.clone(), + atype: clean_number(uid.atype, DroppedPart::Atype, dropped), + ext: clean_ext(uid.ext.as_ref(), DroppedPart::UidExt, dropped), + }) +} + +fn clean_entry(eid: &ResolvedEid, dropped: &mut Vec) -> Option { + let uids: Vec = eid + .uids + .iter() + .filter_map(|uid| clean_uid(uid, dropped)) + .collect(); + if uids.is_empty() { + return None; + } + Some(KvEid { + uids, + inserter: clean_text(eid.inserter.as_deref(), DroppedPart::Inserter, dropped), + matcher: clean_text(eid.matcher.as_deref(), DroppedPart::Matcher, dropped), + mm: clean_number(eid.mm, DroppedPart::Mm, dropped), + ext: clean_ext(eid.ext.as_ref(), DroppedPart::EidExt, dropped), + }) +} + +/// Groups `eids` by source, in order, and applies every limit of spec 4.5: +/// the browser parser's field and UID rules, then per-source caps the +/// browser parser does not have: duplicates, the first 8 entries and 32 UIDs +/// of a source, and `MAX_ENTRY_SET_BYTES` as [`entry_set_bytes`] measures +/// the stored set. A set over that is discarded whole, never trimmed: field +/// caps do not guarantee the set cap. Each set's `exp`, `ra` and `ru` are +/// the earliest its kept entries set; a dropped or discarded entry's times +/// do not count. +/// +/// Source claims and operator narrowing are the coordinator's, which drops +/// a set for an unclaimed source before using the result. +/// +/// # Performance +/// +/// Serializes each set once to measure it, and compares each entry with +/// the at most 8 entries already kept. +#[must_use] +pub(crate) fn entry_sets(eids: &[ResolvedEid], now: UnixTime) -> EntrySets { + let mut result = EntrySets::default(); + for eid in eids { + let mut dropped = Vec::new(); + let cleaned = clean_entry(eid, &mut dropped); + let set = result + .sets + .entry(eid.source.clone()) + .or_insert_with(|| EntrySet { + entries: Vec::new(), + expires_at: None, + refresh_at: None, + refresh_until: None, + }); + match cleaned { + None => dropped.push(DroppedPart::EmptyEntry), + Some(entry) if set.entries.contains(&entry) => { + dropped.push(DroppedPart::DuplicateEntry) + } + Some(_) if set.entries.len() >= MAX_ENTRIES_PER_SOURCE => { + dropped.push(DroppedPart::ExtraEntry) + } + Some(mut entry) => { + let used: usize = set.entries.iter().map(|kept| kept.uids.len()).sum(); + let room = MAX_UIDS_PER_SOURCE.saturating_sub(used); + if entry.uids.len() > room { + dropped.extend(iter::repeat_n( + DroppedPart::ExtraUid, + entry.uids.len() - room, + )); + entry.uids.truncate(room); + } + if entry.uids.is_empty() { + dropped.push(DroppedPart::ExtraEntry); + } else { + set.expires_at = + earliest(set.expires_at, eid.expiry.map(|when| when.resolve(now))); + set.refresh_at = + earliest(set.refresh_at, eid.refresh.map(|when| when.resolve(now))); + set.refresh_until = earliest(set.refresh_until, eid.refresh_until); + set.entries.push(entry); + } + } + } + result + .drops + .extend(dropped.into_iter().map(|part| EntryDrop { + source: eid.source.clone(), + part, + })); + } + result.sets.retain(|source, set| { + if set.entries.is_empty() { + return false; + } + let fits = entry_set_bytes(&set.entries) <= MAX_ENTRY_SET_BYTES; + if !fits { + result.oversized.push(source.clone()); + } + fits + }); + result +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use serde_json::json; + + use super::*; + use crate::ec::kv_types::MAX_EID_FIELD_BYTES; + + fn source(name: &str) -> SourceDomain { + name.parse().expect("should parse the source") + } + + fn uid(id: &str) -> ResolvedUid { + ResolvedUid::new(id) + } + + fn kv_uid(id: &str) -> KvUid { + KvUid { + id: id.to_owned(), + atype: None, + ext: None, + } + } + + fn now() -> UnixTime { + UnixTime::from_secs(1_000) + } + + fn parts(sets: &EntrySets) -> Vec { + sets.drops.iter().map(|drop| drop.part).collect() + } + + #[test] + fn empty_found_is_no_match() { + let resolution = Resolution::found([]); + assert!(resolution.is_no_match(), "should be no_match"); + assert!(resolution.eids().is_empty(), "should hold no entries"); + } + + #[test] + fn a_one_uid_entry_becomes_a_one_entry_set_without_eid_level_fields() { + let a = source("a.example.com"); + let resolution = Resolution::found([ResolvedEid::new(&a, uid("u").with_atype(3))]); + + let sets = entry_sets(resolution.eids(), now()); + + assert_eq!( + sets.sets[&a].entries, + vec![KvEid { + uids: vec![KvUid { + id: "u".to_owned(), + atype: Some(3), + ext: None + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + }], + "should produce the one-UID shape browser ingest stores" + ); + assert!( + sets.drops.is_empty() && sets.oversized.is_empty(), + "should drop nothing" + ); + } + + #[test] + fn entries_group_by_source_and_keep_every_uid_and_field_in_order() { + let a = source("a.example.com"); + let b = source("b.example.com"); + let first = ResolvedEid::new( + &a, + uid("a1") + .with_atype(1) + .with_ext(json!({ "rtiPartner": "TDID" })), + ) + .with_uid(uid("a2")) + .with_inserter("inserter.example.com") + .with_matcher("matcher.example.com") + .with_mm(2) + .with_ext(json!({ "segments": ["s"] })); + let other = ResolvedEid::new(&b, uid("b1")); + let second = ResolvedEid::new(&a, uid("a3")); + + let sets = entry_sets(&[first, other, second], now()); + + assert_eq!( + sets.sets[&a].entries, + vec![ + KvEid { + uids: vec![ + KvUid { + id: "a1".to_owned(), + atype: Some(1), + ext: Some(json!({ "rtiPartner": "TDID" })) + }, + kv_uid("a2"), + ], + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(2), + ext: Some(json!({ "segments": ["s"] })), + }, + KvEid { + uids: vec![kv_uid("a3")], + inserter: None, + matcher: None, + mm: None, + ext: None + }, + ], + "should keep a's entries in the order returned" + ); + assert_eq!(sets.sets[&b].entries.len(), 1, "should group b on its own"); + assert!(sets.drops.is_empty(), "should drop nothing"); + } + + #[test] + fn bad_fields_are_dropped_and_their_entries_kept() { + let a = source("a.example.com"); + let entry = ResolvedEid::new( + &a, + uid("u") + .with_atype(-1) + .with_ext(json!(["not", "an", "object"])), + ) + .with_inserter(" ") + .with_matcher(&"m".repeat(MAX_EID_FIELD_BYTES + 1)) + .with_mm(-2) + .with_ext(json!("not an object")); + + let sets = entry_sets(&[entry], now()); + + assert_eq!( + sets.sets[&a].entries, + vec![KvEid { + uids: vec![kv_uid("u")], + inserter: None, + matcher: None, + mm: None, + ext: None + }], + "should keep the entry without its bad fields" + ); + assert_eq!( + parts(&sets), + vec![ + DroppedPart::Atype, + DroppedPart::UidExt, + DroppedPart::Inserter, + DroppedPart::Matcher, + DroppedPart::Mm, + DroppedPart::EidExt, + ], + "should name each dropped field" + ); + } + + #[test] + fn bad_uids_are_dropped_and_an_entry_without_uids_is_discarded() { + let a = source("a.example.com"); + let mixed = ResolvedEid::new(&a, uid(" ")).with_uid(uid("kept")); + let empty = ResolvedEid::new(&a, uid(&"x".repeat(MAX_UID_LENGTH + 1))) + .with_inserter("inserter.example.com"); + + let sets = entry_sets(&[mixed, empty], now()); + + assert_eq!( + sets.sets[&a].entries.len(), + 1, + "should discard the entry left with no UID" + ); + assert_eq!( + sets.sets[&a].entries[0].uids, + vec![kv_uid("kept")], + "should keep the valid UID" + ); + assert_eq!( + parts(&sets), + vec![DroppedPart::Uid, DroppedPart::Uid, DroppedPart::EmptyEntry], + "should name the dropped UIDs and the discarded entry" + ); + } + + #[test] + fn a_source_with_no_kept_entry_is_not_named() { + let a = source("a.example.com"); + + let sets = entry_sets(&[ResolvedEid::new(&a, uid(""))], now()); + + assert!( + !sets.sets.contains_key(&a), + "should leave the stored set as it is" + ); + assert_eq!( + parts(&sets), + vec![DroppedPart::Uid, DroppedPart::EmptyEntry], + "should record the dropped UID and the discarded entry" + ); + assert!( + sets.all_discarded(), + "should report that every named set was discarded" + ); + } + + #[test] + fn all_discarded_only_when_nothing_was_kept() { + let a = source("a.example.com"); + let b = source("b.example.com"); + + let mixed = entry_sets( + &[ + ResolvedEid::new(&a, uid("")), + ResolvedEid::new(&b, uid("kept")), + ], + now(), + ); + let empty = entry_sets(&[], now()); + + assert!( + !mixed.all_discarded(), + "should not report a result that kept one set" + ); + assert!( + !empty.all_discarded(), + "should not report a result that named nothing" + ); + } + + #[test] + fn duplicates_and_entries_and_uids_over_the_limits_are_dropped() { + let a = source("a.example.com"); + let mut eids = vec![ + ResolvedEid::new(&a, uid("e0")), + ResolvedEid::new(&a, uid("e0")), + ]; + // 7 more entries of 4 UIDs: 1 + 28 = 29 UIDs, then a ninth entry. + for entry in 1..=7 { + let mut eid = ResolvedEid::new(&a, uid(&format!("e{entry}u0"))); + for index in 1..4 { + eid = eid.with_uid(uid(&format!("e{entry}u{index}"))); + } + eids.push(eid); + } + eids.push(ResolvedEid::new(&a, uid("ninth"))); + + let sets = entry_sets(&eids, now()); + + let set = &sets.sets[&a]; + assert_eq!( + set.entries.len(), + MAX_ENTRIES_PER_SOURCE, + "should keep 8 entries" + ); + let uid_count: usize = set.entries.iter().map(|entry| entry.uids.len()).sum(); + assert_eq!( + uid_count, 29, + "should stay under the UID cap in this control" + ); + assert_eq!( + parts(&sets), + vec![DroppedPart::DuplicateEntry, DroppedPart::ExtraEntry], + "should drop the duplicate and the ninth entry" + ); + + let mut wide = ResolvedEid::new(&a, uid("u0")); + for index in 1..=MAX_UIDS_PER_SOURCE { + wide = wide.with_uid(uid(&format!("u{index}"))); + } + let sets = entry_sets(&[wide, ResolvedEid::new(&a, uid("later"))], now()); + assert_eq!( + sets.sets[&a].entries.len(), + 1, + "should discard the entry left with no room" + ); + assert_eq!( + sets.sets[&a].entries[0].uids.len(), + MAX_UIDS_PER_SOURCE, + "should keep 32 UIDs" + ); + assert_eq!( + parts(&sets), + vec![ + DroppedPart::ExtraUid, + DroppedPart::ExtraUid, + DroppedPart::ExtraEntry + ], + "should drop the 33rd UID and the entry with no room" + ); + } + + #[test] + fn an_oversized_set_is_discarded_whole_and_others_are_kept() { + let a = source("a.example.com"); + let b = source("b.example.com"); + + let sets = entry_sets( + &[oversized_entry(&a), ResolvedEid::new(&b, uid("small"))], + now(), + ); + + assert!( + !sets.sets.contains_key(&a), + "should never trim an oversized set" + ); + assert_eq!( + sets.oversized, + vec![a.clone()], + "should report the oversized source" + ); + assert!(sets.sets.contains_key(&b), "should keep the other source"); + assert!( + !sets.all_discarded(), + "should not report the result as all discarded" + ); + + let only = entry_sets(&[oversized_entry(&a)], now()); + assert!( + only.all_discarded(), + "should report a result whose only set is oversized" + ); + } + + fn oversized_entry(source: &SourceDomain) -> ResolvedEid { + let mut big = ResolvedEid::new(source, uid(&"x".repeat(500))); + for index in 0..8 { + big = big.with_uid(uid(&format!("{index}{}", "x".repeat(499)))); + } + big + } + + #[test] + fn a_set_takes_the_earliest_times_its_entries_set() { + let a = source("a.example.com"); + let first = ResolvedEid::new(&a, uid("one")) + .with_expires_in(Duration::from_secs(100)) + .with_refresh_at(UnixTime::from_secs(1_080)) + .with_refresh_until(UnixTime::from_secs(5_000)); + let second = ResolvedEid::new(&a, uid("two")) + .with_expires_at(UnixTime::from_secs(1_050)) + .with_refresh_in(Duration::from_secs(90)) + .with_refresh_until(UnixTime::from_secs(4_000)); + let third = ResolvedEid::new(&a, uid("three")); + + let set = entry_sets(&[first, second, third], now()) + .sets + .remove(&a) + .expect("should keep a"); + + assert_eq!( + set.expires_at, + Some(UnixTime::from_secs(1_050)), + "should take the earliest expiry" + ); + assert_eq!( + set.refresh_at, + Some(UnixTime::from_secs(1_080)), + "should take the earliest refresh" + ); + assert_eq!( + set.refresh_until, + Some(UnixTime::from_secs(4_000)), + "should take the earliest refresh limit" + ); + } + + #[test] + fn dropped_entries_do_not_count_toward_timing() { + let a = source("a.example.com"); + let kept = ResolvedEid::new(&a, uid("one")).with_expires_at(UnixTime::from_secs(2_000)); + let duplicate = + ResolvedEid::new(&a, uid("one")).with_expires_at(UnixTime::from_secs(1_100)); + let empty = ResolvedEid::new(&a, uid("")).with_refresh_at(UnixTime::from_secs(1_010)); + + let set = entry_sets(&[kept, duplicate, empty], now()) + .sets + .remove(&a) + .expect("should keep a"); + + assert_eq!( + set.expires_at, + Some(UnixTime::from_secs(2_000)), + "should ignore the expiry of a dropped duplicate" + ); + assert_eq!( + set.refresh_at, None, + "should ignore the refresh time of a discarded entry" + ); + } + + #[test] + fn invalidations_chain_and_state_changes_are_reported() { + let a = source("a.example.com"); + let b = source("b.example.com"); + let resolution = Resolution::found([ResolvedEid::new(&a, uid("u"))]) + .also_invalidate(&b) + .with_retry_in(Duration::from_secs(60)) + .replacing_state(&json!({ "signature": "s" })); + assert_eq!(resolution.invalidated(), &[b], "should invalidate b"); + assert_eq!( + resolution.retry_in(), + Some(Duration::from_secs(60)), + "should keep the retry delay" + ); + assert_eq!( + resolution.state_change(), + StateChange::Replace(r#"{"signature":"s"}"#), + "should replace the state" + ); + assert_eq!( + Resolution::no_match().clearing_state().state_change(), + StateChange::Clear, + "should clear the state" + ); + assert_eq!( + Resolution::no_match().state_change(), + StateChange::Keep, + "should keep the state by default" + ); + } + + #[test] + fn resolution_debug_redacts_state() { + let resolution = + Resolution::no_match().replacing_state(&json!({ "signature": "secret-state" })); + + let debug = format!("{resolution:?}"); + + assert!( + !debug.contains("secret-state"), + "should not print provider state" + ); + assert!( + debug.contains("Replace([redacted])"), + "should show that state is replaced" + ); + } + + #[test] + fn oversized_state_is_marked_invalid_not_dropped_silently() { + let resolution = Resolution::found([ResolvedEid::new(&source("a.example.com"), uid("u"))]) + .replacing_state(&"x".repeat(2000)); + assert!( + resolution.state_rejected(), + "should flag the state as rejected" + ); + } + + #[test] + fn oversized_ext_is_dropped_and_the_uid_and_entry_kept() { + let pad = json!({ "pad": "x".repeat(MAX_PARTNER_EXT_BYTES) }); + let resolved = uid("u").with_ext(pad.clone()); + assert_eq!(resolved.id(), "u", "should keep the UID"); + assert!(resolved.ext().is_none(), "should drop the UID-level ext"); + let small = uid("u").with_ext(json!({ "linkType": 2 })); + assert_eq!( + small.ext(), + Some(&json!({ "linkType": 2 })), + "should keep a small ext" + ); + + let eid = ResolvedEid::new(&source("a.example.com"), uid("u")).with_ext(pad); + assert_eq!(eid.uids().len(), 1, "should keep the entry"); + assert!(eid.ext().is_none(), "should drop the EID-level ext"); + } + + #[test] + fn debug_redacts_every_uid() { + let eid = ResolvedEid::new(&source("a.example.com"), uid("secret-one")) + .with_uid(uid("secret-two")); + let debug = format!("{eid:?} {:?}", eid.uids()[0]); + assert!(!debug.contains("secret"), "should not print a UID"); + assert!(debug.contains("a.example.com"), "should print the source"); + } + + #[test] + fn opted_out_is_reported() { + assert!( + Resolution::opted_out().is_opted_out(), + "should report the opt-out" + ); + assert!( + !Resolution::no_match().is_opted_out(), + "should not report an opt-out for no_match" + ); + } +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 2c4c14698c..b835018bac 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -87,6 +87,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { "ec/identity/descriptor.rs", include_str!("ec/identity/descriptor.rs"), ), + ( + "ec/identity/exchange.rs", + include_str!("ec/identity/exchange.rs"), + ), ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), ("ec/identity/input.rs", include_str!("ec/identity/input.rs")), ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), From 8357365451499d238e65d760759452ff0a275ddc Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:02:01 +0530 Subject: [PATCH 19/89] Add the IdentityProvider trait Signed-off-by: dhruv8sh --- .../src/ec/identity/mod.rs | 4 +- .../src/ec/identity/provider.rs | 156 ++++++++++++++++++ .../src/migration_guards.rs | 4 + 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 crates/trusted-server-core/src/ec/identity/provider.rs diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 0d79d61239..5e54e476d5 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -1,6 +1,6 @@ //! Server-side identity providers. //! -//! An `IdentityProvider` turns request signals into `OpenRTB` EIDs. +//! An [`IdentityProvider`] turns request signals into `OpenRTB` EIDs. //! Providers only build requests and parse buffered responses. The //! coordinator owns eligibility, consent, transport limits and KV //! write-back, so every page load makes at most one browser-driven KV @@ -10,6 +10,7 @@ pub mod descriptor; pub mod exchange; pub mod ids; pub mod input; +pub mod provider; pub mod resolution; pub use descriptor::{ @@ -19,6 +20,7 @@ pub use descriptor::{ pub use exchange::{ProviderRequest, ProviderResponse}; pub use ids::{IdentityProviderId, SourceDomain, UnixTime}; pub use input::{CaptureInput, HashedEmail, ProviderConsent, Region, ResolveInput, StoredId}; +pub use provider::{IdentityProvider, IdentityProviderConfig}; pub use resolution::{ IdentityProviderError, Resolution, ResolvedEid, ResolvedUid, StateChange, When, }; diff --git a/crates/trusted-server-core/src/ec/identity/provider.rs b/crates/trusted-server-core/src/ec/identity/provider.rs new file mode 100644 index 0000000000..923d45ca9f --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/provider.rs @@ -0,0 +1,156 @@ +//! The identity provider trait. + +use error_stack::Report; + +use super::descriptor::ProviderDescriptor; +use super::exchange::{ProviderRequest, ProviderResponse}; +use super::input::{CaptureInput, ResolveInput}; +use super::resolution::{IdentityProviderError, Resolution}; +use crate::settings::IntegrationConfig; + +/// A vendor that turns server-side request signals into EIDs. +/// +/// Only [`descriptor`](Self::descriptor) is required. A provider adds a +/// capability by declaring it in its descriptor and overriding the matching +/// methods: +/// +/// - active resolution: [`Resolves`](super::descriptor::Resolves), plus [`request`](Self::request) and +/// [`response`](Self::response) +/// - passive capture: [`Captures`](super::descriptor::Captures), plus [`capture`](Self::capture) +/// +/// Methods added to this trait later get default bodies, so existing +/// providers keep compiling. No method takes generic parameters, so the +/// trait stays object-safe. Every default body returns the +/// [`IdentityProviderError::not_implemented`] marker, so a capability that is +/// declared but not implemented is caught by the harness. +pub trait IdentityProvider: Send + Sync { + /// What this provider is, what it may write, what it needs and which + /// consent it requires. + fn descriptor(&self) -> &ProviderDescriptor; + + /// Builds the call for this request, or `None` to skip it this time. + /// + /// Skipping is not a failure: the coordinator waits `refresh_after_sec` + /// before asking again (section 8). + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::UndeclaredInput`] when it reads + /// something its descriptor did not declare, or another + /// [`IdentityProviderError`] when the request cannot be built. + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let _ = input; + Err(IdentityProviderError::not_implemented("request")) + } + + /// Turns the buffered response to [`request`](Self::request) into a + /// resolution. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Rejected`] for a refusal and + /// [`IdentityProviderError::Malformed`] for a body it cannot parse. + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let _ = (response, input); + Err(IdentityProviderError::not_implemented("response")) + } + + /// Reads IDs from a response TS proxied for this provider, with no + /// outbound call. [`Resolution::no_match`] means the response held + /// nothing to capture, and costs nothing: capture never backs off. + /// + /// # Errors + /// + /// Returns an [`IdentityProviderError`] only for a provider bug. A body + /// it does not recognise is `Ok(Resolution::no_match())`, not an error. + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + let _ = input; + Err(IdentityProviderError::not_implemented("capture")) + } +} + +/// Implemented by each identity integration's config type. +/// +/// Builds the provider's descriptor from config alone, with no secrets, so +/// `ts config validate` checks ownership against the same descriptor the +/// running provider uses. +pub trait IdentityProviderConfig: IntegrationConfig { + /// Describes the provider this config enables. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + fn build_descriptor(&self) -> Result>; +} + +// Keeps the trait object-safe. Fails to compile if a method gains generics. +const _: Option<&dyn IdentityProvider> = None; +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use http::Method; + + use super::*; + use crate::ec::identity::descriptor::Captures; + use crate::ec::identity::exchange::ProviderResponse; + use crate::ec::identity::ids::UnixTime; + use crate::ec::identity::input::{CaptureInput, ProviderConsent}; + + struct DescriptorOnly(ProviderDescriptor); + + impl IdentityProvider for DescriptorOnly { + fn descriptor(&self) -> &ProviderDescriptor { + &self.0 + } + } + + #[test] + fn default_capability_methods_return_the_not_implemented_marker() { + let provider = DescriptorOnly( + ProviderDescriptor::builder("p".parse().expect("should parse the provider id")) + .with_source("example.com".parse().expect("should parse the source")) + .with_captures(Captures::new().with_path(Method::POST, "/integrations/p/api")) + .build() + .expect("should build"), + ); + let boxed: Box = Box::new(provider); + let response = ProviderResponse::for_test(http::StatusCode::OK, b"{}"); + let consent = ProviderConsent::default(); + let stored = BTreeMap::new(); + let input = CaptureInput::new( + boxed.descriptor(), + "/integrations/p/api", + &response, + &stored, + &consent, + "www.example.com", + UnixTime::from_secs(1), + ); + + let error = boxed + .capture(&input) + .expect_err("should return the not-implemented marker"); + assert!( + error.current_context().is_not_implemented(), + "should carry the marker" + ); + assert!( + !IdentityProviderError::config("other") + .current_context() + .is_not_implemented(), + "should not treat an ordinary Config error as the marker" + ); + } +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index b835018bac..140290f82c 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -94,6 +94,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), ("ec/identity/input.rs", include_str!("ec/identity/input.rs")), ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), + ( + "ec/identity/provider.rs", + include_str!("ec/identity/provider.rs"), + ), ( "ec/identity/resolution.rs", include_str!("ec/identity/resolution.rs"), From d63697b948be26f93d5c7701b0425234c656225c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:07:11 +0530 Subject: [PATCH 20/89] Evaluate provider consent policies against decoded consent Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/consent/mod.rs | 2 +- .../src/ec/identity/descriptor.rs | 2 +- .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/policy.rs | 338 ++++++++++++++++++ .../src/migration_guards.rs | 4 + 5 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/policy.rs diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 76aa6ad859..184222b926 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -280,7 +280,7 @@ fn has_eu_tcf_signal(raw_tc_present: bool, gpp_section_ids: Option<&[u16]>) -> b /// Returns the effective decoded TCF consent for enforcement decisions. #[must_use] -fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { +pub(crate) fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { ctx.tcf.as_ref().or_else(|| { let g = ctx.gpp.as_ref()?; g.eu_tcf.as_ref() diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index 9476b2ff86..7ea5d7aafb 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -684,7 +684,7 @@ impl ConsentPolicy { } /// Whether this is generic pull's parity policy. - // Temporary: first used by a later identity task. + // Temporary: its only caller, `policy_allows`, is not yet used. #[allow(dead_code)] #[must_use] pub(crate) fn is_eid_persistence_only(&self) -> bool { diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 5e54e476d5..711389b1d3 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -10,6 +10,7 @@ pub mod descriptor; pub mod exchange; pub mod ids; pub mod input; +pub(crate) mod policy; pub mod provider; pub mod resolution; diff --git a/crates/trusted-server-core/src/ec/identity/policy.rs b/crates/trusted-server-core/src/ec/identity/policy.rs new file mode 100644 index 0000000000..134d12f37e --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/policy.rs @@ -0,0 +1,338 @@ +//! Evaluates a provider's [`ConsentPolicy`] against a request's consent. + +use super::descriptor::{ConsentPolicy, UsOptOut}; +use crate::consent::jurisdiction::Jurisdiction; +use crate::consent::types::PrivacyFlag; +use crate::consent::{ConsentContext, effective_tcf}; + +/// Signals the consent context does not carry. +// Temporary: first used by a later identity task. +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct PolicySignals { + /// `DNT: 1` was present. + pub(crate) dnt: bool, +} + +/// Where TCF applies: `gdpr_applies`, or a jurisdiction of GDPR or unknown. +// Temporary: first used by a later identity task. +#[allow(dead_code)] +#[must_use] +pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool { + consent.gdpr_applies + || matches!( + consent.jurisdiction, + Jurisdiction::Gdpr | Jurisdiction::Unknown + ) +} + +/// Whether the request carries a decoded US sale opt-out, from GPP or from +/// the US Privacy string. +// Temporary: first used by a later identity task. +#[allow(dead_code)] +fn us_sale_opt_out(consent: &ConsentContext) -> bool { + consent.gpp.as_ref().and_then(|gpp| gpp.us_sale_opt_out) == Some(true) + || consent + .us_privacy + .as_ref() + .is_some_and(|usp| usp.opt_out_sale == PrivacyFlag::Yes) +} + +/// Whether `policy` allows a provider under this request's consent. The +/// global EC and EID-persistence gates are checked separately (eligibility +/// steps 1 and 2) and are not repeated here. +// Temporary: first used by a later identity task. +#[allow(dead_code)] +#[must_use] +pub(crate) fn policy_allows( + policy: &ConsentPolicy, + consent: &ConsentContext, + signals: PolicySignals, +) -> bool { + if policy.is_eid_persistence_only() { + return true; + } + // Fail closed: a raw string with no decoded form blocks. + if (consent.raw_tc_string.is_some() && consent.tcf.is_none()) + || (consent.raw_gpp_string.is_some() && consent.gpp.is_none()) + || (consent.raw_us_privacy.is_some() && consent.us_privacy.is_none()) + { + return false; + } + if tcf_applies(consent) { + // A TC string or a GPP TCF section. A header-only GPP string has + // neither, so TCF consent is still missing. + let Some(tcf) = effective_tcf(consent) else { + return false; + }; + if !policy + .tcf_purposes() + .iter() + .all(|purpose| tcf.has_purpose_consent(usize::from(*purpose))) + { + return false; + } + if let Some(vendor) = policy.tcf_vendor() + && !tcf.has_vendor_consent(vendor) + { + return false; + } + } + if !policy.allows_us_opt_out(UsOptOut::Sale) && us_sale_opt_out(consent) { + return false; + } + if !policy.allows_gpc() && consent.gpc { + return false; + } + if !policy.allows_dnt() && signals.dnt { + return false; + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::ConsentContext; + use crate::consent::gpp::decode_gpp_string; + use crate::consent::jurisdiction::Jurisdiction; + use crate::consent::types::TcfConsent; + use crate::consent::us_privacy::decode_us_privacy; + use crate::ec::identity::descriptor::{ConsentPolicy, UsOptOut}; + + #[derive(Default)] + struct Signals { + gdpr: bool, + purposes: Vec, + vendors: Vec, + us_sale_opt_out: bool, + gpc: bool, + raw_tc_without_decode: bool, + gpp_header_only: bool, + tcf_in_gpp: bool, + } + + fn tcf_consent(purposes: &[usize], vendors: &[u16]) -> TcfConsent { + let mut purpose_consents = vec![false; 24]; + for purpose in purposes { + purpose_consents[purpose - 1] = true; + } + TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 1, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: vendors.to_vec(), + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + } + } + + // Takes the builder by value so call sites read as a struct literal. + #[allow(clippy::needless_pass_by_value)] + fn consent_with(signals: Signals) -> ConsentContext { + let mut consent = ConsentContext { + gdpr_applies: signals.gdpr, + gpc: signals.gpc, + jurisdiction: if signals.gdpr { + Jurisdiction::Gdpr + } else { + Jurisdiction::UsState("CA".to_owned()) + }, + ..ConsentContext::default() + }; + if signals.gdpr && !signals.tcf_in_gpp && !signals.purposes.is_empty() { + consent.tcf = Some(tcf_consent(&signals.purposes, &signals.vendors)); + } + if signals.us_sale_opt_out { + consent.raw_us_privacy = Some("1YYN".to_owned()); + consent.us_privacy = + Some(decode_us_privacy("1YYN").expect("should decode a US Privacy string")); + } + if signals.raw_tc_without_decode { + consent.raw_tc_string = Some("undecodable".to_owned()); + } + if signals.gpp_header_only || signals.tcf_in_gpp { + consent.raw_gpp_string = Some("DBAA".to_owned()); + let mut gpp = + decode_gpp_string("DBAA").expect("should decode a header-only GPP string"); + if signals.tcf_in_gpp { + gpp.eu_tcf = Some(tcf_consent(&signals.purposes, &signals.vendors)); + } + consent.gpp = Some(gpp); + consent.gpp_section_ids = Some(Vec::new()); + } + consent + } + + #[test] + fn strict_blocks_each_signal_it_decodes() { + let strict = ConsentPolicy::strict(); + let allowed = consent_with(Signals::default()); + assert!( + policy_allows(&strict, &allowed, PolicySignals { dnt: false }), + "should allow a request with no signal set" + ); + + for (name, signals, dnt) in [ + ( + "gdpr without purpose 4", + Signals { + gdpr: true, + purposes: vec![1], + ..Signals::default() + }, + false, + ), + ( + "us sale opt-out", + Signals { + us_sale_opt_out: true, + ..Signals::default() + }, + false, + ), + ( + "gpc", + Signals { + gpc: true, + ..Signals::default() + }, + false, + ), + ("dnt", Signals::default(), true), + ( + "undecodable tc string", + Signals { + raw_tc_without_decode: true, + ..Signals::default() + }, + false, + ), + ] { + assert!( + !policy_allows(&strict, &consent_with(signals), PolicySignals { dnt }), + "should block under strict: {name}" + ); + } + } + + #[test] + fn relaxations_and_vendor_consent() { + let gpc = consent_with(Signals { + gpc: true, + ..Signals::default() + }); + assert!( + policy_allows( + &ConsentPolicy::strict().allowing_gpc(), + &gpc, + PolicySignals { dnt: false } + ), + "should allow GPC when the policy relaxes it" + ); + let sale = consent_with(Signals { + us_sale_opt_out: true, + ..Signals::default() + }); + assert!( + policy_allows( + &ConsentPolicy::strict().allowing_us_opt_out(UsOptOut::Sale), + &sale, + PolicySignals { dnt: false } + ), + "should allow a sale opt-out when the policy relaxes it" + ); + let gdpr = consent_with(Signals { + gdpr: true, + purposes: vec![1, 4], + vendors: vec![9999], + ..Signals::default() + }); + assert!( + policy_allows( + &ConsentPolicy::strict().with_tcf_vendor(9999), + &gdpr, + PolicySignals { dnt: false } + ), + "should allow a granted vendor" + ); + assert!( + !policy_allows( + &ConsentPolicy::strict().with_tcf_vendor(1234), + &gdpr, + PolicySignals { dnt: false } + ), + "should block a vendor without consent" + ); + } + + #[test] + fn eid_persistence_only_ignores_provider_signals() { + let consent = consent_with(Signals { + gpc: true, + us_sale_opt_out: true, + ..Signals::default() + }); + assert!( + policy_allows( + &ConsentPolicy::eid_persistence_only(), + &consent, + PolicySignals { dnt: true } + ), + "should check only the global gates for generic pull's parity policy, which the coordinator applies separately" + ); + } + + #[test] + fn a_header_only_gpp_string_does_not_block_strict() { + let header_only = consent_with(Signals { + gpp_header_only: true, + ..Signals::default() + }); + assert!( + policy_allows( + &ConsentPolicy::strict(), + &header_only, + PolicySignals { dnt: false } + ), + "should not block on a decoded GPP string with no applicable section" + ); + + let under_gdpr = consent_with(Signals { + gdpr: true, + gpp_header_only: true, + ..Signals::default() + }); + assert!( + !policy_allows( + &ConsentPolicy::strict(), + &under_gdpr, + PolicySignals { dnt: false } + ), + "should still need TCF consent where TCF applies" + ); + + let gpp_tcf = consent_with(Signals { + gdpr: true, + purposes: vec![1, 4], + tcf_in_gpp: true, + ..Signals::default() + }); + assert!( + policy_allows( + &ConsentPolicy::strict(), + &gpp_tcf, + PolicySignals { dnt: false } + ), + "should accept TCF consent from a GPP TCF section" + ); + } +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 140290f82c..b978138849 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -94,6 +94,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), ("ec/identity/input.rs", include_str!("ec/identity/input.rs")), ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), + ( + "ec/identity/policy.rs", + include_str!("ec/identity/policy.rs"), + ), ( "ec/identity/provider.rs", include_str!("ec/identity/provider.rs"), From ef4e51cb21d982ce881529112b5fb7a9c115d246 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:10:04 +0530 Subject: [PATCH 21/89] Add record revisions, state revisions and UID hashes A removal hashes every UID of the record it removes with an unkeyed SHA-256, since the row already holds the UIDs in plain text. A record without rv has the revision Unset. Only the rate-limit visitor key is derived from ec.passphrase. Signed-off-by: dhruv8sh --- .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/revision.rs | 297 ++++++++++++++++++ .../src/migration_guards.rs | 4 + 3 files changed, 302 insertions(+) create mode 100644 crates/trusted-server-core/src/ec/identity/revision.rs diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 711389b1d3..ad9b56c154 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -13,6 +13,7 @@ pub mod input; pub(crate) mod policy; pub mod provider; pub mod resolution; +pub(crate) mod revision; pub use descriptor::{ Access, Captures, ConsentPolicy, EidProvenance, Input, ProviderDescriptor, diff --git a/crates/trusted-server-core/src/ec/identity/revision.rs b/crates/trusted-server-core/src/ec/identity/revision.rs new file mode 100644 index 0000000000..521b89a54c --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/revision.rs @@ -0,0 +1,297 @@ +//! Record revisions, state revisions, UID hashes and the visitor key. + +use hmac::{Hmac, Mac as _}; +use sha2::{Digest as _, Sha256}; + +use super::input::RandomSource; +use crate::ec::kv_types::{KvEid, KvPartnerId, RECORD_REVISION_HEX_LEN}; +use crate::settings::Settings; + +type HmacSha256 = Hmac; + +/// The visitor key, derived from `ec.passphrase` once per settings load. +/// +/// It is the only keyed value: nothing stored in the row is keyed +/// (spec 9.3). +#[derive(Clone)] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) struct IdentityKeys { + visitor: [u8; 32], +} + +impl core::fmt::Debug for IdentityKeys { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter.write_str("IdentityKeys([redacted])") + } +} + +impl IdentityKeys { + /// Derives the visitor key from `ec.passphrase`. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn from_settings(settings: &Settings) -> Self { + let passphrase = settings.ec.passphrase.expose().as_bytes(); + let mut mac = + HmacSha256::new_from_slice(passphrase).expect("should accept any HMAC key length"); + mac.update(b"trusted-server/identity/visitor/v1"); + Self { + visitor: mac.finalize().into_bytes().into(), + } + } +} + +fn hex16(digest: &[u8]) -> String { + hex::encode(&digest[..RECORD_REVISION_HEX_LEN / 2]) +} + +/// A record's stored revision: 16 lowercase hex digits. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) struct RecordRevision(String); + +impl RecordRevision { + /// Wraps a stored `rv`. `KvEntry::validate` has checked the format. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn from_stored(value: &str) -> Self { + Self(value.to_owned()) + } + + /// The hex digits. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +/// A record's revision (spec 5.4). `Unset` for a record without `rv`, which +/// only a schema version 1 row holds, until its next write gives it one. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) enum Revision { + /// The record has no `rv`. + Unset, + /// The record's `rv`. + Set(RecordRevision), +} + +/// A fresh random revision, drawn on every write of a record. +#[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) fn new_revision(random: &dyn RandomSource) -> RecordRevision { + let mut bytes = [0u8; RECORD_REVISION_HEX_LEN / 2]; + random.fill(&mut bytes); + RecordRevision(hex::encode(bytes)) +} + +/// The revision of a stored record. One revision covers the source's whole +/// entry set. +#[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) fn record_revision(record: &KvPartnerId) -> Revision { + record.rv.as_deref().map_or(Revision::Unset, |rv| { + Revision::Set(RecordRevision::from_stored(rv)) + }) +} + +/// The hash of a provider's `ps.st` a call read, so a later apply can tell +/// whether another writer changed it since. +#[derive(Debug, Clone, PartialEq, Eq)] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) struct StateRevision(String); + +impl StateRevision { + /// The first 16 hex digits of the SHA-256 of `st` as compact JSON, or of + /// the empty string when there is none. Compact JSON is never empty, so + /// absent and present state never share a revision. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn of(st: Option<&serde_json::Value>) -> Self { + let bytes = st + .and_then(|value| serde_json::to_vec(value).ok()) + .unwrap_or_default(); + Self(hex16(&Sha256::digest(bytes))) + } +} + +/// The hash stored in `ps.iv` or `ps.ov` for a removed UID: the first 16 hex +/// digits of its unkeyed SHA-256 (spec 6.1). +#[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) fn uid_hash(uid: &str) -> String { + hex16(&Sha256::digest(uid.as_bytes())) +} + +/// One hash per UID of a removed set, in set order, without duplicates. +#[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) fn set_hashes(set: &[KvEid]) -> Vec { + let mut hashes = Vec::new(); + for uid in set.iter().flat_map(|entry| &entry.uids) { + let hash = uid_hash(&uid.id); + if !hashes.contains(&hash) { + hashes.push(hash); + } + } + hashes +} + +/// Appends hashes to a `ps.iv` or `ps.ov` list, oldest first. A hash +/// already present moves to the end. The caller truncates. +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) fn push_hashes(list: &mut Vec, hashes: Vec) { + for hash in hashes { + list.retain(|existing| *existing != hash); + list.push(hash); + } +} + +/// The per-visitor rate-limit key component, from the full EC ID. +#[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] +pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String { + let mut mac = HmacSha256::new_from_slice(&keys.visitor).expect("should accept a 32-byte key"); + mac.update(ec_id.as_bytes()); + hex16(&mac.finalize().into_bytes()) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use sha2::{Digest as _, Sha256}; + + use super::*; + use crate::ec::identity::input::SeededRandom; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + use crate::test_support::tests::create_test_settings; + + fn uid(id: &str) -> KvUid { + KvUid { + id: id.to_owned(), + ..KvUid::default() + } + } + + #[test] + fn new_revisions_are_hex_and_differ() { + let random = SeededRandom::new(7); + let first = new_revision(&random); + let second = new_revision(&random); + assert_eq!(first.as_str().len(), 16, "should be 16 hex digits"); + assert!( + first + .as_str() + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), + "should be lowercase hex" + ); + assert_ne!( + first, second, + "should give two writes of the same UID different revisions" + ); + } + + #[test] + fn record_revision_is_rv_or_unset() { + let with_rv = KvPartnerId { + rv: Some("0123456789abcdef".to_owned()), + ..KvPartnerId::new("u") + }; + assert_eq!( + record_revision(&with_rv), + Revision::Set(RecordRevision::from_stored("0123456789abcdef")), + "should use rv" + ); + assert_eq!( + record_revision(&KvPartnerId::browser("u", Some(10))), + Revision::Unset, + "should be Unset without rv, whatever the UID and at" + ); + } + + #[test] + fn uid_hashes_are_unkeyed_sha256_prefixes() { + let expected = hex::encode(&Sha256::digest(b"example-uid")[..8]); + assert_eq!( + uid_hash("example-uid"), + expected, + "should be the first 16 hex digits of SHA-256" + ); + + let mut other_settings = create_test_settings(); + other_settings.ec.passphrase = + crate::redacted::Redacted::new("another-example-passphrase-value".to_owned()); + assert_ne!( + visitor_key(&IdentityKeys::from_settings(&other_settings), "ec"), + visitor_key(&IdentityKeys::from_settings(&create_test_settings()), "ec"), + "should key only the visitor key by the passphrase" + ); + } + + #[test] + fn a_removal_hashes_every_uid_of_the_set_and_push_moves_repeats_to_the_end() { + let set = vec![ + KvEid { + uids: vec![uid("a"), uid("b")], + ..KvEid::default() + }, + KvEid { + uids: vec![uid("c"), uid("a")], + ..KvEid::default() + }, + ]; + + let hashes = set_hashes(&set); + + let expected: Vec = ["a", "b", "c"].iter().map(|id| uid_hash(id)).collect(); + assert_eq!(hashes, expected, "should hash each UID once, in set order"); + assert_eq!( + set_hashes(&KvPartnerId::new("u").eids), + vec![uid_hash("u")], + "should cost one hash for a one-UID record" + ); + + let mut list = vec![uid_hash("a"), uid_hash("z")]; + push_hashes(&mut list, vec![uid_hash("a"), uid_hash("y")]); + assert_eq!( + list, + vec![uid_hash("z"), uid_hash("a"), uid_hash("y")], + "should move a repeat to the end" + ); + } + + #[test] + fn state_revisions_tell_absent_and_changed_state_apart() { + let state = json!({ "token": "A0" }); + assert_eq!( + StateRevision::of(Some(&state)), + StateRevision::of(Some(&state.clone())), + "should be stable" + ); + assert_ne!( + StateRevision::of(Some(&state)), + StateRevision::of(Some(&json!({ "token": "A1" }))), + "should change with the state" + ); + assert_ne!( + StateRevision::of(None), + StateRevision::of(Some(&json!(""))), + "should tell absent from empty" + ); + } +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index b978138849..6f05598d32 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -106,6 +106,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { "ec/identity/resolution.rs", include_str!("ec/identity/resolution.rs"), ), + ( + "ec/identity/revision.rs", + include_str!("ec/identity/revision.rs"), + ), ("ec/kv.rs", include_str!("ec/kv.rs")), ("ec/kv_types.rs", include_str!("ec/kv_types.rs")), ("ec/mod.rs", include_str!("ec/mod.rs")), From feb30a54667c87d7557377ebb40987cb252a79ee Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:19:37 +0530 Subject: [PATCH 22/89] Filter served KV IDs through one servability check Expired IDs, IDs from disabled or opted-out writers, and IDs whose writer or owner policy fails are no longer sent to bidders, the admin view or /identify. /identify uses the request's consent, never the row's stored snapshot. A stored record goes out as one EID per stored entry. A one-UID record goes out exactly as before, x-ts-eids keeps its per-entry truncation, and /identify keeps returning one UID. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 116 +++++- crates/trusted-server-core/src/ec/admin.rs | 57 ++- crates/trusted-server-core/src/ec/eids.rs | 264 +++++++++++-- crates/trusted-server-core/src/ec/identify.rs | 130 ++++++- .../src/ec/identity/catalog.rs | 125 ++++++ .../src/ec/identity/descriptor.rs | 2 - .../src/ec/identity/mod.rs | 4 + .../src/ec/identity/policy.rs | 21 +- .../src/ec/identity/servable.rs | 363 ++++++++++++++++++ .../src/ec/identity/test_support.rs | 96 +++++ .../src/migration_guards.rs | 12 + crates/trusted-server-core/src/openrtb.rs | 3 +- crates/trusted-server-core/src/publisher.rs | 8 + 13 files changed, 1144 insertions(+), 57 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/catalog.rs create mode 100644 crates/trusted-server-core/src/ec/identity/servable.rs create mode 100644 crates/trusted-server-core/src/ec/identity/test_support.rs diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 15eab65773..f415815de7 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -14,8 +14,13 @@ use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; +use crate::ec::checked_current_timestamp; use crate::ec::client_eids::parse_client_eids; use crate::ec::eids::{resolve_partner_ids, to_eids}; +use crate::ec::identity::UnixTime; +use crate::ec::identity::catalog::ProviderCatalog; +use crate::ec::identity::policy::PolicySignals; +use crate::ec::identity::servable::{ServeContext, opted_out_providers}; use crate::ec::kv::KvIdentityGraph; use crate::ec::partner::normalize_partner_source_domain; use crate::ec::prebid_eids::parse_prebid_eids_cookie; @@ -317,7 +322,12 @@ pub async fn handle_auction( if !matches!(auction_kv_snapshot, EcKvSnapshot::NotRead) { ec_context.set_kv_snapshot(auction_kv_snapshot.clone()); } - let eids = resolve_auction_eids(&auction_kv_snapshot, registry, ec_context); + let eids = resolve_auction_eids( + &auction_kv_snapshot, + registry, + ec_context, + PolicySignals::from_headers(http_req.headers()), + ); // Look up geo for device info. let geo = services @@ -441,10 +451,13 @@ pub async fn handle_auction( /// Returns `None` when any prerequisite is missing (no KV store, no partner /// store, no EC, consent denied). On KV or partner-resolution errors, logs a /// warning and returns empty EIDs so the auction can proceed in degraded mode. +/// A stored record is sent only when the shared servability check passes under +/// this request's consent and `signals`. pub(crate) fn resolve_auction_eids( snapshot: &EcKvSnapshot, registry: Option<&PartnerRegistry>, ec_context: &EcContext, + signals: PolicySignals, ) -> Option> { let registry = registry?; @@ -462,7 +475,23 @@ pub(crate) fn resolve_auction_eids( return Some(Vec::new()); } - let resolved = resolve_partner_ids(registry, entry); + // A zero time would make every expiry check pass (spec 4.4), so a failed + // clock resolves no KV IDs. + let Some(now) = checked_current_timestamp() else { + return Some(Vec::new()); + }; + // Until the request's identity runtime is on `EcContext`, no provider is + // configured, so only unowned records without a writer are servable. + let catalog = ProviderCatalog::empty(); + let opted_out = opted_out_providers(entry); + let serve = ServeContext { + now: UnixTime::from_secs(now), + consent: ec_context.consent(), + signals, + catalog: &catalog, + opted_out: &opted_out, + }; + let resolved = resolve_partner_ids(registry, entry, &serve); Some(to_eids(&resolved)) } @@ -1674,7 +1703,12 @@ mod tests { let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); - let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); + let result = resolve_auction_eids( + &EcKvSnapshot::NotRead, + Some(®istry), + &ec_context, + PolicySignals::default(), + ); assert!( result.is_some_and(|eids| eids.is_empty()), "should degrade to empty EIDs without a snapshot" @@ -1686,7 +1720,12 @@ mod tests { let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); - let result = resolve_auction_eids(&EcKvSnapshot::NotRead, None, &ec_context); + let result = resolve_auction_eids( + &EcKvSnapshot::NotRead, + None, + &ec_context, + PolicySignals::default(), + ); assert!( result.is_none(), "should return None when registry is missing" @@ -1699,7 +1738,12 @@ mod tests { let ec_id = format!("{}.ABC123", "a".repeat(64)); let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); - let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); + let result = resolve_auction_eids( + &EcKvSnapshot::NotRead, + Some(®istry), + &ec_context, + PolicySignals::default(), + ); assert!( result.is_none(), "should return None when consent is denied" @@ -1711,7 +1755,12 @@ mod tests { let registry = PartnerRegistry::empty(); let ec_context = make_ec_context(Jurisdiction::NonRegulated, None); - let result = resolve_auction_eids(&EcKvSnapshot::NotRead, Some(®istry), &ec_context); + let result = resolve_auction_eids( + &EcKvSnapshot::NotRead, + Some(®istry), + &ec_context, + PolicySignals::default(), + ); assert!( result.is_none(), "should return None when no EC value is present" @@ -1727,7 +1776,12 @@ mod tests { let snapshot = EcKvSnapshot::Failed { ec_id: ec_id.clone(), }; - let result = resolve_auction_eids(&snapshot, Some(®istry), &ec_context); + let result = resolve_auction_eids( + &snapshot, + Some(®istry), + &ec_context, + PolicySignals::default(), + ); let eids = result.expect("should return Some on KV error (degraded mode)"); assert!( eids.is_empty(), @@ -1735,6 +1789,54 @@ mod tests { ); } + #[test] + fn resolve_auction_eids_without_providers_matches_pre_identity_output() { + let partners = vec![ + counting_test_partner("a.example.com"), + counting_test_partner("b.example.com"), + ]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let mut entry = crate::ec::kv_types::KvEntry::minimal("a.example.com", "ua", 1); + entry.ids.insert( + "b.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId::new("ub"), + ); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + + let eids = resolve_auction_eids( + &snapshot, + Some(®istry), + &ec_context, + PolicySignals::default(), + ) + .expect("should resolve EIDs"); + + let atype = crate::settings::EcPartner::default_openrtb_atype(); + let expected: Vec = [("a.example.com", "ua"), ("b.example.com", "ub")] + .into_iter() + .map(|(source, uid)| { + Eid::new( + source.to_owned(), + vec![crate::openrtb::Uid { + id: uid.to_owned(), + atype: Some(atype), + ext: None, + }], + ) + }) + .collect(); + assert_eq!( + eids, expected, + "should send exactly the EIDs sent before identity providers existed" + ); + } + #[test] fn resolve_client_auction_eids_falls_back_to_ts_eids_cookie() { let cookie_payload = json!([ diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 04dc13e507..c1125c8f32 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -26,13 +26,20 @@ use serde_json::Value as JsonValue; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; +use crate::consent::ConsentContext; +use crate::consent::jurisdiction::Jurisdiction; use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; use crate::openrtb::Eid; +use super::checked_current_timestamp; use super::eids::{resolve_partner_ids, to_eids}; use super::generation::is_valid_ec_id; +use super::identity::UnixTime; +use super::identity::catalog::ProviderCatalog; +use super::identity::policy::PolicySignals; +use super::identity::servable::{ServeContext, opted_out_providers}; use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; use super::kv_types::{KvEntry, KvMetadata}; @@ -507,7 +514,27 @@ fn iso_timestamp(unix_seconds: u64) -> Option { /// Derives the auction EID view for a valid entry, mirroring the filters in /// [`resolve_partner_ids`] and reporting why each stored ID was skipped. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { - let resolved = resolve_partner_ids(registry, entry); + // The admin view has no request, so it reads with permissive consent. A + // failed clock resolves no KV IDs, as the auction does (spec 4.4). + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }; + let catalog = ProviderCatalog::empty(); + let opted_out = opted_out_providers(entry); + let now = checked_current_timestamp(); + let serve = ServeContext { + now: UnixTime::from_secs(now.unwrap_or(0)), + consent: &consent, + signals: PolicySignals::default(), + catalog: &catalog, + opted_out: &opted_out, + }; + let resolved = if now.is_some() { + resolve_partner_ids(registry, entry, &serve) + } else { + Vec::new() + }; let eids = to_eids(&resolved); let mut skipped = Vec::new(); @@ -518,7 +545,14 @@ fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEid match registry.get(source_domain) { None => "not_in_registry", Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", - Some(_) => continue, + Some(_) + if resolved + .iter() + .any(|item| item.source_domain == *source_domain) => + { + continue; + } + Some(_) => "not_servable", } }; skipped.push(SkippedPartnerId { @@ -1069,6 +1103,25 @@ mod tests { ); } + #[test] + fn auction_view_reports_an_expired_record_as_not_servable() { + let mut entry = KvEntry::minimal("bidstream.example", "uid-expired", 1_741_824_000); + entry + .ids + .get_mut("bidstream.example") + .expect("should hold the source") + .exp = Some(50); + + let view = build_auction_view(&test_registry(), &entry); + + assert!(view.eids.is_empty(), "should not resolve the expired ID"); + assert_eq!(view.skipped.len(), 1, "should explain the dropped ID"); + assert_eq!( + view.skipped[0].reason, "not_servable", + "should report the servability reason" + ); + } + #[test] fn returns_entry_with_auction_view() { let ec_id = test_ec_id(); diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 837ad581c8..1c70675f7d 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -2,7 +2,9 @@ //! //! Used by both `/_ts/api/v1/identify` and `/auction` to resolve source-domain //! keyed IDs from KV entries, convert them to `OpenRTB` EID structures, and -//! build base64-encoded response headers. +//! build base64-encoded response headers. KV IDs are filtered by the shared +//! servability check in [`super::identity::servable`], and a stored record +//! goes out as one EID per stored entry. use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use error_stack::{Report, ResultExt as _}; @@ -10,30 +12,43 @@ use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; -use super::kv_types::KvEntry; +use super::identity::servable::{ServeContext, is_servable}; +use super::identity::{IdentityProviderId, SourceDomain}; +use super::kv_types::{KvEid, KvEntry}; use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. pub const MAX_EIDS_HEADER_BYTES: usize = 4096; -/// A source-domain keyed ID resolved from a KV entry against the partner registry. +/// A source-domain keyed record resolved from a KV entry against the partner +/// registry and the servability check. /// -/// Only includes partners with `bidstream_enabled = true` and a non-empty UID. +/// Only includes partners with `bidstream_enabled = true`, a non-empty UID and +/// a servable record. pub struct ResolvedPartnerId { /// The partner's identity source domain and EC KV `ids` key. pub source_domain: String, - /// The synced user ID value. + /// The first UID of the first stored entry. pub uid: String, - /// `OpenRTB` agent type for this partner's identifiers. + /// `OpenRTB` agent type, used for a UID without a stored one. pub openrtb_atype: i32, + /// The source's stored entry set, in order. One entry for a one-UID record. + pub entries: Vec, + /// The provider that wrote the record. `None` for browser ingest and + /// version 1 rows. + pub writer: Option, } /// Resolves source-domain keyed IDs from a KV entry against the partner registry. /// -/// Filters to partners with `bidstream_enabled = true` and non-empty UIDs, -/// sorted deterministically by source domain. +/// Filters to partners with `bidstream_enabled = true`, non-empty UIDs and +/// servable records, sorted deterministically by source domain. #[must_use] -pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec { +pub(crate) fn resolve_partner_ids( + registry: &PartnerRegistry, + entry: &KvEntry, + serve: &ServeContext<'_>, +) -> Vec { let mut resolved = Vec::new(); for (source_domain, partner_uid) in &entry.ids { @@ -48,10 +63,22 @@ pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec() else { + continue; + }; + if !is_servable(serve, &source, partner_uid) { + continue; + } + resolved.push(ResolvedPartnerId { source_domain: partner.source_domain.clone(), uid: uid.to_owned(), openrtb_atype: partner.openrtb_atype, + entries: partner_uid.eids.clone(), + writer: partner_uid + .w + .as_deref() + .and_then(|writer| writer.parse().ok()), }); } @@ -59,24 +86,40 @@ pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec Vec { resolved .iter() - .map(|item| { - Eid::new( - item.source_domain.clone(), - vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], - ) + .flat_map(|item| { + item.entries + .iter() + .map(move |entry| stored_entry_to_eid(item, entry)) }) .collect() } +fn stored_entry_to_eid(item: &ResolvedPartnerId, entry: &KvEid) -> Eid { + let uids = entry + .uids + .iter() + .map(|uid| Uid { + id: uid.id.clone(), + atype: Some(uid.atype.unwrap_or(item.openrtb_atype)), + ext: uid.ext.clone(), + }) + .collect(); + Eid { + source: item.source_domain.clone(), + uids, + inserter: entry.inserter.clone(), + matcher: entry.matcher.clone(), + mm: entry.mm, + ext: entry.ext.clone(), + } +} + /// Builds a base64-encoded EID header value, truncating if needed. /// /// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds @@ -146,9 +189,58 @@ pub fn encode_eids_header(eids: &[Eid]) -> Result<(String, bool), Report, + } + + impl ServeFixture { + fn new() -> Self { + Self { + consent: test_consent_allowing_everything(), + catalog: ProviderCatalog::empty(), + opted_out: BTreeSet::new(), + } + } + + fn at(&self, now: u64) -> ServeContext<'_> { + ServeContext { + now: UnixTime::from_secs(now), + consent: &self.consent, + signals: PolicySignals::default(), + catalog: &self.catalog, + opted_out: &self.opted_out, + } + } + } + + fn registry_with(sources: &[&str]) -> PartnerRegistry { + let partners: Vec = sources + .iter() + .map(|source| EcPartner { + openrtb_atype: 99, + ..make_test_partner(source) + }) + .collect(); + PartnerRegistry::from_config(&partners).expect("should build registry") + } fn make_test_partner(source_domain: &str) -> EcPartner { EcPartner { @@ -179,16 +271,15 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; - entry.ids.insert( - "zeta.example.com".to_owned(), - super::super::kv_types::KvPartnerId::new("uid-z"), - ); - entry.ids.insert( - "alpha.example.com".to_owned(), - super::super::kv_types::KvPartnerId::new("uid-a"), - ); - - let resolved = resolve_partner_ids(®istry, &entry); + entry + .ids + .insert("zeta.example.com".to_owned(), KvPartnerId::new("uid-z")); + entry + .ids + .insert("alpha.example.com".to_owned(), KvPartnerId::new("uid-a")); + + let fixture = ServeFixture::new(); + let resolved = resolve_partner_ids(®istry, &entry, &fixture.at(100)); let source_domains: Vec<&str> = resolved .iter() .map(|item| item.source_domain.as_str()) @@ -201,6 +292,111 @@ mod tests { ); } + #[test] + fn resolve_partner_ids_skips_expired_records_and_uses_stored_atype() { + let registry = registry_with(&["a.example.com", "b.example.com"]); + let mut entry = KvEntry::minimal("a.example.com", "ua", 1); + entry.ids.insert( + "b.example.com".to_owned(), + KvPartnerId { + exp: Some(50), + ..KvPartnerId::new("ub") + }, + ); + entry + .ids + .get_mut("a.example.com") + .expect("should hold source a") + .eids[0] + .uids[0] + .atype = Some(3); + let fixture = ServeFixture::new(); + + let resolved = resolve_partner_ids(®istry, &entry, &fixture.at(100)); + + assert_eq!(resolved.len(), 1, "should drop the expired b"); + assert_eq!(resolved[0].source_domain, "a.example.com", "should keep a"); + assert_eq!( + to_eids(&resolved)[0].uids[0].atype, + Some(3), + "should prefer the stored atype" + ); + } + + #[test] + fn a_one_uid_record_goes_out_as_on_main() { + let registry = registry_with(&["a.example.com"]); + let entry = KvEntry::minimal("a.example.com", "ua", 1); + let fixture = ServeFixture::new(); + + let eids = to_eids(&resolve_partner_ids(®istry, &entry, &fixture.at(100))); + + assert_eq!( + eids, + vec![Eid::new( + "a.example.com".to_owned(), + vec![Uid { + id: "ua".to_owned(), + atype: Some(99), + ext: None + }], + )], + "should be the EID main sends" + ); + } + + #[test] + fn a_full_record_goes_out_as_one_eid_per_stored_entry() { + let registry = registry_with(&["a.example.com"]); + let mut set = example_entry_set(); + set[1].uids[0].atype = None; + let mut entry = KvEntry::minimal("b.example.com", "unregistered", 1); + entry + .ids + .insert("a.example.com".to_owned(), full_record(None, set)); + let fixture = ServeFixture::new(); + + let eids = to_eids(&resolve_partner_ids(®istry, &entry, &fixture.at(100))); + + assert_eq!( + eids, + vec![ + Eid { + source: "a.example.com".to_owned(), + uids: vec![ + Uid { + id: "example-a".to_owned(), + atype: Some(1), + ext: None + }, + Uid { + id: "example-b".to_owned(), + atype: Some(3), + ext: Some(json!({ "stype": "ppuid" })) + }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + Eid { + source: "a.example.com".to_owned(), + uids: vec![Uid { + id: "example-c".to_owned(), + atype: Some(99), + ext: None + }], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ], + "should send every entry in stored order, a UID without atype taking the partner's" + ); + } + #[test] fn to_eids_maps_resolved_ids_correctly() { let resolved = vec![ @@ -208,16 +404,22 @@ mod tests { uid: "LR_xyz".to_owned(), source_domain: "liveramp.com".to_owned(), openrtb_atype: 3, + entries: KvPartnerId::new("LR_xyz".to_owned()).eids, + writer: None, }, ResolvedPartnerId { uid: "ID5_abc".to_owned(), source_domain: "id5-sync.com".to_owned(), openrtb_atype: 1, + entries: KvPartnerId::new("ID5_abc".to_owned()).eids, + writer: None, }, ResolvedPartnerId { uid: "pair-id".to_owned(), source_domain: "google.com".to_owned(), openrtb_atype: 571187, + entries: KvPartnerId::new("pair-id".to_owned()).eids, + writer: None, }, ]; @@ -247,6 +449,8 @@ mod tests { uid: format!("uid_{}", "x".repeat(100)), source_domain: format!("partner-{idx}.example.com"), openrtb_atype: 3, + entries: KvPartnerId::new(format!("uid_{}", "x".repeat(100))).eids, + writer: None, }); } @@ -266,6 +470,8 @@ mod tests { uid: "u1".to_owned(), source_domain: "ssp.com".to_owned(), openrtb_atype: 3, + entries: KvPartnerId::new("u1".to_owned()).eids, + writer: None, }]; let (encoded, truncated) = diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index e320b817f3..9770ca8169 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -16,6 +16,12 @@ use crate::openrtb::{Eid, Uid}; use crate::settings::Settings; use super::EcContext; +use super::checked_current_timestamp; +use super::identity::SourceDomain; +use super::identity::UnixTime; +use super::identity::catalog::ProviderCatalog; +use super::identity::policy::PolicySignals; +use super::identity::servable::{ServeContext, is_servable, opted_out_providers}; use super::kv::KvIdentityGraph; use super::kv_types::KvPartnerId; use super::log_id; @@ -96,14 +102,36 @@ pub fn handle_identify( // write back with the live-entry TTL. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { - // Extract only this partner's UID. - if let Some(partner_uid) = entry - .ids - .get(&partner.source_domain) - .and_then(KvPartnerId::first_uid) - .filter(|partner_uid| !partner_uid.is_empty()) - { - uid = Some(partner_uid.to_owned()); + // Extract only this partner's UID, and only when the shared + // servability check passes under this request's consent, never + // the row's stored snapshot. A failed clock serves nothing, + // because a zero time would make every expiry check pass. + if let Some(now) = checked_current_timestamp() { + // Until the request's identity runtime is on `EcContext`, no + // provider is configured. + let catalog = ProviderCatalog::empty(); + let opted_out = opted_out_providers(&entry); + let serve = ServeContext { + now: UnixTime::from_secs(now), + consent: ec_context.consent(), + signals: PolicySignals::from_headers(req.headers()), + catalog: &catalog, + opted_out: &opted_out, + }; + if let Some(partner_uid) = entry + .ids + .get(&partner.source_domain) + .filter(|record| { + partner + .source_domain + .parse::() + .is_ok_and(|source| is_servable(&serve, &source, record)) + }) + .and_then(KvPartnerId::first_uid) + .filter(|partner_uid| !partner_uid.is_empty()) + { + uid = Some(partner_uid.to_owned()); + } } // Evaluate cluster size lazily for identify responses. Existing @@ -340,6 +368,8 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::identity::test_support::{example_entry_set, full_record}; + use crate::ec::kv_types::KvEntry; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -645,6 +675,90 @@ mod tests { ); } + fn identify_body(kv: &KvIdentityGraph, ec_id: &str) -> serde_json::Value { + let settings = create_test_settings(); + let partners = vec![make_test_partner("ssp.example.com", VALID_API_TOKEN)]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let req = Request::builder() + .method("GET") + .uri("https://edge.test-publisher.com/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(ec_id)); + + let response = handle_identify(&settings, kv, ®istry, &req, &ec_context) + .expect("should construct identify response"); + assert_eq!(response.status(), StatusCode::OK, "should answer 200"); + serde_json::from_slice(&response.into_body().into_bytes().unwrap_or_default()) + .expect("should decode identify response JSON") + } + + #[test] + fn handle_identify_omits_an_expired_record_and_keeps_a_valid_one() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("ssp.example.com", "expired-uid", 1); + entry + .ids + .get_mut("ssp.example.com") + .expect("should hold the partner source") + .exp = Some(50); + entry.ids.insert( + "other.example.com".to_owned(), + KvPartnerId::new("valid-uid"), + ); + let kv = KvIdentityGraph::in_memory("identify-store"); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + + let body = identify_body(&kv, &ec_id); + + assert!( + body.get("uid").is_none() && body.get("eid").is_none(), + "should not return the expired ID" + ); + + let mut live = KvEntry::minimal("ssp.example.com", "live-uid", 1); + live.ids + .get_mut("ssp.example.com") + .expect("should hold the partner source") + .exp = Some(u64::MAX); + let kv = KvIdentityGraph::in_memory("identify-store"); + kv.create(&ec_id, &live).expect("should seed KV entry"); + + let body = identify_body(&kv, &ec_id); + + assert_eq!(body["uid"], "live-uid", "should return the unexpired ID"); + } + + #[test] + fn handle_identify_returns_one_uid_for_a_full_record() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); + entry.ids.insert( + "ssp.example.com".to_owned(), + full_record(None, example_entry_set()), + ); + let kv = KvIdentityGraph::in_memory("identify-store"); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + + let body = identify_body(&kv, &ec_id); + + assert_eq!(body["uid"], "example-a", "should return the first UID"); + let expected = Eid::new( + "ssp.example.com".to_owned(), + vec![Uid { + id: "example-a".to_owned(), + atype: Some(EcPartner::default_openrtb_atype()), + ext: None, + }], + ); + assert_eq!( + body["eid"], + serde_json::to_value(expected).expect("should serialize the expected EID"), + "should return the one-UID shape, never the full set" + ); + } + #[test] fn handle_identify_denies_mismatched_browser_origin() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/identity/catalog.rs b/crates/trusted-server-core/src/ec/identity/catalog.rs new file mode 100644 index 0000000000..62cc02b9fa --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/catalog.rs @@ -0,0 +1,125 @@ +//! The enabled identity providers and source ownership. + +// Temporary: the narrowing, claim and capture-only accessors are first used by +// later identity tasks. Remove this when they land. +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::sync::Arc; + +use super::ids::{IdentityProviderId, SourceDomain}; +use super::provider::IdentityProvider; + +/// Enabled providers, their narrowed sources and source owners. Built once +/// per settings load. +#[derive(Clone)] +pub(crate) struct ProviderCatalog { + providers: BTreeMap>, + sources: BTreeMap>, + owners: BTreeMap, +} + +impl ProviderCatalog { + /// A catalog with no providers. + #[must_use] + pub(crate) fn empty() -> Self { + Self::new(Vec::new(), BTreeMap::new()) + } + + /// Builds the catalog. Validation (ownership, duplicates) happens in + /// config validation before this is called. + #[must_use] + pub(crate) fn new( + providers: Vec>, + owners: BTreeMap, + ) -> Self { + let providers: BTreeMap<_, _> = providers + .into_iter() + .map(|provider| (provider.descriptor().id().clone(), provider)) + .collect(); + let sources = providers + .iter() + .map(|(id, provider)| (id.clone(), provider.descriptor().sources().to_vec())) + .collect(); + Self { + providers, + sources, + owners, + } + } + + /// Narrows providers to the listed sources their descriptors declare + /// (spec 10.1). Config validation has rejected a source a descriptor + /// does not declare and a narrowing to nothing. + #[must_use] + pub(crate) fn with_narrowing( + mut self, + narrowing: BTreeMap>, + ) -> Self { + for (id, listed) in narrowing { + if let Some(sources) = self.sources.get_mut(&id) { + sources.retain(|source| listed.contains(source)); + } + } + self + } + + /// A provider by ID. + #[must_use] + pub(crate) fn get(&self, id: &IdentityProviderId) -> Option<&Arc> { + self.providers.get(id) + } + + /// Every provider, ordered by ID. + pub(crate) fn providers(&self) -> impl Iterator> { + self.providers.values() + } + + /// Whether `id` is an enabled provider. + #[must_use] + pub(crate) fn is_configured(&self, id: &IdentityProviderId) -> bool { + self.providers.contains_key(id) + } + + /// The owner of `source`, if any. + #[must_use] + pub(crate) fn owner_of(&self, source: &SourceDomain) -> Option<&IdentityProviderId> { + self.owners.get(source) + } + + /// A provider's sources after narrowing. Empty for an unknown provider. + #[must_use] + pub(crate) fn sources_of(&self, id: &IdentityProviderId) -> &[SourceDomain] { + self.sources.get(id).map_or(&[], Vec::as_slice) + } + + /// Every provider that claims `source` after narrowing, ordered by ID. + pub(crate) fn claimants<'a>( + &'a self, + source: &'a SourceDomain, + ) -> impl Iterator { + self.sources + .iter() + .filter(move |(_, sources)| sources.contains(source)) + .map(|(id, _)| id) + } + + /// Whether `id` captures and never resolves. + #[must_use] + pub(crate) fn is_capture_only(&self, id: &IdentityProviderId) -> bool { + self.get(id).is_some_and(|provider| { + let descriptor = provider.descriptor(); + descriptor.captures().is_some() && descriptor.resolves().is_none() + }) + } + + /// The narrowed sources of a capture-only provider. Empty for any other. + #[must_use] + pub(crate) fn capture_only_sources(&self, id: &IdentityProviderId) -> &[SourceDomain] { + if self.is_capture_only(id) { + self.sources_of(id) + } else { + &[] + } + } +} diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index 7ea5d7aafb..aed5a6cf9d 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -684,8 +684,6 @@ impl ConsentPolicy { } /// Whether this is generic pull's parity policy. - // Temporary: its only caller, `policy_allows`, is not yet used. - #[allow(dead_code)] #[must_use] pub(crate) fn is_eid_persistence_only(&self) -> bool { self.eid_persistence_only diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index ad9b56c154..1f38af3c93 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -6,6 +6,7 @@ //! write-back, so every page load makes at most one browser-driven KV //! write. See `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. +pub(crate) mod catalog; pub mod descriptor; pub mod exchange; pub mod ids; @@ -14,6 +15,9 @@ pub(crate) mod policy; pub mod provider; pub mod resolution; pub(crate) mod revision; +pub(crate) mod servable; +#[cfg(any(test, feature = "test-utils"))] +pub mod test_support; pub use descriptor::{ Access, Captures, ConsentPolicy, EidProvenance, Input, ProviderDescriptor, diff --git a/crates/trusted-server-core/src/ec/identity/policy.rs b/crates/trusted-server-core/src/ec/identity/policy.rs index 134d12f37e..49531faf02 100644 --- a/crates/trusted-server-core/src/ec/identity/policy.rs +++ b/crates/trusted-server-core/src/ec/identity/policy.rs @@ -6,17 +6,26 @@ use crate::consent::types::PrivacyFlag; use crate::consent::{ConsentContext, effective_tcf}; /// Signals the consent context does not carry. -// Temporary: first used by a later identity task. -#[allow(dead_code)] #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct PolicySignals { /// `DNT: 1` was present. pub(crate) dnt: bool, } +impl PolicySignals { + /// Reads the signals a request's headers carry. + #[must_use] + pub(crate) fn from_headers(headers: &http::HeaderMap) -> Self { + Self { + dnt: headers + .get("dnt") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim() == "1"), + } + } +} + /// Where TCF applies: `gdpr_applies`, or a jurisdiction of GDPR or unknown. -// Temporary: first used by a later identity task. -#[allow(dead_code)] #[must_use] pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool { consent.gdpr_applies @@ -28,8 +37,6 @@ pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool { /// Whether the request carries a decoded US sale opt-out, from GPP or from /// the US Privacy string. -// Temporary: first used by a later identity task. -#[allow(dead_code)] fn us_sale_opt_out(consent: &ConsentContext) -> bool { consent.gpp.as_ref().and_then(|gpp| gpp.us_sale_opt_out) == Some(true) || consent @@ -41,8 +48,6 @@ fn us_sale_opt_out(consent: &ConsentContext) -> bool { /// Whether `policy` allows a provider under this request's consent. The /// global EC and EID-persistence gates are checked separately (eligibility /// steps 1 and 2) and are not repeated here. -// Temporary: first used by a later identity task. -#[allow(dead_code)] #[must_use] pub(crate) fn policy_allows( policy: &ConsentPolicy, diff --git a/crates/trusted-server-core/src/ec/identity/servable.rs b/crates/trusted-server-core/src/ec/identity/servable.rs new file mode 100644 index 0000000000..613c3c879e --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/servable.rs @@ -0,0 +1,363 @@ +//! The one servability check shared by the bidstream, the admin view and +//! `/_ts/api/v1/identify` (spec 7.1). + +use std::collections::BTreeSet; + +use super::catalog::ProviderCatalog; +use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::policy::{PolicySignals, policy_allows}; +use crate::consent::ConsentContext; +use crate::ec::kv_types::{KvEntry, KvPartnerId}; + +/// What a servability decision needs from the request. +pub(crate) struct ServeContext<'a> { + pub(crate) now: UnixTime, + pub(crate) consent: &'a ConsentContext, + pub(crate) signals: PolicySignals, + pub(crate) catalog: &'a ProviderCatalog, + /// The providers with a `ps.o` in the row the request read. + pub(crate) opted_out: &'a BTreeSet, +} + +/// The providers with a sticky opt-out (`ps.o`) in the row the request read. +/// +/// Read from KV only, never from the request (spec 7.1). A key that is not a +/// valid provider ID cannot name a configured provider and is skipped. +#[must_use] +pub(crate) fn opted_out_providers(entry: &KvEntry) -> BTreeSet { + entry + .ps + .iter() + .filter(|(_, state)| state.o.is_some()) + .filter_map(|(provider, _)| provider.parse().ok()) + .collect() +} + +/// Whether a stored record, the source's whole entry set, may be sent from +/// the server. +#[must_use] +pub(crate) fn is_servable( + context: &ServeContext<'_>, + source: &SourceDomain, + record: &KvPartnerId, +) -> bool { + if record.eids.is_empty() { + return false; + } + if record.exp.is_some_and(|exp| context.now.as_secs() >= exp) { + return false; + } + let writer = record + .w + .as_deref() + .and_then(|writer| writer.parse::().ok()); + if record.w.is_some() { + let Some(writer) = writer.as_ref() else { + return false; + }; + let Some(provider) = context.catalog.get(writer) else { + return false; + }; + if context.opted_out.contains(writer) + || !policy_allows( + provider.descriptor().consent_policy(), + context.consent, + context.signals, + ) + { + return false; + } + } + if let Some(owner) = context.catalog.owner_of(source) { + let Some(owner_provider) = context.catalog.get(owner) else { + return false; + }; + let descriptor = owner_provider.descriptor(); + if !policy_allows( + descriptor.consent_policy(), + context.consent, + context.signals, + ) { + return false; + } + if descriptor.is_expiring(source) + && (writer.as_ref() != Some(owner) || record.exp.is_none()) + { + return false; + } + } + true +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + use std::sync::Arc; + + use super::*; + use crate::ec::identity::descriptor::{Captures, ConsentPolicy, ProviderDescriptor, Resolves}; + use crate::ec::identity::provider::IdentityProvider; + use crate::ec::identity::test_support::{ + example_entry_set, full_record, test_consent_allowing_everything, + test_consent_gdpr_without_vendor, + }; + use crate::ec::kv_types::KvPartnerId; + + struct Fixed(ProviderDescriptor); + impl IdentityProvider for Fixed { + fn descriptor(&self) -> &ProviderDescriptor { + &self.0 + } + } + + fn provider( + id: &str, + source: &str, + policy: ConsentPolicy, + expiring: bool, + ) -> Arc { + let mut builder = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_source(source.parse().expect("should parse the source")) + .with_resolves(Resolves::new().with_endpoint("https://api.example.com")) + .with_consent_policy(policy); + if expiring { + builder = builder.with_expiring_ids(); + } + Arc::new(Fixed(builder.build().expect("should build"))) + } + + fn record(writer: Option<&str>, exp: Option) -> KvPartnerId { + KvPartnerId { + w: writer.map(str::to_owned), + exp, + ..KvPartnerId::new("u") + } + } + + fn check( + catalog: &ProviderCatalog, + opted_out: &BTreeSet, + source: &str, + record: &KvPartnerId, + ) -> bool { + let consent = test_consent_allowing_everything(); + let context = ServeContext { + now: UnixTime::from_secs(100), + consent: &consent, + signals: PolicySignals::default(), + catalog, + opted_out, + }; + is_servable( + &context, + &source.parse().expect("should parse the source"), + record, + ) + } + + #[test] + fn expiry_writer_presence_and_opt_out() { + let catalog = ProviderCatalog::new( + vec![provider( + "p", + "a.example.com", + ConsentPolicy::eid_persistence_only(), + false, + )], + BTreeMap::new(), + ); + let none = BTreeSet::new(); + assert!( + check( + &catalog, + &none, + "a.example.com", + &record(Some("p"), Some(101)) + ), + "should serve a valid record" + ); + assert!( + !check( + &catalog, + &none, + "a.example.com", + &record(Some("p"), Some(100)) + ), + "should treat now == exp as expired" + ); + assert!( + !check( + &catalog, + &none, + "a.example.com", + &record(Some("gone"), None) + ), + "should not serve an unknown writer" + ); + let opted: BTreeSet<_> = ["p".parse().expect("should parse the provider ID")] + .into_iter() + .collect(); + assert!( + !check(&catalog, &opted, "a.example.com", &record(Some("p"), None)), + "should not serve an opted-out writer" + ); + assert!( + check(&catalog, &none, "b.example.com", &record(None, None)), + "should serve an unowned browser record" + ); + } + + #[test] + fn a_full_record_is_servable_or_not_as_a_whole() { + let catalog = ProviderCatalog::new( + vec![provider( + "p", + "a.example.com", + ConsentPolicy::eid_persistence_only(), + false, + )], + BTreeMap::new(), + ); + let none = BTreeSet::new(); + let valid = KvPartnerId { + exp: Some(101), + ..full_record(Some("p"), example_entry_set()) + }; + let expired = KvPartnerId { + exp: Some(100), + ..full_record(Some("p"), example_entry_set()) + }; + assert!( + check(&catalog, &none, "a.example.com", &valid), + "should serve the whole valid set" + ); + assert!( + !check(&catalog, &none, "a.example.com", &expired), + "should never serve an expired set" + ); + } + + #[test] + fn rollback_rule_for_expiring_owned_sources() { + let owner: IdentityProviderId = "owner".parse().expect("should parse the provider ID"); + let catalog = ProviderCatalog::new( + vec![provider( + "owner", + "a.example.com", + ConsentPolicy::eid_persistence_only(), + true, + )], + [( + "a.example.com".parse().expect("should parse the source"), + owner, + )] + .into_iter() + .collect(), + ); + let none = BTreeSet::new(); + assert!( + check( + &catalog, + &none, + "a.example.com", + &record(Some("owner"), Some(200)) + ), + "should serve the owner's value" + ); + assert!( + !check(&catalog, &none, "a.example.com", &record(None, None)), + "should never serve a browser-seeded value from the server" + ); + assert!( + !check( + &catalog, + &none, + "a.example.com", + &record(Some("owner"), None) + ), + "should not serve a value with a stripped exp" + ); + } + + #[test] + fn narrowing_limits_claims_and_capture_only_sources() { + let capture_only: Arc = Arc::new(Fixed( + ProviderDescriptor::builder("c".parse().expect("should parse the provider ID")) + .with_source("a.example.com".parse().expect("should parse the source")) + .with_source("b.example.com".parse().expect("should parse the source")) + .with_captures(Captures::new().with_path_prefix("/integrations/c/")) + .build() + .expect("should build"), + )); + let resolver = provider( + "p", + "a.example.com", + ConsentPolicy::eid_persistence_only(), + false, + ); + let a: SourceDomain = "a.example.com".parse().expect("should parse the source"); + let b: SourceDomain = "b.example.com".parse().expect("should parse the source"); + let c: IdentityProviderId = "c".parse().expect("should parse the provider ID"); + let p: IdentityProviderId = "p".parse().expect("should parse the provider ID"); + + let catalog = ProviderCatalog::new(vec![capture_only, resolver], BTreeMap::new()) + .with_narrowing([(c.clone(), vec![b.clone()])].into_iter().collect()); + + assert_eq!( + catalog.sources_of(&c), + std::slice::from_ref(&b), + "should narrow the capture-only provider" + ); + assert!( + catalog.is_capture_only(&c) && !catalog.is_capture_only(&p), + "should tell capture-only apart" + ); + assert_eq!( + catalog.capture_only_sources(&c), + std::slice::from_ref(&b), + "should list its narrowed sources" + ); + assert!( + catalog.capture_only_sources(&p).is_empty(), + "should list none for a resolver" + ); + assert_eq!( + catalog.claimants(&a).collect::>(), + vec![&p], + "should drop a narrowed-away claim" + ); + assert_eq!( + catalog.claimants(&b).collect::>(), + vec![&c], + "should keep the narrowed claim" + ); + } + + #[test] + fn writer_policy_applies_to_unowned_sources() { + let strict = ConsentPolicy::strict().with_tcf_vendor(9999); + let catalog = ProviderCatalog::new( + vec![provider("p", "a.example.com", strict, false)], + BTreeMap::new(), + ); + let consent = test_consent_gdpr_without_vendor(9999); + let none = BTreeSet::new(); + let context = ServeContext { + now: UnixTime::from_secs(100), + consent: &consent, + signals: PolicySignals::default(), + catalog: &catalog, + opted_out: &none, + }; + let source = "a.example.com".parse().expect("should parse the source"); + assert!( + !is_servable(&context, &source, &record(Some("p"), None)), + "should not serve when the writer's vendor is denied" + ); + assert!( + is_servable(&context, &source, &record(None, None)), + "should leave a browser value unaffected" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs new file mode 100644 index 0000000000..4c2938bf2c --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -0,0 +1,96 @@ +//! Shared helpers for identity tests. + +#![cfg_attr( + not(test), + allow( + dead_code, + reason = "the test-utils library build compiles the crate-private helpers without their unit-test callers" + ) +)] + +use serde_json::json; + +use crate::consent::ConsentContext; +use crate::consent::jurisdiction::Jurisdiction; +use crate::consent::types::TcfConsent; +use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; + +/// A request with no consent signal and no regulated jurisdiction. +#[must_use] +pub fn test_consent_allowing_everything() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + } +} + +/// A GDPR request whose TCF string grants every purpose but not `vendor_id`. +#[must_use] +pub fn test_consent_gdpr_without_vendor(vendor_id: u16) -> ConsentContext { + let vendors: Vec = (1..=20).filter(|vendor| *vendor != vendor_id).collect(); + ConsentContext { + gdpr_applies: true, + jurisdiction: Jurisdiction::Gdpr, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 1, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: vec![true; 24], + purpose_legitimate_interests: vec![false; 24], + vendor_consents: vendors, + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } +} + +/// Spec 6.1's example set: two entries, three UIDs, with UID-level and +/// EID-level fields. +#[must_use] +pub fn example_entry_set() -> Vec { + vec![ + KvEid { + uids: vec![ + KvUid { + id: "example-a".to_owned(), + atype: Some(1), + ext: None, + }, + KvUid { + id: "example-b".to_owned(), + atype: Some(3), + ext: Some(json!({ "stype": "ppuid" })), + }, + ], + inserter: Some("example.com".to_owned()), + mm: Some(3), + ..KvEid::default() + }, + KvEid { + uids: vec![KvUid { + id: "example-c".to_owned(), + atype: Some(1), + ext: None, + }], + matcher: Some("partner.example.com".to_owned()), + ..KvEid::default() + }, + ] +} + +/// A record holding `set`, written by `writer`. +#[must_use] +pub fn full_record(writer: Option<&str>, set: Vec) -> KvPartnerId { + KvPartnerId { + w: writer.map(str::to_owned), + eids: set, + ..KvPartnerId::default() + } +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 6f05598d32..199e8cde0a 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -83,6 +83,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/finalize.rs", include_str!("ec/finalize.rs")), ("ec/generation.rs", include_str!("ec/generation.rs")), ("ec/identify.rs", include_str!("ec/identify.rs")), + ( + "ec/identity/catalog.rs", + include_str!("ec/identity/catalog.rs"), + ), ( "ec/identity/descriptor.rs", include_str!("ec/identity/descriptor.rs"), @@ -106,6 +110,14 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { "ec/identity/resolution.rs", include_str!("ec/identity/resolution.rs"), ), + ( + "ec/identity/servable.rs", + include_str!("ec/identity/servable.rs"), + ), + ( + "ec/identity/test_support.rs", + include_str!("ec/identity/test_support.rs"), + ), ( "ec/identity/revision.rs", include_str!("ec/identity/revision.rs"), diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 086bb6ce8f..e586a0900c 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -67,7 +67,8 @@ pub struct ConsentedProvidersSettings { /// An Extended User ID entry from an identity provider. /// /// Mirrors `OpenRTB` 2.6 §3.2.27. Browser entries keep the EID-level fields -/// Prebid set; KV-resolved entries leave them empty. +/// Prebid set; KV-resolved entries carry the EID-level fields a provider +/// stored, and a record from browser ingest or a version 1 row has none. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Eid { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 86f6bdbbab..7ba3e1b19d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -66,6 +66,7 @@ use crate::creative_opportunities::{ AdStackGateInput, AssemblyMode, CreativeOpportunitiesConfig, RuntimeAdStackExpected, evaluate_ad_stack_gate, }; +use crate::ec::identity::policy::PolicySignals; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::ec::{EcContext, EidSyncSource}; @@ -4941,6 +4942,7 @@ pub async fn handle_publisher_request( kv_snapshot: ec_context.kv_snapshot(), partner_registry: auction.registry, ec_context, + signals: PolicySignals::from_headers(auction_client_request.headers()), services, geo: geo.as_ref(), path_label: "Server-side", @@ -5537,6 +5539,9 @@ struct AuctionEidTargeting<'a> { kv_snapshot: &'a crate::ec::EcKvSnapshot, partner_registry: Option<&'a PartnerRegistry>, ec_context: &'a EcContext, + /// Policy signals the consent context does not carry, read from the + /// request's headers. + signals: PolicySignals, services: &'a RuntimeServices, geo: Option<&'a GeoInfo>, /// Prefix for the consent-stripped warning (e.g. `"Server-side"`). @@ -5566,6 +5571,7 @@ fn apply_auction_eids_and_device( targeting.kv_snapshot, targeting.partner_registry, targeting.ec_context, + targeting.signals, ); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); @@ -7116,6 +7122,7 @@ pub async fn handle_page_bids( kv_snapshot: &page_bids_kv_snapshot, partner_registry: auction.registry, ec_context, + signals: PolicySignals::from_headers(req.headers()), services, geo: geo.as_ref(), path_label: "Page-bids", @@ -8017,6 +8024,7 @@ mod tests { kv_snapshot: &snapshot, partner_registry: Some(®istry), ec_context: &ec_context, + signals: PolicySignals::default(), services: &services, geo: None, path_label: "Test", From de9717121b80f6888adfa0ab2b73743f59194daa Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:26:27 +0530 Subject: [PATCH 23/89] Add IdentityMutation and its merge into a live identity row Removals apply first and are never skipped. An opt-out removes every record its provider wrote, and for a capture-only provider the browser records of its sources, hashing each removed UID into ps.ov; a later clear empties ps.ov. An applied invalidation hashes every UID of the live set into ps.iv, and a fill is skipped when any of its UIDs is in the iv or ov of a provider that claims the source. Stateful results apply atomically, refresh anchors survive while their owner holds state, an unconfigured provider keeps its opt-out and ov, and an invalid row falls back to removals only. apply_removals_only applies a removal-only mutation for the capture removal write and the removal-only retry. Signed-off-by: dhruv8sh --- .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/mutation.rs | 1579 +++++++++++++++++ .../src/ec/identity/revision.rs | 22 - .../src/ec/identity/test_support.rs | 51 + .../src/migration_guards.rs | 4 + 5 files changed, 1635 insertions(+), 22 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/mutation.rs diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 1f38af3c93..f357281cec 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -11,6 +11,7 @@ pub mod descriptor; pub mod exchange; pub mod ids; pub mod input; +pub(crate) mod mutation; pub(crate) mod policy; pub mod provider; pub mod resolution; diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs new file mode 100644 index 0000000000..e7ca939806 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -0,0 +1,1579 @@ +//! One request's identity changes as a single mutation, and its merge into +//! the live KV row (spec 5.4). + +// Temporary: the coordinator, the first non-test caller, arrives in a later +// task. Remove this when it lands. +#![allow(dead_code)] + +use std::collections::BTreeMap; + +use super::catalog::ProviderCatalog; +use super::ids::{IdentityProviderId, SourceDomain}; +use super::input::RandomSource; +use super::revision::{ + Revision, StateRevision, new_revision, push_hashes, record_revision, set_hashes, +}; +use crate::ec::kv_types::{ + KvEntry, KvNetwork, KvOptOut, KvPartnerId, KvProviderState, MAX_INVALIDATED_UIDS, + MAX_PROVIDER_STATES, +}; + +/// A change to one source's record (spec 5.4). +/// +/// Each `record` is the source's whole record: its `eids`, `w`, the timing +/// fields and `rv`. Every change writes or removes a source's set whole. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum PartnerIdChange { + /// From browser ingest (`record.w` is `None`) or an unowned provider + /// (`record.w` names it). Fills a source that holds nothing usable. + FillMissing { + source: SourceDomain, + record: KvPartnerId, + }, + /// From the owning provider, whose ID `record.w` holds. + OwnerWrite { + source: SourceDomain, + record: KvPartnerId, + expected: Option, + }, + /// From `Resolution::invalidate` or `also_invalidate`. `provider` is the + /// provider that invalidated, whose `ps.iv` receives the hashes. + OwnerRemove { + source: SourceDomain, + provider: IdentityProviderId, + expected: Revision, + }, + /// From `Resolution::opted_out`. Removes the source's record if `writer` + /// wrote it, at any revision, and for a capture-only `writer` also a + /// record no provider wrote. Its hashes go to the writer's `ps.ov`. + WriterRemove { + source: SourceDomain, + writer: IdentityProviderId, + }, +} + +impl PartnerIdChange { + fn is_removal(&self) -> bool { + matches!(self, Self::OwnerRemove { .. } | Self::WriterRemove { .. }) + } +} + +/// What a change does to a provider's private state `ps.st`. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) enum StateUpdate { + /// Leave `st` as it is. + #[default] + Keep, + /// Replace `st`. + Replace(serde_json::Value), + /// Remove `st`. + Clear, +} + +/// A change to one provider's `ps` entry. It has no `iv` or `ov` field: +/// those hashes come only from applied removals. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct ProviderStateChange { + pub(crate) n: Option, + pub(crate) f: Option, + pub(crate) opt_out: Option, + pub(crate) opt_out_clear: Option, + pub(crate) st: StateUpdate, +} + +impl ProviderStateChange { + fn is_noop(&self) -> bool { + *self == Self::default() + } + + fn has_removal(&self) -> bool { + self.opt_out.is_some() || self.opt_out_clear.is_some() + } + + fn removals(&self) -> Self { + Self { + opt_out: self.opt_out.clone(), + opt_out_clear: self.opt_out_clear, + ..Self::default() + } + } + + fn merge(&mut self, later: Self) { + self.n = self.n.max(later.n); + self.f = later.f.or(self.f); + if later.opt_out.as_ref().map(|o| o.at) >= self.opt_out.as_ref().map(|o| o.at) { + self.opt_out = later.opt_out.or_else(|| self.opt_out.take()); + } + self.opt_out_clear = self.opt_out_clear.max(later.opt_out_clear); + if later.st != StateUpdate::Keep { + self.st = later.st; + } + } +} + +/// One provider call's outcome. It applies as a unit: every ID change must +/// apply on its own and the state revision must still match. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ProviderResult { + pub(crate) provider: IdentityProviderId, + /// Positive changes only. Removals go to the mutation's own list. + pub(crate) ids: Vec, + pub(crate) state: ProviderStateChange, + /// `None` means the call read no state, so the state check always passes. + pub(crate) expected_state: Option, +} + +/// Everything one request changes in the identity row. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct IdentityMutation { + partner_ids: Vec, + provider_state: BTreeMap, + results: Vec, + cluster_size: Option, +} + +impl IdentityMutation { + /// Adds a change that stands alone. + pub(crate) fn add_change(&mut self, change: PartnerIdChange) { + self.partner_ids.push(change); + } + + /// Adds a provider call's result. + pub(crate) fn add_result(&mut self, result: ProviderResult) { + self.results.push(result); + } + + /// Merges a state change into any earlier one for the same provider: + /// later `n`, last `f`, the later opt-out and clear, last non-`Keep` + /// `st`. + pub(crate) fn merge_state( + &mut self, + provider: IdentityProviderId, + change: ProviderStateChange, + ) { + self.provider_state + .entry(provider) + .or_default() + .merge(change); + } + + /// Sets the cluster size to write. + pub(crate) fn set_cluster_size(&mut self, size: u32) { + self.cluster_size = Some(size); + } + + /// Whether nothing would change. + #[must_use] + pub(crate) fn is_empty(&self) -> bool { + self.partner_ids.is_empty() + && self.results.is_empty() + && self.cluster_size.is_none() + && self + .provider_state + .values() + .all(ProviderStateChange::is_noop) + } + + /// Whether any removal, opt-out or clear is present. + #[must_use] + pub(crate) fn has_removals(&self) -> bool { + self.partner_ids.iter().any(PartnerIdChange::is_removal) + || self + .provider_state + .values() + .any(ProviderStateChange::has_removal) + || self.results.iter().any(|result| result.state.has_removal()) + } + + /// A copy holding only the removals, opt-outs and clears, for the + /// removal write and the removal-only retry. + #[must_use] + pub(crate) fn removals(&self) -> Self { + let mut copy = Self { + partner_ids: self + .partner_ids + .iter() + .filter(|change| change.is_removal()) + .cloned() + .collect(), + ..Self::default() + }; + let states = self.provider_state.iter().chain( + self.results + .iter() + .map(|result| (&result.provider, &result.state)), + ); + for (provider, state) in states { + if state.has_removal() { + copy.merge_state(provider.clone(), state.removals()); + } + } + copy + } + + /// Every provider state change that applies regardless of its result: + /// the standalone ones, and each result's. + fn all_states(&self) -> impl Iterator { + self.provider_state.iter().chain( + self.results + .iter() + .map(|result| (&result.provider, &result.state)), + ) + } +} + +/// What the merge needs besides the mutation and the live row. +pub(crate) struct ApplyContext<'a> { + pub(crate) now: u64, + pub(crate) random: &'a dyn RandomSource, + pub(crate) catalog: &'a ProviderCatalog, +} + +/// The merged row. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Applied { + pub(crate) entry: KvEntry, + /// Whether the result differs from the live row. A caller skips the + /// write when it is `false`. + pub(crate) changed: bool, + /// Whether only the removals were applied, because the full result + /// failed validation. + pub(crate) removals_only: bool, +} + +/// Merges `mutation` into `live` (spec 5.4). +/// +/// Apply order: +/// +/// 1. A tombstone (`consent.ok == false`) gives `None`. +/// 2. Removals, never skipped: +/// - Every `OwnerRemove` whose `expected` equals `record_revision(live_record)`. +/// A version 1 record matches `Revision::Unset`. It appends +/// `set_hashes(&live_record.eids)` to `ps[provider].iv`. +/// - Every `WriterRemove` whose live record has `w == writer`, or, when +/// `writer` is capture-only and the source is one of its +/// `capture_only_sources`, has no `w`. It appends the removed record's +/// hashes to `ps[writer].ov`. +/// - Each new opt-out applies when `at > live.o.at` (or there is no live +/// `o`) and `at >= live.oc` (or there is no live `oc`). Applying it +/// stores `o` and removes every live record that provider wrote, at any +/// revision, and for a capture-only provider every record of its sources +/// that no provider wrote, each adding its hashes to `ps.ov`. +/// - Then each clear (`opt_out_clear`) that is strictly later than the +/// provider's live `o.at` is stored as `oc` and empties `ps.ov`. A clear +/// that is not later, or that finds no live opt-out, is dropped and +/// writes nothing. `ps.iv` is never emptied. +/// - Hashes always come from the record KV held. A removal that no longer +/// applies adds none. +/// 3. Positive changes: +/// - A provider is opted out when its live `o` is set and its `oc` is not +/// strictly later than `o.at`. Every change whose `record.w` is opted out +/// is skipped, `OwnerWrite` and `FillMissing` alike. +/// - A `FillMissing` applies when the source is absent, or its record is +/// expired, or its record's writer is not configured. It is skipped when +/// the hash of any UID of its set is in the `ps.iv` or `ps.ov` of any +/// provider in `catalog.claimants(source)`. A browser `FillMissing` +/// (`record.w` is `None`) is also skipped when the source belongs to an +/// opted-out capture-only provider's `capture_only_sources`. +/// - An `OwnerWrite` applies when the live revision equals `expected`, or +/// when `expected` is `None` and the source has no valid record with +/// `w == record.w`. A browser-seeded value never blocks the owner. +/// - A written record replaces the live record whole, never merges into it, +/// so no entry of the old set survives. It gets a fresh `rv` and +/// `at = record.at.or(now)`, and keeps the change's `w`. +/// - Each `ProviderResult` applies only when every one of its ID changes +/// would apply on its own and, when `expected_state` is `Some`, +/// `StateRevision::of(live ps.st)` equals it. Its state change, `n` and +/// `f` then apply with its IDs. Otherwise the whole result is dropped. +/// 4. `ps` changes: `n` takes the later value, `f` is replaced, `st` is +/// replaced or cleared. Then `cluster_size`. Then truncate each `ps.iv` and +/// `ps.ov` from the front to `MAX_INVALIDATED_UIDS`. Then prune the `ps` +/// entries of providers the catalog does not hold: each keeps only `o`, +/// `oc` and `ov`, so a retired provider's opt-out survives its removal from +/// config and holds again if it comes back, and an entry with none of them +/// is dropped. While `ps` is over `MAX_PROVIDER_STATES`, drop retained +/// entries first, oldest `o.at` first. Then drop expired records, except +/// refresh anchors: the source's owner is configured and holds a `st`, and +/// the record's `ru` has not passed. +/// 5. `validate()`. On failure, rebuild from the live entry through +/// [`apply_removals_only`], and if that also fails, give `None` and log. +/// `validate()` checks the entry-set limits of spec 4.5, so a set that +/// breaks one is dropped whole here and never trimmed. +/// 6. `changed` is whether the result differs from `live`. +#[must_use] +pub(crate) fn apply( + mutation: &IdentityMutation, + live: &KvEntry, + context: &ApplyContext<'_>, +) -> Option { + if !live.consent.ok { + return None; + } + + let mut entry = live.clone(); + apply_removals(&mut entry, mutation, context); + let accepted = apply_positive(&mut entry, mutation, context); + apply_state(&mut entry, mutation, &accepted); + finish_removal_fields(&mut entry, context); + drop_expired_records(&mut entry, context); + + if let Err(message) = entry.validate() { + log::warn!("Identity mutation produced an invalid row, keeping removals only: {message}"); + return apply_removals_only(mutation, live, context).map(|applied| Applied { + removals_only: true, + ..applied + }); + } + let changed = &entry != live; + Some(Applied { + entry, + changed, + removals_only: false, + }) +} + +/// Applies only spec 5.4's steps 1, 2, 4 (`o`, `oc`, `iv` and `ov`), 5 and 6, +/// for the capture removal write and the removal-only retry. It ignores every +/// positive change, `n`, `f`, `st` and `cluster_size`. +#[must_use] +pub(crate) fn apply_removals_only( + mutation: &IdentityMutation, + live: &KvEntry, + context: &ApplyContext<'_>, +) -> Option { + if !live.consent.ok { + return None; + } + + let mut entry = live.clone(); + apply_removals(&mut entry, mutation, context); + finish_removal_fields(&mut entry, context); + + if let Err(message) = entry.validate() { + log::warn!("Identity removals produced an invalid row, dropping them: {message}"); + return None; + } + let changed = &entry != live; + Some(Applied { + entry, + changed, + removals_only: true, + }) +} + +fn state_mut<'a>(entry: &'a mut KvEntry, provider: &IdentityProviderId) -> &'a mut KvProviderState { + entry.ps.entry(provider.to_string()).or_default() +} + +/// Removes the record at `source` when `matches` accepts it, and hashes its +/// set into the list `list` picks from `provider`'s state. +fn remove_record( + entry: &mut KvEntry, + source: &str, + matches: impl FnOnce(&KvPartnerId) -> bool, + provider: &IdentityProviderId, + list: fn(&mut KvProviderState) -> &mut Vec, +) { + if !entry.ids.get(source).is_some_and(matches) { + return; + } + if let Some(record) = entry.ids.remove(source) { + let hashes = set_hashes(&record.eids); + push_hashes(list(state_mut(entry, provider)), hashes); + } +} + +fn invalidated(state: &mut KvProviderState) -> &mut Vec { + &mut state.iv +} + +fn opted_out_hashes(state: &mut KvProviderState) -> &mut Vec { + &mut state.ov +} + +fn apply_removals(entry: &mut KvEntry, mutation: &IdentityMutation, context: &ApplyContext<'_>) { + for change in &mutation.partner_ids { + match change { + PartnerIdChange::OwnerRemove { + source, + provider, + expected, + } => remove_record( + entry, + source.as_str(), + |record| record_revision(record) == *expected, + provider, + invalidated, + ), + PartnerIdChange::WriterRemove { source, writer } => { + let capture_only = context + .catalog + .capture_only_sources(writer) + .contains(source); + remove_record( + entry, + source.as_str(), + |record| match record.w.as_deref() { + Some(written_by) => written_by == writer.as_str(), + None => capture_only, + }, + writer, + opted_out_hashes, + ); + } + PartnerIdChange::FillMissing { .. } | PartnerIdChange::OwnerWrite { .. } => {} + } + } + + for (provider, change) in mutation.all_states() { + if let Some(opt_out) = &change.opt_out { + apply_opt_out(entry, provider, opt_out, context); + } + } + for (provider, change) in mutation.all_states() { + if let Some(clear) = change.opt_out_clear { + apply_clear(entry, provider, clear); + } + } +} + +fn apply_opt_out( + entry: &mut KvEntry, + provider: &IdentityProviderId, + opt_out: &KvOptOut, + context: &ApplyContext<'_>, +) { + let live = entry.ps.get(provider.as_str()); + let newer = live + .and_then(|state| state.o.as_ref()) + .is_none_or(|existing| opt_out.at > existing.at); + let not_cleared_later = live + .and_then(|state| state.oc) + .is_none_or(|cleared| opt_out.at >= cleared); + if !(newer && not_cleared_later) { + return; + } + state_mut(entry, provider).o = Some(opt_out.clone()); + + let capture_only_sources = context.catalog.capture_only_sources(provider); + let doomed: Vec = entry + .ids + .iter() + .filter(|(source, record)| match record.w.as_deref() { + Some(written_by) => written_by == provider.as_str(), + None => capture_only_sources + .iter() + .any(|owned| owned.as_str() == source.as_str()), + }) + .map(|(source, _)| source.clone()) + .collect(); + for source in doomed { + remove_record(entry, &source, |_| true, provider, opted_out_hashes); + } +} + +fn apply_clear(entry: &mut KvEntry, provider: &IdentityProviderId, clear: u64) { + let Some(state) = entry.ps.get_mut(provider.as_str()) else { + return; + }; + if state.o.as_ref().is_some_and(|opt_out| clear > opt_out.at) { + state.oc = Some(clear); + state.ov.clear(); + } +} + +fn is_opted_out(entry: &KvEntry, provider: &str) -> bool { + entry + .ps + .get(provider) + .and_then(|state| state.o.as_ref().map(|o| (o, state.oc))) + .is_some_and(|(opt_out, cleared)| cleared.is_none_or(|cleared| cleared <= opt_out.at)) +} + +fn is_expired(record: &KvPartnerId, now: u64) -> bool { + record.exp.is_some_and(|exp| exp <= now) +} + +fn change_applies(entry: &KvEntry, change: &PartnerIdChange, context: &ApplyContext<'_>) -> bool { + match change { + PartnerIdChange::FillMissing { source, record } => { + fill_applies(entry, source, record, context) + } + PartnerIdChange::OwnerWrite { + source, + record, + expected, + } => { + if record + .w + .as_deref() + .is_some_and(|writer| is_opted_out(entry, writer)) + { + return false; + } + let live = entry.ids.get(source.as_str()); + match expected { + Some(expected) => live.is_some_and(|live| record_revision(live) == *expected), + None => !live.is_some_and(|live| { + live.w.is_some() && live.w == record.w && !is_expired(live, context.now) + }), + } + } + PartnerIdChange::OwnerRemove { .. } | PartnerIdChange::WriterRemove { .. } => false, + } +} + +fn fill_applies( + entry: &KvEntry, + source: &SourceDomain, + record: &KvPartnerId, + context: &ApplyContext<'_>, +) -> bool { + match &record.w { + Some(writer) if is_opted_out(entry, writer) => return false, + Some(_) => {} + None => { + let blocked = context.catalog.providers().any(|provider| { + let id = provider.descriptor().id(); + is_opted_out(entry, id.as_str()) + && context.catalog.capture_only_sources(id).contains(source) + }); + if blocked { + return false; + } + } + } + + let open = entry.ids.get(source.as_str()).is_none_or(|live| { + is_expired(live, context.now) + || live.w.as_deref().is_some_and(|writer| { + !writer + .parse::() + .is_ok_and(|id| context.catalog.is_configured(&id)) + }) + }); + if !open { + return false; + } + + let hashes = set_hashes(&record.eids); + !context.catalog.claimants(source).any(|claimant| { + entry.ps.get(claimant.as_str()).is_some_and(|state| { + hashes + .iter() + .any(|hash| state.iv.contains(hash) || state.ov.contains(hash)) + }) + }) +} + +fn write_change(entry: &mut KvEntry, change: &PartnerIdChange, context: &ApplyContext<'_>) { + let (PartnerIdChange::FillMissing { source, record } + | PartnerIdChange::OwnerWrite { source, record, .. }) = change + else { + return; + }; + let mut written = record.clone(); + written.rv = Some(new_revision(context.random).as_str().to_owned()); + written.at = record.at.or(Some(context.now)); + entry.ids.insert(source.to_string(), written); +} + +/// Applies the positive changes and returns the state changes that go with +/// them: every standalone one, and those of the results that applied. +fn apply_positive<'a>( + entry: &mut KvEntry, + mutation: &'a IdentityMutation, + context: &ApplyContext<'_>, +) -> Vec<(&'a IdentityProviderId, &'a ProviderStateChange)> { + for change in &mutation.partner_ids { + if !change.is_removal() && change_applies(entry, change, context) { + write_change(entry, change, context); + } + } + + let mut accepted: Vec<_> = mutation.provider_state.iter().collect(); + for result in &mutation.results { + let changes = || result.ids.iter().filter(|change| !change.is_removal()); + let state_matches = result.expected_state.as_ref().is_none_or(|expected| { + let live = entry + .ps + .get(result.provider.as_str()) + .and_then(|state| state.st.as_ref()); + StateRevision::of(live) == *expected + }); + if state_matches && changes().all(|change| change_applies(entry, change, context)) { + for change in changes() { + write_change(entry, change, context); + } + accepted.push((&result.provider, &result.state)); + } + } + accepted +} + +fn apply_state( + entry: &mut KvEntry, + mutation: &IdentityMutation, + accepted: &[(&IdentityProviderId, &ProviderStateChange)], +) { + for (provider, change) in accepted { + if change.n.is_some() || change.f.is_some() || change.st != StateUpdate::Keep { + let state = state_mut(entry, provider); + state.n = state.n.max(change.n); + if let Some(failures) = change.f { + state.f = failures; + } + match &change.st { + StateUpdate::Keep => {} + StateUpdate::Replace(value) => state.st = Some(value.clone()), + StateUpdate::Clear => state.st = None, + } + } + } + if let Some(size) = mutation.cluster_size { + entry.network = Some(KvNetwork { + cluster_size: Some(size), + }); + } +} + +fn is_configured(entry_key: &str, context: &ApplyContext<'_>) -> bool { + entry_key + .parse::() + .is_ok_and(|id| context.catalog.is_configured(&id)) +} + +fn finish_removal_fields(entry: &mut KvEntry, context: &ApplyContext<'_>) { + for state in entry.ps.values_mut() { + for list in [&mut state.iv, &mut state.ov] { + let excess = list.len().saturating_sub(MAX_INVALIDATED_UIDS); + list.drain(..excess); + } + } + + for (provider, state) in &mut entry.ps { + if !is_configured(provider, context) { + *state = KvProviderState { + o: state.o.take(), + oc: state.oc.take(), + ov: std::mem::take(&mut state.ov), + ..KvProviderState::default() + }; + } + } + entry + .ps + .retain(|_, state| *state != KvProviderState::default()); + + while entry.ps.len() > MAX_PROVIDER_STATES { + let oldest = entry + .ps + .iter() + .filter(|(provider, _)| !is_configured(provider, context)) + .min_by_key(|(_, state)| state.o.as_ref().map(|o| o.at)) + .map(|(provider, _)| provider.clone()); + let Some(oldest) = oldest else { + break; + }; + entry.ps.remove(&oldest); + } +} + +fn drop_expired_records(entry: &mut KvEntry, context: &ApplyContext<'_>) { + let ps = &entry.ps; + entry.ids.retain(|source, record| { + if !is_expired(record, context.now) { + return true; + } + let anchored = source.parse::().ok().is_some_and(|source| { + context.catalog.owner_of(&source).is_some_and(|owner| { + context.catalog.is_configured(owner) + && ps + .get(owner.as_str()) + .is_some_and(|state| state.st.is_some()) + }) + }); + anchored && record.ru.is_some_and(|ru| context.now < ru) + }); +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::ec::identity::catalog::ProviderCatalog; + use crate::ec::identity::input::SeededRandom; + use crate::ec::identity::revision::{RecordRevision, StateRevision, set_hashes, uid_hash}; + use crate::ec::identity::test_support::{ + example_entry_set, fixed_capture_provider, fixed_provider, full_record, + }; + use crate::ec::kv_types::{ + KvEntry, KvOptOut, KvPartnerId, KvProviderState, MAX_INVALIDATED_UIDS, MAX_PROVIDER_STATES, + }; + + const NOW: u64 = 1_000; + + fn src(name: &str) -> SourceDomain { + name.parse().expect("should parse the source") + } + + fn pid(name: &str) -> IdentityProviderId { + name.parse().expect("should parse the provider ID") + } + + fn rev(value: &str) -> Revision { + Revision::Set(RecordRevision::from_stored(value)) + } + + /// `owner` owns a.example.com, `other` claims b.example.com unowned, and + /// the capture-only `cap` is narrowed to c.example.com. + fn catalog() -> ProviderCatalog { + ProviderCatalog::new( + vec![ + fixed_provider("owner", &["a.example.com"]), + fixed_provider("other", &["b.example.com"]), + fixed_capture_provider("cap", &["c.example.com"]), + ], + [(src("a.example.com"), pid("owner"))].into_iter().collect(), + ) + } + + fn run(mutation: &IdentityMutation, live: &KvEntry) -> Option { + let random = SeededRandom::new(1); + let catalog = catalog(); + apply( + mutation, + live, + &ApplyContext { + now: NOW, + random: &random, + catalog: &catalog, + }, + ) + } + + fn run_removals(mutation: &IdentityMutation, live: &KvEntry) -> Option { + let random = SeededRandom::new(1); + let catalog = catalog(); + apply_removals_only( + mutation, + live, + &ApplyContext { + now: NOW, + random: &random, + catalog: &catalog, + }, + ) + } + + fn written(writer: &str, uid: &str, rv: &str) -> KvPartnerId { + KvPartnerId { + w: Some(writer.to_owned()), + rv: Some(rv.to_owned()), + at: Some(900), + ..KvPartnerId::new(uid) + } + } + + fn opt_out(at: u64) -> ProviderStateChange { + ProviderStateChange { + opt_out: Some(KvOptOut { + at, + fp: "1".to_owned(), + }), + ..ProviderStateChange::default() + } + } + + fn fill(source: &str, record: KvPartnerId) -> PartnerIdChange { + PartnerIdChange::FillMissing { + source: src(source), + record, + } + } + + #[test] + fn tombstone_drops_everything() { + let mut mutation = IdentityMutation::default(); + mutation.add_change(fill("b.example.com", KvPartnerId::browser("u", Some(NOW)))); + assert!( + run(&mutation, &KvEntry::tombstone(1)).is_none(), + "should drop on a tombstone" + ); + assert!( + run_removals(&mutation, &KvEntry::tombstone(1)).is_none(), + "should drop removals on a tombstone" + ); + } + + #[test] + fn fill_missing_fills_absent_expired_and_unconfigured_only() { + let mut live = KvEntry::minimal("b.example.com", "existing", 1); + live.ids.insert( + "d.example.com".to_owned(), + KvPartnerId { + exp: Some(NOW), + ..KvPartnerId::new("old") + }, + ); + live.ids.insert( + "f.example.com".to_owned(), + written("retired", "stale", "1111111111111111"), + ); + let mut mutation = IdentityMutation::default(); + for source in [ + "b.example.com", + "d.example.com", + "e.example.com", + "f.example.com", + ] { + mutation.add_change(fill(source, KvPartnerId::browser("new", Some(NOW)))); + } + + let applied = run(&mutation, &live).expect("should apply"); + + let uid = |source: &str| applied.entry.ids[source].first_uid().map(str::to_owned); + assert_eq!( + uid("b.example.com").as_deref(), + Some("existing"), + "should not replace a live value" + ); + assert_eq!( + uid("d.example.com").as_deref(), + Some("new"), + "should replace an expired value" + ); + assert_eq!( + uid("e.example.com").as_deref(), + Some("new"), + "should fill an absent source" + ); + assert_eq!( + uid("f.example.com").as_deref(), + Some("new"), + "should treat an unconfigured writer as absent" + ); + let record = &applied.entry.ids["e.example.com"]; + assert_eq!( + (record.at, record.w.as_deref()), + (Some(NOW), None), + "should be a browser record" + ); + assert!( + record.rv.is_some(), + "should give every record it writes an rv" + ); + } + + #[test] + fn owner_write_checks_revision_and_gets_a_fresh_rv() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "v1", "1111111111111111"), + ); + let write = |expected| { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { + w: Some("owner".to_owned()), + ..KvPartnerId::new("v2") + }, + expected, + }); + run(&mutation, &live).expect("should apply") + }; + + assert!( + !write(Some(rev("2222222222222222"))).changed, + "should lose a stale revision to the newer write" + ); + let applied = write(Some(rev("1111111111111111"))); + let record = &applied.entry.ids["a.example.com"]; + assert_eq!(record.first_uid(), Some("v2"), "should write"); + assert_eq!(record.w.as_deref(), Some("owner"), "should keep the writer"); + assert_ne!( + record.rv.as_deref(), + Some("1111111111111111"), + "should draw a new rv" + ); + assert_eq!(record.at, Some(NOW), "should stamp at"); + } + + #[test] + fn owner_write_replaces_the_whole_set_and_leaves_other_sources() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { + rv: Some("1111111111111111".to_owned()), + ..full_record(Some("owner"), example_entry_set()) + }, + ); + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { + w: Some("owner".to_owned()), + ..KvPartnerId::new("only") + }, + expected: Some(rev("1111111111111111")), + }); + + let applied = run(&mutation, &live).expect("should apply"); + + assert_eq!( + applied.entry.ids["a.example.com"], + KvPartnerId { + rv: applied.entry.ids["a.example.com"].rv.clone(), + at: Some(NOW), + w: Some("owner".to_owned()), + ..KvPartnerId::new("only") + }, + "should keep no entry of the old set" + ); + assert_eq!( + applied.entry.ids["b.example.com"].first_uid(), + Some("x"), + "should leave other sources alone" + ); + + let mut full = IdentityMutation::default(); + full.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: full_record(Some("owner"), example_entry_set()), + expected: None, + }); + let applied = run(&full, &KvEntry::minimal("b.example.com", "x", 1)).expect("should apply"); + assert_eq!( + applied.entry.ids["a.example.com"].eids, + example_entry_set(), + "should write every entry, in order" + ); + } + + #[test] + fn browser_seeded_and_version_1_values_never_block_the_owner() { + let live = KvEntry::minimal("a.example.com", "seeded", 1); + let mut write = IdentityMutation::default(); + write.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { + w: Some("owner".to_owned()), + ..KvPartnerId::new("owned") + }, + expected: None, + }); + assert_eq!( + run(&write, &live).expect("should apply").entry.ids["a.example.com"].first_uid(), + Some("owned"), + "should let the owner take over a seeded value" + ); + + let mut remove = IdentityMutation::default(); + remove.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: Revision::Unset, + }); + let applied = run(&remove, &live).expect("should apply"); + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should match a record without rv as Unset" + ); + assert_eq!( + applied.entry.ps["owner"].iv, + vec![uid_hash("seeded")], + "should hash the removed UID into iv" + ); + } + + #[test] + fn an_applied_invalidation_hashes_every_uid_of_the_live_set() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { + rv: Some("1111111111111111".to_owned()), + ..full_record(Some("owner"), example_entry_set()) + }, + ); + let invalidate = |expected: &str| { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev(expected), + }); + run(&mutation, &live).expect("should apply") + }; + + let applied = invalidate("1111111111111111"); + let stale = invalidate("2222222222222222"); + + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should remove the set" + ); + assert_eq!( + applied.entry.ps["owner"].iv, + set_hashes(&example_entry_set()), + "should hash every UID it removed" + ); + assert!( + stale.entry.ids.contains_key("a.example.com"), + "should keep a set at another revision" + ); + assert!( + stale + .entry + .ps + .get("owner") + .is_none_or(|state| state.iv.is_empty()), + "should add no hash" + ); + } + + #[test] + fn iv_keeps_the_128_most_recent_hashes() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + let old: Vec = (0..MAX_INVALIDATED_UIDS) + .map(|index| format!("{index:016x}")) + .collect(); + live.ps.insert( + "owner".to_owned(), + KvProviderState { + iv: old.clone(), + ..KvProviderState::default() + }, + ); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "u", "1111111111111111"), + ); + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("1111111111111111"), + }); + + let applied = run(&mutation, &live).expect("should apply"); + + let iv = &applied.entry.ps["owner"].iv; + assert_eq!(iv.len(), MAX_INVALIDATED_UIDS, "should cap at 128"); + assert_eq!(iv[..127], old[1..], "should drop the oldest hash"); + assert_eq!(iv[127], uid_hash("u"), "should append the new hash"); + } + + #[test] + fn opt_out_removes_what_the_writer_wrote_fills_ov_and_skips_its_changes() { + let mut live = KvEntry::minimal("e.example.com", "browser", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "v1", "1111111111111111"), + ); + live.ids.insert( + "b.example.com".to_owned(), + written("owner", "unowned-by-owner", "2222222222222222"), + ); + let mut mutation = IdentityMutation::default(); + mutation.merge_state(pid("owner"), opt_out(NOW)); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { + w: Some("owner".to_owned()), + ..KvPartnerId::new("v2") + }, + expected: None, + }); + + let applied = run(&mutation, &live).expect("should apply"); + + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should remove the owned record" + ); + assert!( + !applied.entry.ids.contains_key("b.example.com"), + "should remove the unowned record it wrote" + ); + assert_eq!( + applied.entry.ids["e.example.com"].first_uid(), + Some("browser"), + "should leave other records" + ); + let state = &applied.entry.ps["owner"]; + assert_eq!( + state.o.as_ref().map(|o| o.at), + Some(NOW), + "should store the opt-out" + ); + assert_eq!( + state.ov, + vec![uid_hash("v1"), uid_hash("unowned-by-owner")], + "should hash what it removed into ov" + ); + assert!(state.iv.is_empty(), "should leave iv for invalidations"); + } + + #[test] + fn a_capture_only_opt_out_removes_browser_records_of_its_sources_and_blocks_their_fill() { + let mut live = KvEntry::minimal("c.example.com", "browser-c", 1); + live.ids + .insert("e.example.com".to_owned(), KvPartnerId::new("browser-e")); + let mut mutation = IdentityMutation::default(); + mutation.merge_state(pid("cap"), opt_out(NOW)); + + let applied = run(&mutation, &live).expect("should apply"); + + assert!( + !applied.entry.ids.contains_key("c.example.com"), + "should remove a browser record of its source" + ); + assert!( + applied.entry.ids.contains_key("e.example.com"), + "should leave a source it is not narrowed to" + ); + + let mut refill = IdentityMutation::default(); + refill.add_change(fill( + "c.example.com", + KvPartnerId::browser("another", Some(NOW)), + )); + let applied = run(&refill, &applied.entry).expect("should apply"); + assert!( + !applied.entry.ids.contains_key("c.example.com"), + "should skip a browser fill while it is opted out" + ); + } + + #[test] + fn opt_out_and_clear_ordering() { + let live = KvEntry::minimal("b.example.com", "x", 1); + let mut same_second = IdentityMutation::default(); + same_second.merge_state( + pid("owner"), + ProviderStateChange { + opt_out_clear: Some(NOW), + ..opt_out(NOW) + }, + ); + let applied = run(&same_second, &live).expect("should apply"); + assert!( + applied.entry.ps["owner"].o.is_some(), + "should stay opted out" + ); + assert_eq!( + applied.entry.ps["owner"].oc, None, + "should drop a clear that is not later" + ); + + let mut no_opt_out = IdentityMutation::default(); + no_opt_out.merge_state( + pid("owner"), + ProviderStateChange { + opt_out_clear: Some(NOW), + ..ProviderStateChange::default() + }, + ); + assert!( + !run(&no_opt_out, &live).expect("should apply").changed, + "should drop a clear with nothing to clear" + ); + + let mut opted = live.clone(); + opted.ps.insert( + "owner".to_owned(), + KvProviderState { + o: Some(KvOptOut { + at: 900, + fp: "1".to_owned(), + }), + iv: vec![uid_hash("invalidated")], + ov: vec![uid_hash("opted-out")], + ..KvProviderState::default() + }, + ); + let mut clear = IdentityMutation::default(); + clear.merge_state( + pid("owner"), + ProviderStateChange { + opt_out_clear: Some(NOW), + ..ProviderStateChange::default() + }, + ); + let state = &run(&clear, &opted).expect("should apply").entry.ps["owner"]; + assert_eq!(state.oc, Some(NOW), "should store a later clear"); + assert!(state.ov.is_empty(), "should empty ov"); + assert_eq!( + state.iv, + vec![uid_hash("invalidated")], + "should never empty iv" + ); + + let mut older = IdentityMutation::default(); + older.merge_state(pid("owner"), opt_out(899)); + assert!( + !run(&older, &opted).expect("should apply").changed, + "should drop an opt-out not after the live one" + ); + } + + #[test] + fn fill_missing_is_skipped_when_any_uid_is_in_a_claimants_iv_or_ov() { + let mut live = KvEntry::minimal("e.example.com", "x", 1); + live.ps.insert( + "owner".to_owned(), + KvProviderState { + iv: vec![uid_hash("example-b")], + ..KvProviderState::default() + }, + ); + live.ps.insert( + "other".to_owned(), + KvProviderState { + ov: vec![uid_hash("removed")], + ..KvProviderState::default() + }, + ); + let mut mutation = IdentityMutation::default(); + mutation.add_change(fill( + "a.example.com", + full_record(None, example_entry_set()), + )); + mutation.add_change(fill( + "b.example.com", + KvPartnerId::browser("removed", Some(NOW)), + )); + mutation.add_change(fill( + "d.example.com", + KvPartnerId::browser("removed", Some(NOW)), + )); + + let applied = run(&mutation, &live).expect("should apply"); + + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should block an owned set on its second UID" + ); + assert!( + !applied.entry.ids.contains_key("b.example.com"), + "should block an unowned source by its claimant's ov" + ); + assert!( + applied.entry.ids.contains_key("d.example.com"), + "should not block a source no denying provider claims" + ); + } + + #[test] + fn provider_result_applies_atomically() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "newer", "3333333333333333"), + ); + live.ps.insert( + "owner".to_owned(), + KvProviderState { + st: Some(json!({ "token": "B" })), + ..KvProviderState::default() + }, + ); + let result = |expected, expected_state| ProviderResult { + provider: pid("owner"), + ids: vec![PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: KvPartnerId { + w: Some("owner".to_owned()), + ..KvPartnerId::new("mine") + }, + expected: Some(expected), + }], + state: ProviderStateChange { + st: StateUpdate::Replace(json!({ "token": "A" })), + f: Some(0), + ..ProviderStateChange::default() + }, + expected_state, + }; + + let mut lost_id = IdentityMutation::default(); + lost_id.add_result(result( + rev("1111111111111111"), + Some(StateRevision::of(Some(&json!({ "token": "B" })))), + )); + let mut lost_state = IdentityMutation::default(); + lost_state.add_result(result( + rev("3333333333333333"), + Some(StateRevision::of(Some(&json!({ "token": "A0" })))), + )); + let mut wins = IdentityMutation::default(); + wins.add_result(result( + rev("3333333333333333"), + Some(StateRevision::of(Some(&json!({ "token": "B" })))), + )); + + for (name, mutation) in [("an ID", lost_id), ("the state", lost_state)] { + assert!( + !run(&mutation, &live).expect("should apply").changed, + "should drop the whole result when {name} lost" + ); + } + let applied = run(&wins, &live).expect("should apply"); + assert_eq!( + applied.entry.ids["a.example.com"].first_uid(), + Some("mine"), + "should write the ID" + ); + assert_eq!( + applied.entry.ps["owner"].st, + Some(json!({ "token": "A" })), + "should write the state with it" + ); + } + + #[test] + fn refresh_anchor_survives_until_ru_or_state_loss() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { + exp: Some(NOW - 1), + ru: Some(NOW + 100), + ..written("owner", "expired", "1111111111111111") + }, + ); + live.ps.insert( + "owner".to_owned(), + KvProviderState { + st: Some(json!({ "r": 1 })), + ..KvProviderState::default() + }, + ); + let mut unrelated = IdentityMutation::default(); + unrelated.set_cluster_size(1); + + assert!( + run(&unrelated, &live) + .expect("should apply") + .entry + .ids + .contains_key("a.example.com"), + "should keep the anchor" + ); + live.ps + .get_mut("owner") + .expect("should hold the owner's state") + .st = None; + assert!( + !run(&unrelated, &live) + .expect("should apply") + .entry + .ids + .contains_key("a.example.com"), + "should drop it" + ); + } + + #[test] + fn unconfigured_provider_state_keeps_o_oc_and_ov() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + let kept = KvProviderState { + o: Some(KvOptOut { + at: 7, + fp: "1".to_owned(), + }), + oc: Some(6), + ov: vec![uid_hash("removed")], + ..KvProviderState::default() + }; + live.ps.insert( + "retired".to_owned(), + KvProviderState { + n: Some(5), + iv: vec![uid_hash("invalidated")], + st: Some(json!(1)), + ..kept.clone() + }, + ); + live.ps.insert( + "gone".to_owned(), + KvProviderState { + n: Some(5), + ..KvProviderState::default() + }, + ); + live.ps.insert( + "other".to_owned(), + KvProviderState { + n: Some(5), + ..KvProviderState::default() + }, + ); + let mut mutation = IdentityMutation::default(); + mutation.set_cluster_size(1); + + let applied = run(&mutation, &live).expect("should apply"); + + assert_eq!( + applied.entry.ps["retired"], kept, + "should keep only o, oc and ov" + ); + assert!( + !applied.entry.ps.contains_key("gone"), + "should drop an entry with none of them" + ); + assert_eq!( + applied.entry.ps["other"].n, + Some(5), + "should leave a configured provider untouched" + ); + } + + #[test] + fn retained_opt_outs_are_pruned_first_when_over_the_state_cap() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ps.insert( + "other".to_owned(), + KvProviderState { + n: Some(5), + ..KvProviderState::default() + }, + ); + for (at, index) in (100_u64..).zip(0..MAX_PROVIDER_STATES) { + live.ps.insert( + format!("retired-{index:02}"), + KvProviderState { + o: Some(KvOptOut { + at, + fp: "1".to_owned(), + }), + ..KvProviderState::default() + }, + ); + } + let mut mutation = IdentityMutation::default(); + mutation.set_cluster_size(1); + + let applied = run(&mutation, &live).expect("should apply"); + + assert_eq!( + applied.entry.ps.len(), + MAX_PROVIDER_STATES, + "should fit the cap" + ); + assert!( + applied.entry.ps.contains_key("other"), + "should keep the configured provider" + ); + assert!( + !applied.entry.ps.contains_key("retired-00"), + "should drop the oldest retained opt-out" + ); + assert!( + applied.entry.ps.contains_key("retired-01"), + "should keep the newer ones" + ); + } + + #[test] + fn an_invalid_result_falls_back_to_removals_only() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "v1", "1111111111111111"), + ); + let mut oversized = example_entry_set(); + oversized[0].ext = Some(json!({ "segments": "x".repeat(2_000) })); + oversized[1].ext = Some(json!({ "segments": "y".repeat(2_000) })); + let mut mutation = IdentityMutation::default(); + mutation.add_change(fill("d.example.com", full_record(Some("other"), oversized))); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("1111111111111111"), + }); + + let applied = run(&mutation, &live).expect("should apply removals"); + + assert!( + applied.removals_only, + "should fail validation for a set over MAX_ENTRY_SET_BYTES" + ); + assert!( + !applied.entry.ids.contains_key("d.example.com"), + "should drop the set whole, never trim it" + ); + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should still apply the removal" + ); + } + + #[test] + fn apply_removals_only_ignores_everything_but_removals() { + let mut live = KvEntry::minimal("b.example.com", "x", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "v1", "1111111111111111"), + ); + live.ids.insert( + "d.example.com".to_owned(), + KvPartnerId { + exp: Some(NOW - 1), + ..KvPartnerId::new("expired") + }, + ); + let mut mutation = IdentityMutation::default(); + mutation.add_change(fill( + "e.example.com", + KvPartnerId::browser("new", Some(NOW)), + )); + mutation.merge_state( + pid("other"), + ProviderStateChange { + n: Some(5), + f: Some(2), + ..opt_out(NOW) + }, + ); + mutation.set_cluster_size(3); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("1111111111111111"), + }); + + let applied = run_removals(&mutation, &live).expect("should apply"); + + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should apply the removal" + ); + assert!( + !applied.entry.ids.contains_key("e.example.com"), + "should ignore a positive change" + ); + assert!( + applied.entry.ids.contains_key("d.example.com"), + "should not prune expired records" + ); + let state = &applied.entry.ps["other"]; + assert_eq!((state.n, state.f), (None, 0), "should never touch n or f"); + assert!(state.o.is_some(), "should store the opt-out"); + assert_eq!( + applied.entry.network, live.network, + "should not touch cluster_size" + ); + + let nothing = IdentityMutation::default(); + assert!( + !run_removals(¬hing, &live).expect("should apply").changed, + "should report no change" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/revision.rs b/crates/trusted-server-core/src/ec/identity/revision.rs index 521b89a54c..cca095aee9 100644 --- a/crates/trusted-server-core/src/ec/identity/revision.rs +++ b/crates/trusted-server-core/src/ec/identity/revision.rs @@ -48,23 +48,17 @@ fn hex16(digest: &[u8]) -> String { /// A record's stored revision: 16 lowercase hex digits. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) struct RecordRevision(String); impl RecordRevision { /// Wraps a stored `rv`. `KvEntry::validate` has checked the format. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn from_stored(value: &str) -> Self { Self(value.to_owned()) } /// The hex digits. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn as_str(&self) -> &str { &self.0 } @@ -73,8 +67,6 @@ impl RecordRevision { /// A record's revision (spec 5.4). `Unset` for a record without `rv`, which /// only a schema version 1 row holds, until its next write gives it one. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) enum Revision { /// The record has no `rv`. Unset, @@ -84,8 +76,6 @@ pub(crate) enum Revision { /// A fresh random revision, drawn on every write of a record. #[must_use] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) fn new_revision(random: &dyn RandomSource) -> RecordRevision { let mut bytes = [0u8; RECORD_REVISION_HEX_LEN / 2]; random.fill(&mut bytes); @@ -95,8 +85,6 @@ pub(crate) fn new_revision(random: &dyn RandomSource) -> RecordRevision { /// The revision of a stored record. One revision covers the source's whole /// entry set. #[must_use] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) fn record_revision(record: &KvPartnerId) -> Revision { record.rv.as_deref().map_or(Revision::Unset, |rv| { Revision::Set(RecordRevision::from_stored(rv)) @@ -106,8 +94,6 @@ pub(crate) fn record_revision(record: &KvPartnerId) -> Revision { /// The hash of a provider's `ps.st` a call read, so a later apply can tell /// whether another writer changed it since. #[derive(Debug, Clone, PartialEq, Eq)] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) struct StateRevision(String); impl StateRevision { @@ -115,8 +101,6 @@ impl StateRevision { /// the empty string when there is none. Compact JSON is never empty, so /// absent and present state never share a revision. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn of(st: Option<&serde_json::Value>) -> Self { let bytes = st .and_then(|value| serde_json::to_vec(value).ok()) @@ -128,16 +112,12 @@ impl StateRevision { /// The hash stored in `ps.iv` or `ps.ov` for a removed UID: the first 16 hex /// digits of its unkeyed SHA-256 (spec 6.1). #[must_use] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) fn uid_hash(uid: &str) -> String { hex16(&Sha256::digest(uid.as_bytes())) } /// One hash per UID of a removed set, in set order, without duplicates. #[must_use] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) fn set_hashes(set: &[KvEid]) -> Vec { let mut hashes = Vec::new(); for uid in set.iter().flat_map(|entry| &entry.uids) { @@ -151,8 +131,6 @@ pub(crate) fn set_hashes(set: &[KvEid]) -> Vec { /// Appends hashes to a `ps.iv` or `ps.ov` list, oldest first. A hash /// already present moves to the end. The caller truncates. -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) fn push_hashes(list: &mut Vec, hashes: Vec) { for hash in hashes { list.retain(|existing| *existing != hash); diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 4c2938bf2c..b9090804de 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -8,11 +8,15 @@ ) )] +use std::sync::Arc; + use serde_json::json; use crate::consent::ConsentContext; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::TcfConsent; +use crate::ec::identity::descriptor::{Captures, ConsentPolicy, ProviderDescriptor, Resolves}; +use crate::ec::identity::provider::IdentityProvider; use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; /// A request with no consent signal and no regulated jurisdiction. @@ -94,3 +98,50 @@ pub fn full_record(writer: Option<&str>, set: Vec) -> KvPartnerId { ..KvPartnerId::default() } } + +/// A descriptor-only resolver for `id`, declaring `sources`, with a dummy +/// endpoint and the `eid_persistence_only` policy. +/// +/// # Panics +/// +/// Panics when `id` or a source is not valid. +#[must_use] +pub fn fixed_provider(id: &str, sources: &[&str]) -> Arc { + let mut builder = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_resolves(Resolves::new().with_endpoint("https://api.example.com")) + .with_consent_policy(ConsentPolicy::eid_persistence_only()); + for source in sources { + builder = builder.with_source(source.parse().expect("should parse the source")); + } + Arc::new(DescriptorOnly( + builder.build().expect("should build the descriptor"), + )) +} + +/// A descriptor-only capture-only provider for `id`, declaring `sources`. +/// +/// # Panics +/// +/// Panics when `id` or a source is not valid. +#[must_use] +pub fn fixed_capture_provider(id: &str, sources: &[&str]) -> Arc { + let mut builder = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_captures(Captures::new().with_path_prefix(&format!("/integrations/{id}/"))) + .with_consent_policy(ConsentPolicy::eid_persistence_only()); + for source in sources { + builder = builder.with_source(source.parse().expect("should parse the source")); + } + Arc::new(DescriptorOnly( + builder.build().expect("should build the descriptor"), + )) +} + +struct DescriptorOnly(ProviderDescriptor); + +impl IdentityProvider for DescriptorOnly { + fn descriptor(&self) -> &ProviderDescriptor { + &self.0 + } +} diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 199e8cde0a..61dab90100 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -98,6 +98,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), ("ec/identity/input.rs", include_str!("ec/identity/input.rs")), ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), + ( + "ec/identity/mutation.rs", + include_str!("ec/identity/mutation.rs"), + ), ( "ec/identity/policy.rs", include_str!("ec/identity/policy.rs"), From 1e8e9e778d6c4bac92fdd00cbe0c1726d7a990b0 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:31:34 +0530 Subject: [PATCH 24/89] Add the ts-eids-s mirror payload and its size cap The payload holds the servable server entries as bidders receive them, base64url-encoded, or tells the shim to clear the cookie. A set over 2,048 bytes leaves out whole sources, lowest cookie_priority first, then largest, and max_age never outlives the earliest exp. Signed-off-by: dhruv8sh --- .../src/ec/identity/mirror.rs | 286 ++++++++++++++++++ .../src/ec/identity/mod.rs | 1 + .../src/migration_guards.rs | 4 + 3 files changed, 291 insertions(+) create mode 100644 crates/trusted-server-core/src/ec/identity/mirror.rs diff --git a/crates/trusted-server-core/src/ec/identity/mirror.rs b/crates/trusted-server-core/src/ec/identity/mirror.rs new file mode 100644 index 0000000000..95f911c259 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/mirror.rs @@ -0,0 +1,286 @@ +//! The `ts-eids-s` mirror payload: the servable server set for page scripts, +//! capped to one cookie (spec 6.2). The server never reads the cookie back. + +// Temporary: the coordinator, the first non-test caller, arrives in a later +// task. Remove this when it lands. +#![allow(dead_code)] + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde_json::json; + +use super::ids::IdentityProviderId; +use crate::openrtb::Eid; + +/// Maximum bytes of the encoded cookie value. +pub(crate) const MIRROR_MAX_VALUE_BYTES: usize = 2048; +/// Maximum cookie lifetime: that of `ts-eids`. +pub(crate) const MIRROR_MAX_AGE_SEC: u64 = 86_400; + +/// One source's servable server entries, as bidders receive them. +#[derive(Debug, Clone)] +pub(crate) struct MirrorSource { + pub(crate) source: String, + /// The provider whose `cookie_priority` ranks the source, for the log. + pub(crate) provider: Option, + pub(crate) priority: i32, + /// The source's earliest `exp`, in Unix seconds. + pub(crate) expires_at: Option, + pub(crate) eids: Vec, +} + +/// What the request's KV read gave the mirror. +#[derive(Debug, Clone, Copy)] +pub(crate) enum MirrorRead<'a> { + /// No KV read: the response carries no mirror member. + NotRead, + /// Consent denies EID persistence, or the row is a tombstone, withdrawn + /// or missing. + Cleared, + /// The servable server set, in output order. + Servable(&'a [MirrorSource]), +} + +/// The mirror member of an `/auction` or `/_ts/page-bids` response. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum MirrorPayload { + Value { value: String, max_age: u64 }, + Clear, +} + +impl MirrorPayload { + /// The JSON the shim reads. + #[must_use] + pub(crate) fn to_json(&self) -> serde_json::Value { + match self { + Self::Value { value, max_age } => json!({ "value": value, "max_age": max_age }), + Self::Clear => json!({ "clear": true }), + } + } +} + +fn encode(sources: &[&MirrorSource]) -> String { + let eids: Vec<&Eid> = sources.iter().flat_map(|source| &source.eids).collect(); + let bytes = + serde_json::to_vec(&json!({ "v": 1, "eids": eids })).expect("should serialize EIDs"); + URL_SAFE_NO_PAD.encode(bytes) +} + +fn source_bytes(source: &MirrorSource) -> usize { + serde_json::to_vec(&source.eids).map_or(usize::MAX, |bytes| bytes.len()) +} + +/// The mirror payload for a request (spec 6.2). +/// +/// # Performance +/// +/// Encodes the set once per omission. A set holds at most one entry set per +/// configured source, so this stays small. +#[must_use] +pub(crate) fn mirror_payload(read: MirrorRead<'_>, now: u64) -> Option { + let sources = match read { + MirrorRead::NotRead => return None, + MirrorRead::Cleared => return Some(MirrorPayload::Clear), + MirrorRead::Servable(sources) => sources, + }; + let mut kept: Vec<&MirrorSource> = sources.iter().collect(); + let mut value = encode(&kept); + while value.len() > MIRROR_MAX_VALUE_BYTES && !kept.is_empty() { + let index = (0..kept.len()) + .min_by_key(|&index| { + ( + kept[index].priority, + core::cmp::Reverse(source_bytes(kept[index])), + core::cmp::Reverse(index), + ) + }) + .expect("should hold a source"); + let left_out = kept.remove(index); + log::info!( + "Leaving source '{}' of provider '{}' out of the ts-eids-s mirror, which exceeds {MIRROR_MAX_VALUE_BYTES} bytes", + left_out.source, + left_out + .provider + .as_ref() + .map_or("none", IdentityProviderId::as_str), + ); + value = encode(&kept); + } + if kept.is_empty() { + return Some(MirrorPayload::Clear); + } + let max_age = kept + .iter() + .filter_map(|source| source.expires_at) + .map(|exp| exp.saturating_sub(now).max(1)) + .fold(MIRROR_MAX_AGE_SEC, u64::min); + Some(MirrorPayload::Value { value, max_age }) +} + +#[cfg(test)] +mod tests { + use base64::Engine as _; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use serde_json::json; + + use super::*; + use crate::openrtb::{Eid, Uid}; + + const NOW: u64 = 1_760_000_000; + + fn source(name: &str, priority: i32, expires_at: Option, ids: &[&str]) -> MirrorSource { + MirrorSource { + source: name.to_owned(), + provider: Some("p".parse().expect("should parse the provider ID")), + priority, + expires_at, + eids: vec![Eid::new( + name.to_owned(), + ids.iter() + .map(|id| Uid { + id: (*id).to_owned(), + atype: Some(1), + ext: None, + }) + .collect(), + )], + } + } + + fn decode(payload: &MirrorPayload) -> (serde_json::Value, u64) { + let MirrorPayload::Value { value, max_age } = payload else { + panic!("should hold a value"); + }; + let bytes = URL_SAFE_NO_PAD + .decode(value) + .expect("should be unpadded base64url"); + ( + serde_json::from_slice(&bytes).expect("should be JSON"), + *max_age, + ) + } + + #[test] + fn no_read_is_absent_and_a_denial_or_empty_set_is_clear() { + assert_eq!( + mirror_payload(MirrorRead::NotRead, NOW), + None, + "should leave the cookie as it is" + ); + assert_eq!( + mirror_payload(MirrorRead::Cleared, NOW), + Some(MirrorPayload::Clear), + "should clear on a denial" + ); + assert_eq!( + mirror_payload(MirrorRead::Servable(&[]), NOW), + Some(MirrorPayload::Clear), + "should clear an empty set" + ); + assert_eq!( + MirrorPayload::Clear.to_json(), + json!({ "clear": true }), + "should serialize the clear shape" + ); + } + + #[test] + fn value_is_the_v1_entries_in_unpadded_base64url() { + let sources = [source("example.com", 0, None, &["example-a"])]; + + let payload = + mirror_payload(MirrorRead::Servable(&sources), NOW).expect("should return a payload"); + + let (decoded, max_age) = decode(&payload); + assert_eq!( + decoded, + json!({ "v": 1, "eids": [{ "source": "example.com", "uids": [{ "id": "example-a", "atype": 1 }] }] }), + "should hold the entries exactly as bidders receive them" + ); + assert_eq!( + max_age, MIRROR_MAX_AGE_SEC, + "should live a day without an exp" + ); + let MirrorPayload::Value { value, .. } = &payload else { + panic!("should hold a value") + }; + assert_eq!( + payload.to_json(), + json!({ "value": value, "max_age": MIRROR_MAX_AGE_SEC }), + "should serialize" + ); + } + + #[test] + fn max_age_never_outlives_the_earliest_exp() { + let sources = [ + source("a.example.com", 0, Some(NOW + 3_600), &["a"]), + source("b.example.com", 0, Some(NOW + 600), &["b"]), + source("c.example.com", 0, Some(NOW + 999_999), &["c"]), + ]; + + let (_, max_age) = decode( + &mirror_payload(MirrorRead::Servable(&sources), NOW).expect("should return a payload"), + ); + + assert_eq!(max_age, 600, "should take the earliest exp"); + } + + #[test] + fn an_oversized_set_drops_whole_sources_lowest_priority_then_largest() { + let big = |name: &str, priority, count| { + let ids: Vec = (0..count) + .map(|index| format!("{name}-{index:040}")) + .collect(); + source( + name, + priority, + None, + &ids.iter().map(String::as_str).collect::>(), + ) + }; + let sources = [ + big("keep-high", 10, 6), + big("drop-low-large", 0, 12), + big("keep-low-small", 0, 6), + big("keep-mid", 5, 6), + ]; + + let payload = + mirror_payload(MirrorRead::Servable(&sources), NOW).expect("should return a payload"); + + let MirrorPayload::Value { value, .. } = &payload else { + panic!("should hold a value") + }; + assert!(value.len() <= MIRROR_MAX_VALUE_BYTES, "should fit the cap"); + let (decoded, _) = decode(&payload); + let kept: Vec<&str> = decoded["eids"] + .as_array() + .expect("should be an array") + .iter() + .map(|eid| eid["source"].as_str().expect("should name the source")) + .collect(); + assert_eq!( + kept, + vec!["keep-high", "keep-low-small", "keep-mid"], + "should drop the largest lowest-priority source whole" + ); + } + + #[test] + fn a_set_where_nothing_fits_is_clear() { + let ids: Vec = (0..80).map(|index| format!("only-{index:040}")).collect(); + let sources = [source( + "only.example.com", + 0, + None, + &ids.iter().map(String::as_str).collect::>(), + )]; + + assert_eq!( + mirror_payload(MirrorRead::Servable(&sources), NOW), + Some(MirrorPayload::Clear), + "should clear rather than keep an empty or partial set" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index f357281cec..603f634ed9 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -11,6 +11,7 @@ pub mod descriptor; pub mod exchange; pub mod ids; pub mod input; +pub(crate) mod mirror; pub(crate) mod mutation; pub(crate) mod policy; pub mod provider; diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 61dab90100..da431b223e 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -98,6 +98,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/identity/ids.rs", include_str!("ec/identity/ids.rs")), ("ec/identity/input.rs", include_str!("ec/identity/input.rs")), ("ec/identity/mod.rs", include_str!("ec/identity/mod.rs")), + ( + "ec/identity/mirror.rs", + include_str!("ec/identity/mirror.rs"), + ), ( "ec/identity/mutation.rs", include_str!("ec/identity/mutation.rs"), From cafe2e6eaa60df1751dadd6f0db46b0a8dd9279a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:36:57 +0530 Subject: [PATCH 25/89] Add the shared identity removal write A removal-only mutation is read, applied and written with the withdrawal path's CAS loop: up to five attempts, retrying only a generation mismatch. It writes nothing for a tombstone, a missing row or a removal that already applied. The capture removal write and the removal-only retry of a failed flush both use it. Signed-off-by: dhruv8sh --- .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/mutation.rs | 194 +++++++++ .../src/ec/identity/removal.rs | 385 ++++++++++++++++++ crates/trusted-server-core/src/ec/kv.rs | 30 +- .../src/migration_guards.rs | 4 + 5 files changed, 613 insertions(+), 1 deletion(-) create mode 100644 crates/trusted-server-core/src/ec/identity/removal.rs diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 603f634ed9..7f163dd652 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -15,6 +15,7 @@ pub(crate) mod mirror; pub(crate) mod mutation; pub(crate) mod policy; pub mod provider; +pub(crate) mod removal; pub mod resolution; pub(crate) mod revision; pub(crate) mod servable; diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs index e7ca939806..23de933244 100644 --- a/crates/trusted-server-core/src/ec/identity/mutation.rs +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -139,7 +139,23 @@ impl IdentityMutation { } /// Adds a provider call's result. + /// + /// A removal in `result.ids` is a provider-side mistake: removals go + /// through `partner_ids` and `provider_state`. It is logged by provider + /// and source, never by UID. pub(crate) fn add_result(&mut self, result: ProviderResult) { + for change in result.ids.iter().filter(|change| change.is_removal()) { + let (PartnerIdChange::OwnerRemove { source, .. } + | PartnerIdChange::WriterRemove { source, .. }) = change + else { + continue; + }; + log::warn!( + "Identity provider '{}' put a removal for source '{}' in its result IDs; removals belong in partner_ids", + result.provider, + source.as_str(), + ); + } self.results.push(result); } @@ -185,6 +201,34 @@ impl IdentityMutation { || self.results.iter().any(|result| result.state.has_removal()) } + /// The provider and source of every removal, opt-out and clear, sorted and + /// de-duplicated. An opt-out or clear has no source. Holds no UID, so it + /// is safe to log. + #[must_use] + pub(crate) fn removal_labels(&self) -> Vec<(IdentityProviderId, Option)> { + let mut labels: Vec<_> = self + .partner_ids + .iter() + .filter_map(|change| match change { + PartnerIdChange::OwnerRemove { + source, provider, .. + } => Some((provider.clone(), Some(source.clone()))), + PartnerIdChange::WriterRemove { source, writer } => { + Some((writer.clone(), Some(source.clone()))) + } + _ => None, + }) + .collect(); + labels.extend( + self.all_states() + .filter(|(_, state)| state.has_removal()) + .map(|(provider, _)| (provider.clone(), None)), + ); + labels.sort(); + labels.dedup(); + labels + } + /// A copy holding only the removals, opt-outs and clears, for the /// removal write and the removal-only retry. #[must_use] @@ -243,6 +287,8 @@ pub(crate) struct Applied { /// Merges `mutation` into `live` (spec 5.4). /// +/// A `ps` entry holding only default values is dropped during cleanup. +/// /// Apply order: /// /// 1. A tombstone (`consent.ok == false`) gives `None`. @@ -1576,4 +1622,152 @@ mod tests { "should report no change" ); } + + fn writer_remove(source: &str, writer: &str) -> IdentityMutation { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::WriterRemove { + source: src(source), + writer: pid(writer), + }); + mutation + } + + #[test] + fn a_writer_remove_removes_a_record_the_writer_owns() { + let mut live = KvEntry::minimal("e.example.com", "browser", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("owner", "v1", "1111111111111111"), + ); + + let applied = + run_removals(&writer_remove("a.example.com", "owner"), &live).expect("should apply"); + + assert!( + !applied.entry.ids.contains_key("a.example.com"), + "should remove the record the writer wrote" + ); + assert_eq!( + applied.entry.ps["owner"].ov, + vec![uid_hash("v1")], + "should hash the removed UID into ov" + ); + } + + #[test] + fn a_capture_only_writer_remove_removes_a_browser_record() { + let mut live = KvEntry::minimal("e.example.com", "other", 1); + live.ids + .insert("c.example.com".to_owned(), KvPartnerId::new("browser-c")); + + let applied = + run_removals(&writer_remove("c.example.com", "cap"), &live).expect("should apply"); + + assert!( + !applied.entry.ids.contains_key("c.example.com"), + "should remove the browser record of a capture-only source" + ); + assert_eq!( + applied.entry.ps["cap"].ov, + vec![uid_hash("browser-c")], + "should hash the removed UID into ov" + ); + } + + #[test] + fn a_writer_remove_keeps_another_writers_record() { + let mut live = KvEntry::minimal("e.example.com", "browser", 1); + live.ids.insert( + "a.example.com".to_owned(), + written("other", "v1", "1111111111111111"), + ); + + let applied = + run_removals(&writer_remove("a.example.com", "owner"), &live).expect("should apply"); + + assert!( + applied.entry.ids.contains_key("a.example.com"), + "should keep a record another writer wrote" + ); + assert!(!applied.changed, "should report no change"); + } + + #[test] + fn removals_and_has_removals_round_trip() { + let mut mutation = writer_remove("a.example.com", "owner"); + mutation.add_change(fill("b.example.com", KvPartnerId::browser("u", Some(NOW)))); + mutation.merge_state(pid("other"), opt_out(NOW)); + mutation.merge_state( + pid("cap"), + ProviderStateChange { + n: Some(3), + ..ProviderStateChange::default() + }, + ); + + assert!(mutation.has_removals(), "should see the removals"); + let removals = mutation.removals(); + assert!(removals.has_removals(), "should keep the removals"); + assert_eq!( + removals.removal_labels(), + mutation.removal_labels(), + "should keep every removal label" + ); + assert!( + removals.partner_ids.iter().all(PartnerIdChange::is_removal), + "should drop the positive changes" + ); + assert!( + !removals.provider_state.contains_key(&pid("cap")), + "should drop a state change that holds no removal" + ); + assert!( + !IdentityMutation::default().has_removals(), + "should see no removals in an empty mutation" + ); + } + + #[test] + fn removal_labels_are_sorted_and_deduplicated() { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("1111111111111111"), + }); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: rev("2222222222222222"), + }); + mutation.add_change(PartnerIdChange::WriterRemove { + source: src("b.example.com"), + writer: pid("cap"), + }); + mutation.merge_state(pid("owner"), opt_out(NOW)); + mutation.merge_state( + pid("other"), + ProviderStateChange { + opt_out_clear: Some(NOW), + ..ProviderStateChange::default() + }, + ); + mutation.add_result(ProviderResult { + provider: pid("owner"), + ids: Vec::new(), + state: opt_out(NOW), + expected_state: None, + }); + + assert_eq!( + mutation.removal_labels(), + vec![ + (pid("cap"), Some(src("b.example.com"))), + (pid("other"), None), + (pid("owner"), None), + (pid("owner"), Some(src("a.example.com"))), + ], + "should list each provider and source once, sorted, with no UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/identity/removal.rs b/crates/trusted-server-core/src/ec/identity/removal.rs new file mode 100644 index 0000000000..4bdede3d78 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/removal.rs @@ -0,0 +1,385 @@ +//! The shared removal write: a removal-only mutation written with the +//! withdrawal path's CAS loop (spec 5.2 phase 4, 5.3). + +// Temporary: the capture removal write and the flush retry first use this in +// later identity tasks. Remove this when they land. +#![allow(dead_code)] + +use super::ids::SourceDomain; +use super::mutation::{ApplyContext, IdentityMutation, apply_removals_only}; +use crate::ec::kv::{KvIdentityGraph, MAX_CAS_RETRIES}; +use crate::ec::kv_backend::EcKvWriteOutcome; +use crate::ec::{EcKvSnapshot, log_id}; + +/// Writes the removal write makes at most, as the withdrawal path does. +pub(crate) const REMOVAL_WRITE_ATTEMPTS: u32 = MAX_CAS_RETRIES; + +/// How a removal write ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RemovalWriteOutcome { + /// The removals are in KV. + Written, + /// Nothing to write: a tombstone, a missing row, or already applied. + NothingToWrite, + /// Not written. The caller may try once more after send. + Failed, +} + +/// Reads the row, applies only `removals` and writes them with a CAS, +/// retrying only a generation mismatch. +/// +/// Runs whatever the request's consent: a revocation can arrive when consent +/// is unusable. It does not log a lost removal; see [`log_removal_lost`]. +#[must_use] +pub(crate) fn write_removals( + graph: &KvIdentityGraph, + ec_id: &str, + removals: &IdentityMutation, + context: &ApplyContext<'_>, +) -> RemovalWriteOutcome { + for _attempt in 0..REMOVAL_WRITE_ATTEMPTS { + let (entry, generation) = match graph.load_snapshot(ec_id) { + EcKvSnapshot::Present { + entry, + generation: Some(generation), + .. + } => (entry, generation), + EcKvSnapshot::Missing { .. } => return RemovalWriteOutcome::NothingToWrite, + _ => return RemovalWriteOutcome::Failed, + }; + let Some(applied) = + apply_removals_only(removals, &entry, context).filter(|applied| applied.changed) + else { + return RemovalWriteOutcome::NothingToWrite; + }; + match graph.write_if_generation(ec_id, &applied.entry, generation) { + Ok(EcKvWriteOutcome::Written) => return RemovalWriteOutcome::Written, + Ok(EcKvWriteOutcome::PreconditionFailed) => {} + Err(error) => { + log::warn!( + "Identity removal write failed for '{}': {error:?}", + log_id(ec_id) + ); + return RemovalWriteOutcome::Failed; + } + } + } + log::warn!( + "Identity removal write for '{}': CAS conflict after {REMOVAL_WRITE_ATTEMPTS} attempts", + log_id(ec_id) + ); + RemovalWriteOutcome::Failed +} + +/// Logs a removal that every attempt failed to write. Names providers and +/// sources only, never a UID. +pub(crate) fn log_removal_lost(ec_id: &str, removals: &IdentityMutation) { + for (provider, source) in removals.removal_labels() { + log::error!( + "Identity removal lost for '{}': provider '{provider}', source '{}'", + log_id(ec_id), + source.as_ref().map_or("*", SourceDomain::as_str), + ); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use error_stack::Report; + + use super::*; + use crate::ec::identity::catalog::ProviderCatalog; + use crate::ec::identity::input::SeededRandom; + use crate::ec::identity::mutation::{PartnerIdChange, ProviderStateChange}; + use crate::ec::identity::revision::{RecordRevision, Revision}; + use crate::ec::identity::test_support::fixed_provider; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_backend::{ + EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, + }; + use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId}; + use crate::error::TrustedServerError; + + const EC_ID: &str = "0000000000000000000000000000000000000000000000000000000000000000.ABC123"; + + /// Write counts the store keeps after seeding. + #[derive(Default)] + struct WriteCounts { + all: AtomicUsize, + generation_checked: AtomicUsize, + } + + /// Fails the first `conflicts` generation-checked writes with + /// `PreconditionFailed`, or every generation-checked write with an error + /// when `error` is set. Counts every write it sees. + struct ScriptedEcKv { + inner: InMemoryEcKv, + conflicts: Mutex, + error: bool, + counts: Arc, + } + + impl EcKvStore for ScriptedEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.counts.all.fetch_add(1, Ordering::SeqCst); + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + self.counts + .generation_checked + .fetch_add(1, Ordering::SeqCst); + if self.error { + return Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "per-key write limit".to_owned(), + })); + } + let mut remaining = self.conflicts.lock().expect("should lock the counter"); + if *remaining > 0 { + *remaining -= 1; + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + /// Seeds `entry` with `create`, then wraps the store and zeroes the counts. + fn graph_with( + entry: Option<&KvEntry>, + conflicts: u32, + error: bool, + ) -> (KvIdentityGraph, Arc) { + let counts = Arc::new(WriteCounts::default()); + let graph = KvIdentityGraph::new(ScriptedEcKv { + inner: InMemoryEcKv::new("removal-store"), + conflicts: Mutex::new(conflicts), + error, + counts: Arc::clone(&counts), + }); + if let Some(entry) = entry { + graph.create(EC_ID, entry).expect("should seed the row"); + } + counts.all.store(0, Ordering::SeqCst); + counts.generation_checked.store(0, Ordering::SeqCst); + (graph, counts) + } + + fn removal() -> IdentityMutation { + let mut mutation = IdentityMutation::default(); + mutation.add_change(PartnerIdChange::OwnerRemove { + source: "a.example.com".parse().expect("should parse the source"), + provider: "owner".parse().expect("should parse the provider ID"), + expected: Revision::Set(RecordRevision::from_stored("1111111111111111")), + }); + mutation + } + + fn live() -> KvEntry { + let mut entry = KvEntry::minimal("b.example.com", "x", 1); + entry.ids.insert( + "a.example.com".to_owned(), + KvPartnerId { + w: Some("owner".to_owned()), + rv: Some("1111111111111111".to_owned()), + ..KvPartnerId::new("u") + }, + ); + entry + } + + fn run(graph: &KvIdentityGraph, removals: &IdentityMutation) -> RemovalWriteOutcome { + let random = SeededRandom::new(1); + let catalog = ProviderCatalog::new( + vec![fixed_provider("owner", &["a.example.com"])], + [( + "a.example.com".parse().expect("should parse the source"), + "owner".parse().expect("should parse the provider ID"), + )] + .into_iter() + .collect(), + ); + write_removals( + graph, + EC_ID, + removals, + &ApplyContext { + now: 1_000, + random: &random, + catalog: &catalog, + }, + ) + } + + #[test] + fn writes_the_removal_once_with_a_generation_check() { + let (graph, counts) = graph_with(Some(&live()), 0, false); + + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::Written, + "should write" + ); + + assert_eq!( + counts.generation_checked.load(Ordering::SeqCst), + 1, + "should write once, with IfGenerationMatch" + ); + let (entry, _) = graph + .get(EC_ID) + .expect("should read") + .expect("should hold the row"); + assert!( + !entry.ids.contains_key("a.example.com"), + "should remove the record" + ); + assert_eq!( + entry.ps["owner"].iv.len(), + 1, + "should hash the removed UID into iv" + ); + } + + #[test] + fn retries_only_a_generation_mismatch_up_to_five_writes() { + let (graph, counts) = graph_with(Some(&live()), 2, false); + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::Written, + "should win on the third write" + ); + assert_eq!( + counts.generation_checked.load(Ordering::SeqCst), + 3, + "should re-read and retry each conflict" + ); + + let (graph, counts) = graph_with(Some(&live()), 5, false); + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::Failed, + "should give up after five conflicts" + ); + assert_eq!( + counts.generation_checked.load(Ordering::SeqCst), + 5, + "should stop at five writes" + ); + + let (graph, counts) = graph_with(Some(&live()), 0, true); + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::Failed, + "should stop at a non-CAS error" + ); + assert_eq!( + counts.generation_checked.load(Ordering::SeqCst), + 1, + "should not retry a per-key limit" + ); + } + + #[test] + fn writes_nothing_for_a_tombstone_a_missing_row_or_an_applied_removal() { + let (graph, counts) = graph_with(Some(&KvEntry::tombstone(1)), 0, false); + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::NothingToWrite, + "should skip a tombstone" + ); + assert_eq!( + counts.all.load(Ordering::SeqCst), + 0, + "should not write a tombstone" + ); + + let (graph, counts) = graph_with(None, 0, false); + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::NothingToWrite, + "should skip a missing row" + ); + assert_eq!( + counts.all.load(Ordering::SeqCst), + 0, + "should not write a missing row" + ); + + let (graph, counts) = + graph_with(Some(&KvEntry::minimal("b.example.com", "x", 1)), 0, false); + assert_eq!( + run(&graph, &removal()), + RemovalWriteOutcome::NothingToWrite, + "should skip an applied removal" + ); + assert_eq!( + counts.all.load(Ordering::SeqCst), + 0, + "should not write an applied removal" + ); + } + + #[test] + fn an_opt_out_and_its_clear_are_written_without_positive_changes() { + let (graph, _) = graph_with(Some(&live()), 0, false); + let mut mutation = IdentityMutation::default(); + mutation.merge_state( + "owner".parse().expect("should parse the provider ID"), + ProviderStateChange { + opt_out: Some(KvOptOut { + at: 1_000, + fp: "1".to_owned(), + }), + ..ProviderStateChange::default() + }, + ); + + assert_eq!( + run(&graph, &mutation.removals()), + RemovalWriteOutcome::Written, + "should write the opt-out" + ); + + let (entry, _) = graph + .get(EC_ID) + .expect("should read") + .expect("should hold the row"); + assert!(entry.ps["owner"].o.is_some(), "should store o"); + assert!( + !entry.ids.contains_key("a.example.com"), + "should remove what the provider wrote" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 926797892c..274efafee9 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -25,7 +25,7 @@ use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. -const MAX_CAS_RETRIES: u32 = 5; +pub(crate) const MAX_CAS_RETRIES: u32 = 5; /// Maximum number of keys to request when counting hash-prefix matches /// for cluster size evaluation. Anything above this is clearly a large @@ -861,6 +861,34 @@ impl KvIdentityGraph { } } + /// Writes `entry` only if the row is still at `generation`. + /// + /// The identity removal write and the phase 4 flush share this path, so + /// every identity write is validated and generation-checked. + /// + /// # Errors + /// + /// Returns an error when `entry` does not serialize or validate, or the + /// store write fails for a reason other than a generation mismatch. + // Temporary: the removal write and the flush are its first callers, in + // later identity tasks. Remove this when they land. + #[allow(dead_code)] + pub(crate) fn write_if_generation( + &self, + ec_id: &str, + entry: &KvEntry, + generation: u64, + ) -> Result> { + let (body, metadata) = Self::serialize_entry(entry, self.store_name())?; + self.write_entry( + ec_id, + &body, + &metadata, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(generation), + ) + } + /// Returns `proven` instead of a read outcome that cannot disprove it. /// /// Partner-ID enrichment is best effort. When the caller already held proof diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index da431b223e..c8f256da0b 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -114,6 +114,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { "ec/identity/provider.rs", include_str!("ec/identity/provider.rs"), ), + ( + "ec/identity/removal.rs", + include_str!("ec/identity/removal.rs"), + ), ( "ec/identity/resolution.rs", include_str!("ec/identity/resolution.rs"), From 04c56ae338178df96786293aa038f7463268ea48 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 20:49:02 +0530 Subject: [PATCH 26/89] Add [ec.identity] coordinator settings Adds coordinator policy and per-provider tables, including after_send_timeout_ms and revoke_scope. ec.pull_sync_concurrency is removed: a config that sets it fails validation with an error naming [ec.identity].max_concurrency, and the example config and the integration fixture move their value there. No whole-config snapshot, envelope-hash or CLI snapshot expectations needed updating; the config_payload round-trip test now compares ec.identity.max_concurrency. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/config.rs | 108 ++++- .../trusted-server-core/src/config_payload.rs | 2 +- .../trusted-server-core/src/ec/pull_sync.rs | 14 +- crates/trusted-server-core/src/settings.rs | 369 +++++++++++++++++- .../configs/trusted-server.integration.toml | 4 +- docs/guide/configuration.md | 26 +- trusted-server.example.toml | 24 +- 7 files changed, 524 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index e48f5c2f03..abb81971b4 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -32,7 +32,15 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; +use crate::settings::{ + AssetOriginAuth, IdentitySettings, IntegrationConfig, ProviderMode, Settings, +}; + +/// Largest `[ec.identity].hold_budget_ms`. +const MAX_HOLD_BUDGET_MS: u64 = 1000; + +/// Largest `after_send_timeout_ms` of a provider. +const MAX_AFTER_SEND_TIMEOUT_MS: u64 = 10_000; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] @@ -258,6 +266,8 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Report { + Report::new(TrustedServerError::Configuration { message }) +} + +/// Validates the numeric rules of `[ec.identity]`. +/// +/// Rules that need provider descriptors live in the identity catalog +/// validation instead. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming the first key that is +/// out of range. +pub(crate) fn validate_identity_settings( + settings: &IdentitySettings, +) -> Result<(), Report> { + if settings.hold_budget_ms > MAX_HOLD_BUDGET_MS { + return Err(identity_configuration_error(format!( + "ec.identity.hold_budget_ms must be at most {MAX_HOLD_BUDGET_MS}, got {}", + settings.hold_budget_ms + ))); + } + if settings.max_concurrency == 0 { + return Err(identity_configuration_error( + "ec.identity.max_concurrency must be at least 1".to_owned(), + )); + } + for (id, provider) in &settings.providers { + if provider.refresh_after_sec == 0 { + return Err(identity_configuration_error(format!( + "ec.identity.providers.{id}.refresh_after_sec must be greater than 0" + ))); + } + if !(1..=MAX_AFTER_SEND_TIMEOUT_MS).contains(&provider.after_send_timeout_ms) { + return Err(identity_configuration_error(format!( + "ec.identity.providers.{id}.after_send_timeout_ms must be between 1 and {MAX_AFTER_SEND_TIMEOUT_MS}, got {}", + provider.after_send_timeout_ms + ))); + } + if provider.mode == ProviderMode::Hold + && provider + .timeout_ms + .is_some_and(|timeout| timeout > settings.hold_budget_ms) + { + return Err(identity_configuration_error(format!( + "ec.identity.providers.{id}.timeout_ms must not exceed hold_budget_ms ({})", + settings.hold_budget_ms + ))); + } + } + Ok(()) +} + +/// Rejects config keys that were removed instead of deprecated. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming the replacement of the +/// first removed key that is set. +pub(crate) fn validate_removed_keys(settings: &Settings) -> Result<(), Report> { + if settings.ec.removed_pull_sync_concurrency.is_some() { + return Err(identity_configuration_error( + "ec.pull_sync_concurrency was removed; set [ec.identity].max_concurrency instead" + .to_owned(), + )); + } Ok(()) } @@ -1616,4 +1697,29 @@ password = "production-admin-password-32-bytes" "validation error should mention invalid provider" ); } + + #[test] + fn removed_pull_sync_concurrency_is_rejected_with_its_replacement() { + let toml = crate_test_settings_str().replace("[ec]\n", "[ec]\npull_sync_concurrency = 3\n"); + let mut settings = Settings::from_toml(&toml).expect("should parse the removed key"); + settings.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; + + let deploy = validate_settings_for_deploy(&settings).expect_err("should reject at deploy"); + assert!( + format!("{deploy:?}").contains("[ec.identity].max_concurrency"), + "should name the replacement at deploy, got: {deploy:?}" + ); + let runtime = + validate_settings_for_runtime(&settings).expect_err("should reject at runtime"); + assert!( + format!("{runtime:?}").contains("[ec.identity].max_concurrency"), + "should name the replacement at runtime, got: {runtime:?}" + ); + + let control = crate_test_settings_str() + .replace("[ec]\n", "[ec.identity]\nmax_concurrency = 3\n\n[ec]\n"); + let mut control = Settings::from_toml(&control).expect("should parse the replacement key"); + control.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; + validate_settings_for_deploy(&control).expect("control: should pass at deploy"); + } } diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index c9690b01f5..85f53dba82 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -291,7 +291,7 @@ mod tests { "should preserve publisher domain" ); assert_eq!( - reconstructed.ec.pull_sync_concurrency, original.ec.pull_sync_concurrency, + reconstructed.ec.identity.max_concurrency, original.ec.identity.max_concurrency, "should preserve numeric fields" ); assert_eq!( diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index e01d662da7..a72accc9ea 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -181,7 +181,7 @@ pub fn dispatch_pull_sync( return; } - let max_concurrency = settings.ec.pull_sync_concurrency.max(1); + let max_concurrency = settings.ec.identity.max_concurrency.max(1); let mut in_flight: Vec = Vec::new(); let mut updates = Vec::new(); @@ -943,7 +943,7 @@ mod tests { fn dispatch_pull_sync_aggregates_batches_into_one_bulk_write() { let mut settings = create_test_settings(); // Force one partner per concurrency batch so responses span batches. - settings.ec.pull_sync_concurrency = 1; + settings.ec.identity.max_concurrency = 1; let registry = PartnerRegistry::from_config(&[ pull_enabled_ec_partner("alpha.example.com"), pull_enabled_ec_partner("beta.example.com"), @@ -1002,7 +1002,7 @@ mod tests { #[test] fn dispatch_pull_sync_calls_and_persists_only_missing_partner() { let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; + settings.ec.identity.max_concurrency = 4; let registry = PartnerRegistry::from_config(&[ pull_enabled_ec_partner("alpha.example.com"), pull_enabled_ec_partner("beta.example.com"), @@ -1065,7 +1065,7 @@ mod tests { #[test] fn dispatch_pull_sync_skips_non_present_snapshots() { let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; + settings.ec.identity.max_concurrency = 4; let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) .expect("should build registry"); @@ -1110,7 +1110,7 @@ mod tests { #[test] fn dispatch_pull_sync_skips_tombstone_snapshot() { let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; + settings.ec.identity.max_concurrency = 4; let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) .expect("should build registry"); @@ -1150,7 +1150,7 @@ mod tests { // it must revalidate the live row and cancel rather than leak an // identity the user just withdrew. let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; + settings.ec.identity.max_concurrency = 4; let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) .expect("should build registry"); @@ -1205,7 +1205,7 @@ mod tests { // authorize outbound calls; with no calls to authorize it must not cost // a KV operation. let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; + settings.ec.identity.max_concurrency = 4; let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) .expect("should build registry"); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 3efc157701..e583508508 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -6,7 +6,7 @@ use regex::Regex; use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -20,6 +20,7 @@ use crate::cache_policy::{CachePolicy, CacheVisibility}; use crate::consent_config::ConsentConfig; use crate::constants::INTERNAL_HEADERS; use crate::creative_opportunities::CreativeOpportunitiesConfig; +use crate::ec::identity::IdentityProviderId; use crate::error::TrustedServerError; use crate::host_header::validate_host_header_override_value; use crate::platform::PlatformImageOptimizerRegion; @@ -548,9 +549,17 @@ pub struct Ec { #[serde(default)] pub ec_store: Option, - /// Maximum number of concurrent pull-sync requests. - #[serde(default = "Ec::default_pull_sync_concurrency")] - pub pull_sync_concurrency: usize, + /// Server-side identity provider policy (`[ec.identity]`). + #[serde(default)] + pub identity: IdentitySettings, + + /// Catches the removed `pull_sync_concurrency` key. + /// + /// The key was replaced by `[ec.identity].max_concurrency`. It is parsed + /// only so validation can name its replacement instead of failing with a + /// generic unknown-field message, and it is never serialized. + #[serde(default, rename = "pull_sync_concurrency", skip_serializing)] + pub removed_pull_sync_concurrency: Option, /// Entries with `cluster_size` at or below this value are treated as /// individual users for identity resolution. B2B publishers should @@ -582,12 +591,6 @@ impl Ec { "replace-with-random-ec-passphrase", ]; - /// Default maximum concurrent pull-sync requests. - #[must_use] - pub const fn default_pull_sync_concurrency() -> usize { - 3 - } - /// Default cluster trust threshold. #[must_use] pub const fn default_cluster_trust_threshold() -> u32 { @@ -633,6 +636,195 @@ impl Ec { } } +/// Server-side identity provider policy. +/// +/// Mapped from the `[ec.identity]` TOML section. Unlike the rest of the +/// settings, this table and its provider tables accept unknown keys, so a +/// rollback to an earlier release still loads a config written for a later one. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdentitySettings { + /// Maximum providers dispatched at once. + #[serde(default = "IdentitySettings::default_max_concurrency")] + pub max_concurrency: usize, + + /// Total time a navigation may wait for hold-mode providers. + #[serde(default = "IdentitySettings::default_hold_budget_ms")] + pub hold_budget_ms: u64, + + /// Seconds before a provider that timed out in hold mode is held again. + #[serde(default = "IdentitySettings::default_hold_retry_sec")] + pub hold_retry_sec: u64, + + /// Largest provider response body accepted, in bytes. + #[serde(default = "IdentitySettings::default_max_response_bytes")] + pub max_response_bytes: usize, + + /// Per-provider policy, keyed by identity provider ID. + #[serde(default)] + pub providers: BTreeMap, +} + +impl IdentitySettings { + /// Default maximum concurrent providers. + #[must_use] + pub const fn default_max_concurrency() -> usize { + 4 + } + + /// Default hold budget in milliseconds. + #[must_use] + pub const fn default_hold_budget_ms() -> u64 { + 100 + } + + /// Default hold retry interval in seconds. + #[must_use] + pub const fn default_hold_retry_sec() -> u64 { + 300 + } + + /// Default provider response cap in bytes. + #[must_use] + pub const fn default_max_response_bytes() -> usize { + 65_536 + } + + /// Returns the policy for `id`, or the defaults when it has no table. + #[must_use] + pub fn provider(&self, id: &IdentityProviderId) -> ProviderSettings { + self.providers.get(id.as_str()).cloned().unwrap_or_default() + } +} + +impl Default for IdentitySettings { + fn default() -> Self { + Self { + max_concurrency: Self::default_max_concurrency(), + hold_budget_ms: Self::default_hold_budget_ms(), + hold_retry_sec: Self::default_hold_retry_sec(), + max_response_bytes: Self::default_max_response_bytes(), + providers: BTreeMap::new(), + } + } +} + +/// How a provider's lookup interacts with the navigation response. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ProviderMode { + /// Wait for the provider, within the hold budget, before responding. + Hold, + /// Dispatch after the response is sent. + #[default] + AfterSend, +} + +/// How far a revoke signal from a provider reaches. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RevokeScope { + /// Revoke only the signalling provider's identifiers. + Provider, + /// Revoke every identifier of the user. + Global, +} + +/// Policy for one identity provider (`[ec.identity.providers.]`). +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ProviderSettings { + /// Hold or after-send dispatch. + #[serde(default)] + pub mode: ProviderMode, + + /// Hold-mode timeout; the hold budget when unset. + #[serde(default)] + pub timeout_ms: Option, + + /// Timeout for any dispatch after the response is sent. + #[serde(default = "ProviderSettings::default_after_send_timeout_ms")] + pub after_send_timeout_ms: u64, + + /// Source domains this provider may write; the descriptor's when unset. + #[serde(default)] + pub sources: Option>, + + /// Seconds before a stored ID is refreshed. + #[serde(default = "ProviderSettings::default_refresh_after_sec")] + pub refresh_after_sec: u64, + + /// Provider calls allowed per minute. + #[serde(default = "ProviderSettings::default_max_calls_per_minute")] + pub max_calls_per_minute: u32, + + /// Failures per minute that open the circuit breaker. + #[serde(default = "ProviderSettings::default_breaker_failures_per_minute")] + pub breaker_failures_per_minute: u32, + + /// Ordering of this provider's ID in the cookie when it must be trimmed. + #[serde(default)] + pub cookie_priority: i32, + + /// Scope of this provider's revoke signals; `Provider` when unset. + #[serde(default)] + pub revoke_scope: Option, +} + +impl ProviderSettings { + /// Default after-send timeout in milliseconds. + #[must_use] + pub const fn default_after_send_timeout_ms() -> u64 { + 1000 + } + + /// Default refresh interval in seconds. + #[must_use] + pub const fn default_refresh_after_sec() -> u64 { + 86_400 + } + + /// Default provider calls per minute. + #[must_use] + pub const fn default_max_calls_per_minute() -> u32 { + 1 + } + + /// Default breaker failures per minute. + #[must_use] + pub const fn default_breaker_failures_per_minute() -> u32 { + 50 + } + + /// Hold-mode timeout: `timeout_ms`, else the hold budget. + /// + /// The coordinator caps it at the remaining hold budget. + #[must_use] + pub fn hold_timeout_ms(&self, identity: &IdentitySettings) -> u64 { + self.timeout_ms.unwrap_or(identity.hold_budget_ms) + } + + /// The revoke scope, defaulting to [`RevokeScope::Provider`]. + #[must_use] + pub fn revoke_scope(&self) -> RevokeScope { + self.revoke_scope.unwrap_or(RevokeScope::Provider) + } +} + +impl Default for ProviderSettings { + fn default() -> Self { + Self { + mode: ProviderMode::default(), + timeout_ms: None, + after_send_timeout_ms: Self::default_after_send_timeout_ms(), + sources: None, + refresh_after_sec: Self::default_refresh_after_sec(), + max_calls_per_minute: Self::default_max_calls_per_minute(), + breaker_failures_per_minute: Self::default_breaker_failures_per_minute(), + cookie_priority: 0, + revoke_scope: None, + } + } +} + #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] pub struct Rewrite { @@ -3813,6 +4005,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::config::validate_identity_settings; use regex::Regex; use serde_json::json; use std::collections::BTreeSet; @@ -8511,4 +8704,160 @@ formats = [{{ width = 300, height = 250 }}] ); } } + + #[test] + fn identity_settings_defaults_without_any_table() { + let settings: IdentitySettings = toml::from_str("").expect("should parse an empty table"); + assert_eq!( + settings.max_concurrency, 4, + "should default the concurrency to 4" + ); + assert_eq!( + settings.hold_budget_ms, 100, + "should default the hold budget to 100 ms" + ); + assert_eq!( + settings.hold_retry_sec, 300, + "should default the hold retry to 300 s" + ); + assert_eq!( + settings.max_response_bytes, 65_536, + "should default the response cap to 64 KiB" + ); + let provider = + settings.provider(&"anything".parse().expect("should parse the provider id")); + assert_eq!( + provider.mode, + ProviderMode::AfterSend, + "should default to after-send mode" + ); + assert_eq!( + provider.after_send_timeout_ms, 1000, + "should default the after-send timeout to 1 s" + ); + assert_eq!( + provider.hold_timeout_ms(&settings), + 100, + "should default the hold timeout to the budget" + ); + assert_eq!( + ( + provider.refresh_after_sec, + provider.max_calls_per_minute, + provider.breaker_failures_per_minute, + provider.cookie_priority + ), + (86_400, 1, 50, 0), + "should use the spec 10.1 provider defaults" + ); + assert_eq!( + provider.revoke_scope(), + RevokeScope::Provider, + "should scope revokes to the provider by default" + ); + validate_identity_settings(&settings).expect("should accept the defaults"); + } + + #[test] + fn identity_settings_default_matches_an_empty_table() { + let parsed: IdentitySettings = toml::from_str("").expect("should parse an empty table"); + assert_eq!( + serde_json::to_value(IdentitySettings::default()) + .expect("should serialize the default"), + serde_json::to_value(parsed).expect("should serialize the parsed value"), + "should give Default the serde defaults, not zeros" + ); + let provider: ProviderSettings = + toml::from_str("").expect("should parse an empty provider table"); + assert_eq!( + serde_json::to_value(ProviderSettings::default()) + .expect("should serialize the default"), + serde_json::to_value(provider).expect("should serialize the parsed value"), + "should give the provider Default the serde defaults" + ); + } + + #[test] + fn identity_settings_timeouts_and_bounds_are_enforced() { + let settings: IdentitySettings = toml::from_str( + "hold_budget_ms = 120\n[providers.id5]\nmode = \"hold\"\nafter_send_timeout_ms = 2500\n", + ) + .expect("should parse a hold-mode table"); + let provider = settings.provider(&"id5".parse().expect("should parse the provider id")); + assert_eq!( + provider.hold_timeout_ms(&settings), + 120, + "should use the budget as the hold timeout" + ); + assert_eq!( + provider.after_send_timeout_ms, 2500, + "should read the after-send timeout" + ); + validate_identity_settings(&settings).expect("should accept a 2.5 s after-send timeout"); + + for (name, toml_text) in [ + ("budget over 1000", "hold_budget_ms = 1001"), + ("zero concurrency", "max_concurrency = 0"), + ("zero refresh", "[providers.p]\nrefresh_after_sec = 0"), + ( + "zero after-send timeout", + "[providers.p]\nafter_send_timeout_ms = 0", + ), + ( + "after-send timeout over 10 s", + "[providers.p]\nafter_send_timeout_ms = 10001", + ), + ( + "hold timeout over budget", + "[providers.p]\nmode = \"hold\"\ntimeout_ms = 101", + ), + ] { + let settings: IdentitySettings = + toml::from_str(toml_text).expect("should parse before validation"); + assert!( + validate_identity_settings(&settings).is_err(), + "should reject {name}" + ); + } + let edge: IdentitySettings = toml::from_str("[providers.p]\nafter_send_timeout_ms = 10000") + .expect("should parse the upper bound"); + validate_identity_settings(&edge).expect("control: should accept exactly 10 s"); + toml::from_str::("[providers.p]\nmode = \"sometimes\"") + .expect_err("should reject an unknown mode"); + toml::from_str::("[providers.p]\nrevoke_scope = \"everyone\"") + .expect_err("should reject an unknown revoke scope"); + } + + #[test] + fn identity_settings_revoke_scope_parses_and_keeps_its_default_when_unset() { + let settings: IdentitySettings = + toml::from_str("[providers.lockr]\nrevoke_scope = \"global\"\n") + .expect("should parse the revoke scope"); + let lockr = settings.provider(&"lockr".parse().expect("should parse the provider id")); + assert_eq!( + lockr.revoke_scope(), + RevokeScope::Global, + "should read a global revoke scope" + ); + let other = settings.provider(&"other".parse().expect("should parse the provider id")); + assert_eq!( + other.revoke_scope(), + RevokeScope::Provider, + "control: another provider should keep the provider scope" + ); + toml::from_str::("[providers.lockr]\ncapture_enabled = true\n") + .expect("should still load a key this build does not know"); + } + + #[test] + fn identity_settings_ec_without_identity_table_parses() { + let settings = Settings::from_toml(&crate::test_support::tests::crate_test_settings_str()) + .expect("should parse settings without an identity table"); + assert_eq!( + serde_json::to_value(&settings.ec.identity).expect("should serialize the parsed table"), + serde_json::to_value(IdentitySettings::default()) + .expect("should serialize the default"), + "should give an Ec without [ec.identity] the defaults" + ); + } } diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index c3d9ff53d8..02f597a134 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -12,7 +12,9 @@ proxy_secret = "integration_proxy_secret" [ec] passphrase = "integration_ec_passphrase" ec_store = "ec_identity_store" -pull_sync_concurrency = 3 + +[ec.identity] +max_concurrency = 3 [[ec.partners]] name = "Integration Test Partner" diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index eeed15c2c4..fa436e2ac4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -634,6 +634,31 @@ The legacy `[consent].consent_store` setting has been removed. Trusted Server us Legacy consent-store records are not read or migrated into `ec.ec_store`. Their payload schema is not authoritative EC lifecycle state, so do not copy those records into the identity store. You may retain the old store unchanged for a defined rollback window, then unlink its platform resource binding and delete it. No browser-cookie or EC identity-store migration is required. +### Identity Providers (`[ec.identity]`) + +Policy for server-side identity providers. The table is optional; every key has a default. `ec.pull_sync_concurrency` was removed: a config that sets it fails validation. Use `max_concurrency` instead. + +| Field | Type | Default | Description | +| -------------------- | ------- | ------- | ----------------------------------------------------------------------- | +| `max_concurrency` | Integer | `4` | Maximum providers dispatched at once (at least 1) | +| `hold_budget_ms` | Integer | `100` | Total time a navigation may wait for hold-mode providers (at most 1000) | +| `hold_retry_sec` | Integer | `300` | Seconds before a provider that timed out in hold mode is held again | +| `max_response_bytes` | Integer | `65536` | Largest provider response body accepted | + +Each provider can have an `[ec.identity.providers.]` table: + +| Field | Type | Default | Description | +| ----------------------------- | ------- | ------------ | ------------------------------------------------------------------ | +| `mode` | String | `after_send` | `hold` or `after_send` | +| `timeout_ms` | Integer | none | Hold-mode timeout, at most `hold_budget_ms`; the budget when unset | +| `after_send_timeout_ms` | Integer | `1000` | Timeout for any dispatch after send, `1` to `10000` | +| `sources` | Array | none | Source domains the provider may write | +| `refresh_after_sec` | Integer | `86400` | Seconds before a stored ID is refreshed (greater than 0) | +| `max_calls_per_minute` | Integer | `1` | Provider calls allowed per minute | +| `breaker_failures_per_minute` | Integer | `50` | Failures per minute that open the circuit breaker | +| `cookie_priority` | Integer | `0` | Ordering of the provider's ID when the cookie is trimmed | +| `revoke_scope` | String | `provider` | `provider` or `global` | + ### `[ec]` `passphrase` is a key name in `trusted_server_secrets`; the resolved value must @@ -643,7 +668,6 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. | ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | | `passphrase` | String | Yes | Publisher passphrase used as HMAC key | | `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | | `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | | `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | | `partners` | Array | No | Static partner registry entries | diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 066ad659e3..d84e455c27 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -79,12 +79,32 @@ passphrase = "ec_passphrase" # bound per adapter (e.g. `ec_identity_store` in fastly.toml); edgezero.toml's # logical KV id is `trusted_server_kv`. ec_store = "ec_identity_store" -# Max concurrent partner pull-sync requests. -pull_sync_concurrency = 3 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds +# Server-side identity provider policy (defaults shown except max_concurrency). +[ec.identity] +# Max identity providers dispatched at once. +max_concurrency = 3 +# Total time a navigation may wait for hold-mode providers (at most 1000). +# hold_budget_ms = 100 +# Seconds before a provider that timed out in hold mode is held again. +# hold_retry_sec = 300 +# Largest provider response body accepted, in bytes. +# max_response_bytes = 65536 + +# Per-provider policy; the table name is the identity provider ID. +# [ec.identity.providers.example-provider] +# mode = "after_send" # or "hold" +# timeout_ms = 100 # hold mode only; at most hold_budget_ms +# after_send_timeout_ms = 1000 # 1..=10000 +# refresh_after_sec = 86400 +# max_calls_per_minute = 1 +# breaker_failures_per_minute = 50 +# cookie_priority = 0 +# revoke_scope = "provider" # or "global" + # Example partner configuration. Provision referenced keys in # trusted_server_secrets before validating/pushing. # [[ec.partners]] From 518dcccb161d480c9ee823fbc9d682777a8630eb Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 21:06:31 +0530 Subject: [PATCH 27/89] Add coordinator eligibility and outcome rules Eligibility reads the KV row alone. Only a returning writer runs providers; a hold-mode provider moves after send when the navigation runs no auction. Provider results become whole entry sets per source, an invalidation carries no hash until it applies, a discarded set rejects a stateful provider's positive part, and a result whose every set is discarded is Malformed. An opt-out's fingerprint is a plain consent bitmask. Signed-off-by: dhruv8sh --- .../src/ec/identity/catalog.rs | 10 +- .../src/ec/identity/coordinator.rs | 2937 +++++++++++++++++ .../src/ec/identity/descriptor.rs | 22 +- .../src/ec/identity/exchange.rs | 10 +- .../src/ec/identity/input.rs | 24 +- .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/mutation.rs | 76 +- .../src/ec/identity/policy.rs | 2 +- .../src/ec/identity/resolution.rs | 12 +- .../src/ec/identity/revision.rs | 8 + .../src/ec/identity/test_support.rs | 168 +- .../src/migration_guards.rs | 4 + 12 files changed, 3233 insertions(+), 41 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/coordinator.rs diff --git a/crates/trusted-server-core/src/ec/identity/catalog.rs b/crates/trusted-server-core/src/ec/identity/catalog.rs index 62cc02b9fa..8722aeda2e 100644 --- a/crates/trusted-server-core/src/ec/identity/catalog.rs +++ b/crates/trusted-server-core/src/ec/identity/catalog.rs @@ -1,9 +1,5 @@ //! The enabled identity providers and source ownership. -// Temporary: the narrowing, claim and capture-only accessors are first used by -// later identity tasks. Remove this when they land. -#![allow(dead_code)] - use std::collections::BTreeMap; use std::sync::Arc; @@ -52,6 +48,8 @@ impl ProviderCatalog { /// (spec 10.1). Config validation has rejected a source a descriptor /// does not declare and a narrowing to nothing. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn with_narrowing( mut self, narrowing: BTreeMap>, @@ -71,6 +69,8 @@ impl ProviderCatalog { } /// Every provider, ordered by ID. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn providers(&self) -> impl Iterator> { self.providers.values() } @@ -94,6 +94,8 @@ impl ProviderCatalog { } /// Every provider that claims `source` after narrowing, ordered by ID. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn claimants<'a>( &'a self, source: &'a SourceDomain, diff --git a/crates/trusted-server-core/src/ec/identity/coordinator.rs b/crates/trusted-server-core/src/ec/identity/coordinator.rs new file mode 100644 index 0000000000..123efff7aa --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/coordinator.rs @@ -0,0 +1,2937 @@ +//! Coordinator eligibility and outcomes (spec 4.7 and 8). +//! +//! Pure logic: which of a provider's sources need a call, and what stored +//! changes a call's outcome becomes. Dispatch, the breaker and the burst cap +//! live elsewhere. + +// Temporary: the dispatcher, the first non-test caller, arrives in a later +// task. Remove this when it lands. +#![allow(dead_code)] + +use std::collections::{BTreeMap, BTreeSet}; + +use super::catalog::ProviderCatalog; +use super::descriptor::{ + ConsentPolicy, Input, ProviderDescriptor, ResolveKind, Resolves, UsOptOut, +}; +use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::mutation::{PartnerIdChange, ProviderResult, ProviderStateChange, StateUpdate}; +use super::policy::{PolicySignals, policy_allows, tcf_applies, us_sale_opt_out}; +use super::provider::IdentityProvider; +use super::resolution::{EntrySet, EntrySets, Resolution, ResolvedEid, StateChange, entry_sets}; +use super::revision::{StateRevision, record_revision, uid_hash}; +use crate::consent::{ConsentContext, effective_tcf}; +use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId}; +use crate::settings::{IdentitySettings, ProviderMode, ProviderSettings}; + +/// The highest TCF purpose the fingerprint has a bit for. +const FINGERPRINT_PURPOSES: u8 = 10; +/// Fingerprint bit of the vendor consent. Purposes 1 to 10 hold bits 0 to 9. +const FINGERPRINT_VENDOR_BIT: u32 = 10; +/// Fingerprint bit of the US sale opt-out. +const FINGERPRINT_SALE_BIT: u32 = 11; +/// Fingerprint bit of Global Privacy Control. +const FINGERPRINT_GPC_BIT: u32 = 12; +/// Fingerprint bit of `DNT: 1`. +const FINGERPRINT_DNT_BIT: u32 = 13; + +/// How long a burst-capped provider waits: one limiter window. +const BURST_WAIT_SECS: u64 = 60; +/// The wait after a `no_match` that gives no `retry_in`. +const NO_MATCH_WAIT_SECS: u64 = 86_400; +/// The longest `retry_in` the coordinator honours. +const MAX_RETRY_SECS: u64 = 7 * 86_400; +/// First wait after an input-caused failure. +const INPUT_BACKOFF_BASE_SECS: u64 = 3_600; +/// Longest wait after an input-caused failure. +const INPUT_BACKOFF_CAP_SECS: u64 = 86_400; +/// First wait after a transport failure. +const TRANSPORT_BACKOFF_BASE_SECS: u64 = 300; +/// Longest wait after a transport failure. +const TRANSPORT_BACKOFF_CAP_SECS: u64 = 3_600; + +/// One provider's `ps` entry, as the request read it. +#[derive(Debug, Clone, Default)] +struct ProviderView { + n: Option, + f: u32, + o: Option, + oc: Option, + iv: Vec, + ov: Vec, + st: Option, +} + +/// What eligibility and outcomes read from the KV row the request read +/// (spec 4.7), and nothing else. It never reads `ts-eids-s`: every removal is +/// already in KV (spec 5.3). +#[derive(Debug, Clone, Default)] +pub(crate) struct EligibilityView { + records: BTreeMap, + providers: BTreeMap, +} + +impl EligibilityView { + /// The view of the row the request read. A key that is not a valid + /// source domain or provider ID cannot name a configured one and is + /// skipped. + #[must_use] + pub(crate) fn from_kv(entry: &KvEntry) -> Self { + let records = entry + .ids + .iter() + .filter_map(|(source, record)| { + Some((source.parse::().ok()?, record.clone())) + }) + .collect(); + let providers = entry + .ps + .iter() + .filter_map(|(provider, state)| { + let view = ProviderView { + n: state.n, + f: state.f, + o: state.o.clone(), + oc: state.oc, + iv: state.iv.clone(), + ov: state.ov.clone(), + st: state.st.clone(), + }; + Some((provider.parse::().ok()?, view)) + }) + .collect(); + Self { records, providers } + } + + /// A view of a row with no records and no provider state. + #[must_use] + pub(crate) fn empty() -> Self { + Self::default() + } + + /// The whole record of `source`, if the row holds one. + #[must_use] + pub(crate) fn record(&self, source: &SourceDomain) -> Option<&KvPartnerId> { + self.records.get(source) + } + + /// Every stored record. + pub(crate) fn records(&self) -> impl Iterator { + self.records.iter() + } + + /// The provider's consecutive failures, `f`. + #[must_use] + pub(crate) fn failures(&self, provider: &IdentityProviderId) -> u32 { + self.providers.get(provider).map_or(0, |view| view.f) + } + + /// The provider's `n`: the earliest time it may be called. + #[must_use] + pub(crate) fn next_eligible(&self, provider: &IdentityProviderId) -> Option { + self.providers.get(provider).and_then(|view| view.n) + } + + /// The provider's private state `st`, which a call reads. + #[must_use] + pub(crate) fn state(&self, provider: &IdentityProviderId) -> Option<&serde_json::Value> { + self.providers + .get(provider) + .and_then(|view| view.st.as_ref()) + } + + /// Whether the provider's opt-out removed a UID with this hash, `ps.ov`. + #[must_use] + pub(crate) fn has_opt_out_hash(&self, provider: &IdentityProviderId, hash: &str) -> bool { + self.providers + .get(provider) + .is_some_and(|view| view.ov.iter().any(|known| known == hash)) + } + + /// Whether the provider invalidated a UID with this hash, `ps.iv`. + fn has_invalidated_hash(&self, provider: &IdentityProviderId, hash: &str) -> bool { + self.providers + .get(provider) + .is_some_and(|view| view.iv.iter().any(|known| known == hash)) + } + + /// Whether `o` is set and `oc` is not strictly later than `o.at`. + #[must_use] + pub(crate) fn is_opted_out(&self, provider: &IdentityProviderId) -> bool { + self.providers.get(provider).is_some_and(|view| { + view.o + .as_ref() + .is_some_and(|opt_out| view.oc.is_none_or(|clear| clear <= opt_out.at)) + }) + } + + #[cfg(test)] + fn provider_mut(&mut self, provider: &str) -> &mut ProviderView { + self.providers + .entry(provider.parse().expect("should parse the provider ID")) + .or_default() + } + + #[cfg(test)] + pub(crate) fn set_record_for_test(&mut self, source: &SourceDomain, record: KvPartnerId) { + self.records.insert(source.clone(), record); + } + + #[cfg(test)] + pub(crate) fn set_failures_for_test(&mut self, provider: &str, failures: u32) { + self.provider_mut(provider).f = failures; + } + + #[cfg(test)] + pub(crate) fn set_next_for_test(&mut self, provider: &str, next: u64) { + self.provider_mut(provider).n = Some(next); + } + + #[cfg(test)] + pub(crate) fn set_invalidated_for_test(&mut self, provider: &str, hashes: Vec) { + self.provider_mut(provider).iv = hashes; + } + + #[cfg(test)] + pub(crate) fn set_opt_out_for_test(&mut self, provider: &str, at: u64, clear: Option) { + let view = self.provider_mut(provider); + view.o = Some(KvOptOut { + at, + fp: "0".to_owned(), + }); + view.oc = clear; + } +} + +/// What the request is, as eligibility reads it. +#[derive(Debug, Clone)] +pub(crate) struct RequestFacts { + /// The client looks like a real browser (step 1). + pub(crate) is_real_browser: bool, + /// Global consent allows EC and EID persistence (step 2). + pub(crate) ec_persistence_allowed: bool, + /// The signals the consent context does not carry (step 3). + pub(crate) policy_signals: PolicySignals, + /// This is the writer request, not a stager (step 7). + pub(crate) writer: bool, + /// This is a new visitor's first navigation (step 7). + pub(crate) new_visitor: bool, + /// The navigation runs an auction (step 7). + pub(crate) runs_auction: bool, + /// The request's time. + pub(crate) now: UnixTime, + /// The inputs this request can supply (step 4). + pub(crate) available_inputs: BTreeSet, +} + +/// When a provider is dispatched, relative to the navigation response. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Phase { + /// Before the auction, within the hold budget. + Hold, + /// After the response is sent. + AfterSend, +} + +/// Why a provider was not eligible. Each variant names the step of spec 4.7 +/// that stopped it. The breaker (step 6) and the burst cap (step 8) are +/// checked elsewhere. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SkipReason { + /// The provider declares no `resolves`, so it never makes a call. + NotAResolver, + /// Step 1: the client does not look like a real browser. + NotRealBrowser, + /// Step 2: global consent does not allow EID persistence. + EcPersistenceNotAllowed, + /// Step 3: the provider's consent policy is not met. + ConsentPolicy, + /// Step 4: no source needs a call this time. + NoSourceDue, + /// Step 5: `ps.n` has not passed. + NotYetEligible, + /// Step 5: the visitor opted out of the provider. + OptedOut, + /// Step 7: this is a new visitor's first navigation. + NewVisitor, + /// Step 7: this request is a stager. + NotWriter, + /// Step 7: the provider's mode runs it in the other phase. + WrongPhase, +} + +/// Whether a provider is eligible. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Eligibility { + /// It is, for these sources. + Eligible { + kinds: BTreeMap, + }, + /// It is not. + Skip(SkipReason), +} + +/// What [`eligibility`] reads about one provider. +pub(crate) struct EligibilityInput<'a> { + pub(crate) provider: &'a dyn IdentityProvider, + pub(crate) settings: &'a ProviderSettings, + pub(crate) identity: &'a IdentitySettings, + pub(crate) catalog: &'a ProviderCatalog, + pub(crate) consent: &'a ConsentContext, +} + +/// Whether the provider is eligible for this request, steps 1 to 7 of spec +/// 4.7 in order, and which sources need which kind of call. +#[must_use] +pub(crate) fn eligibility( + input: &EligibilityInput<'_>, + view: &EligibilityView, + facts: &RequestFacts, + phase: Phase, +) -> Eligibility { + let descriptor = input.provider.descriptor(); + let id = descriptor.id(); + if descriptor.resolves().is_none() { + return Eligibility::Skip(SkipReason::NotAResolver); + } + if !facts.is_real_browser { + return Eligibility::Skip(SkipReason::NotRealBrowser); + } + if !facts.ec_persistence_allowed { + return Eligibility::Skip(SkipReason::EcPersistenceNotAllowed); + } + if !policy_allows( + descriptor.consent_policy(), + input.consent, + facts.policy_signals, + ) { + return Eligibility::Skip(SkipReason::ConsentPolicy); + } + let kinds = source_kinds(input, view, facts); + if kinds.is_empty() { + log::debug!("Skipping identity provider '{id}': no source needs a call"); + return Eligibility::Skip(SkipReason::NoSourceDue); + } + if view + .next_eligible(id) + .is_some_and(|next| facts.now.as_secs() < next) + { + return Eligibility::Skip(SkipReason::NotYetEligible); + } + if view.is_opted_out(id) { + return Eligibility::Skip(SkipReason::OptedOut); + } + if facts.new_visitor { + return Eligibility::Skip(SkipReason::NewVisitor); + } + if !facts.writer { + return Eligibility::Skip(SkipReason::NotWriter); + } + let runs_in_phase = match (phase, input.settings.mode) { + (Phase::Hold, ProviderMode::Hold) => facts.runs_auction, + (Phase::AfterSend, ProviderMode::AfterSend) => true, + (Phase::AfterSend, ProviderMode::Hold) => !facts.runs_auction, + (Phase::Hold, ProviderMode::AfterSend) => false, + }; + if !runs_in_phase { + return Eligibility::Skip(SkipReason::WrongPhase); + } + Eligibility::Eligible { kinds } +} + +/// Step 4: the kind each claimed source needs, leaving out a source whose +/// kind the provider does not list or whose needs the request cannot meet. +fn source_kinds( + input: &EligibilityInput<'_>, + view: &EligibilityView, + facts: &RequestFacts, +) -> BTreeMap { + let descriptor = input.provider.descriptor(); + let Some(resolves) = descriptor.resolves() else { + return BTreeMap::new(); + }; + input + .catalog + .sources_of(descriptor.id()) + .iter() + .filter_map(|source| { + let kind = source_kind(input, view, facts, source)?; + let listed = resolves.kinds().contains(&kind); + let met = resolves.needs(kind).is_subset(&facts.available_inputs); + (listed && met).then(|| (source.clone(), kind)) + }) + .collect() +} + +fn source_kind( + input: &EligibilityInput<'_>, + view: &EligibilityView, + facts: &RequestFacts, + source: &SourceDomain, +) -> Option { + let id = input.provider.descriptor().id(); + let record = view.record(source); + match input.catalog.owner_of(source) { + Some(owner) if owner == id => match record { + None => Some(ResolveKind::Initial), + Some(record) => { + owned_record_is_due(input, view, facts, record).then_some(ResolveKind::Refresh) + } + }, + // Config validation lets one provider own a source, so another + // owner means this provider may not resolve it. + Some(_) => None, + None => { + let present = + record.is_some_and(|record| has_unexpired_id(input.catalog, record, facts.now)); + (!present).then_some(ResolveKind::Initial) + } + } +} + +/// Whether a stored record of an owned source is due (spec 4.7 step 4). +fn owned_record_is_due( + input: &EligibilityInput<'_>, + view: &EligibilityView, + facts: &RequestFacts, + record: &KvPartnerId, +) -> bool { + let id = input.provider.descriptor().id(); + let now = facts.now.as_secs(); + let invalidated = record.w.is_none() + && record + .eids + .iter() + .flat_map(|entry| &entry.uids) + .any(|uid| view.has_invalidated_hash(id, &uid_hash(&uid.id))); + if invalidated { + return false; + } + if record.exp.is_some_and(|exp| now >= exp) || record.w.as_deref() != Some(id.as_str()) { + return true; + } + match record.ra { + Some(refresh_at) => now >= refresh_at, + None => record.at.is_none_or(|acquired| { + acquired.saturating_add(input.settings.refresh_after_sec) <= now + }), + } +} + +/// Whether KV holds an unexpired ID in this record for an unowned source. A +/// record hidden only by consent or an opt-out still counts: `FillMissing` +/// would not replace it. +fn has_unexpired_id(catalog: &ProviderCatalog, record: &KvPartnerId, now: UnixTime) -> bool { + if record.eids.is_empty() || record.exp.is_some_and(|exp| now.as_secs() >= exp) { + return false; + } + record.w.as_deref().is_none_or(|writer| { + writer + .parse::() + .is_ok_and(|writer| catalog.is_configured(&writer)) + }) +} + +/// The consent an opt-out's `fp` records, as a lowercase hex bitmask (spec 8). +/// +/// Each signal has a fixed bit, whatever the policy reads: +/// +/// - bits 0 to 9: TCF purposes 1 to 10, set when consent grants the purpose +/// - bit 10: the policy's vendor, set when consent grants it +/// - bit 11: the US sale opt-out, from `us_privacy` or GPP +/// - bit 12: GPC +/// - bit 13: DNT +/// +/// A signal the policy does not read or relaxes stays 0. Later parts append +/// bits after these and keep these where they are. +#[must_use] +pub(crate) fn fingerprint_bits( + policy: &ConsentPolicy, + consent: &ConsentContext, + signals: PolicySignals, +) -> String { + let mut bits: u64 = 0; + if !policy.is_eid_persistence_only() { + if tcf_applies(consent) + && let Some(tcf) = effective_tcf(consent) + { + for purpose in policy.tcf_purposes() { + if (1..=FINGERPRINT_PURPOSES).contains(purpose) + && tcf.has_purpose_consent(usize::from(*purpose)) + { + bits |= 1 << (purpose - 1); + } + } + if let Some(vendor) = policy.tcf_vendor() + && tcf.has_vendor_consent(vendor) + { + bits |= 1 << FINGERPRINT_VENDOR_BIT; + } + } + if !policy.allows_us_opt_out(UsOptOut::Sale) && us_sale_opt_out(consent) { + bits |= 1 << FINGERPRINT_SALE_BIT; + } + if !policy.allows_gpc() && consent.gpc { + bits |= 1 << FINGERPRINT_GPC_BIT; + } + if !policy.allows_dnt() && signals.dnt { + bits |= 1 << FINGERPRINT_DNT_BIT; + } + } + format!("{bits:x}") +} + +/// What one provider call came to. +#[derive(Debug)] +pub(crate) enum CallOutcome { + /// The provider returned a resolution. + Resolved(Resolution), + /// `request` returned `None`. + Skipped, + /// A declared input was unavailable. + InputUnavailable, + /// The stored state did not decode, and the resolution the provider + /// still returned, if any. + InvalidState(Option), + /// The hold budget ran out before the answer. + HoldCutOff, + /// The call failed because of this visitor's input (a 4xx other than 429, + /// `Config`, `UndeclaredInput` or `Malformed`). + InputFailure, + /// A transport failure, timeout, 429 or 5xx. + TransportFailure, + /// The circuit breaker was open, so nothing was sent. + BreakerOpen, + /// The burst cap stopped the call. + BurstCapped, + /// A limiter error meant nothing was sent. + NotDispatched, +} + +/// What an outcome needs besides the provider and the view. +pub(crate) struct OutcomeContext<'a> { + pub(crate) now: UnixTime, + pub(crate) catalog: &'a ProviderCatalog, + pub(crate) settings: &'a ProviderSettings, + pub(crate) identity: &'a IdentitySettings, + /// The state revision the call read, for a provider with private state. + pub(crate) expected_state: Option, + /// The consent bits an opt-out records, from [`fingerprint_bits`]. + pub(crate) fingerprint: &'a str, +} + +/// The stored changes of one outcome. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct OutcomeChanges { + /// Every removal, and the positive changes of a provider without private + /// state. + pub(crate) changes: Vec, + /// The positive changes, state change, `n` and `f` of a provider with + /// private state. They apply as one unit. + pub(crate) result: Option, + /// `n` and `f` of a provider without a result. + pub(crate) state: ProviderStateChange, +} + +/// The stored changes of an outcome: the table of spec 8, row by row. +#[must_use] +pub(crate) fn outcome_to_changes( + provider: &dyn IdentityProvider, + outcome: CallOutcome, + view: &EligibilityView, + context: &OutcomeContext<'_>, +) -> OutcomeChanges { + let id = provider.descriptor().id(); + let now = context.now.as_secs(); + match outcome { + CallOutcome::Resolved(resolution) => resolved_changes(provider, &resolution, view, context), + CallOutcome::Skipped | CallOutcome::InputUnavailable => { + wait_until(now.saturating_add(context.settings.refresh_after_sec)) + } + CallOutcome::InvalidState(resolution) => { + let mut changes = resolution.map_or_else(OutcomeChanges::default, |resolution| { + resolved_changes(provider, &resolution, view, context) + }); + let slot = state_slot(&mut changes); + slot.st = StateUpdate::Clear; + slot.f = None; + if slot.n.is_none() { + slot.n = Some(now.saturating_add(context.settings.refresh_after_sec)); + } + changes + } + CallOutcome::HoldCutOff => wait_until(now.saturating_add(context.identity.hold_retry_sec)), + CallOutcome::InputFailure => failure( + view.failures(id), + now, + INPUT_BACKOFF_BASE_SECS, + INPUT_BACKOFF_CAP_SECS, + ), + CallOutcome::TransportFailure => failure( + view.failures(id), + now, + TRANSPORT_BACKOFF_BASE_SECS, + TRANSPORT_BACKOFF_CAP_SECS, + ), + CallOutcome::BurstCapped => wait_until(now.saturating_add(BURST_WAIT_SECS)), + CallOutcome::BreakerOpen | CallOutcome::NotDispatched => OutcomeChanges::default(), + } +} + +/// The state change the outcome's `st`, `n` and `f` belong to: the result's +/// when there is one, the stand-alone one otherwise. +fn state_slot(changes: &mut OutcomeChanges) -> &mut ProviderStateChange { + match changes.result.as_mut() { + Some(result) => &mut result.state, + None => &mut changes.state, + } +} + +fn wait_until(next: u64) -> OutcomeChanges { + OutcomeChanges { + state: ProviderStateChange { + n: Some(next), + ..ProviderStateChange::default() + }, + ..OutcomeChanges::default() + } +} + +/// `base × 2^(failures-1)`, at most `cap`, in seconds. +fn backoff_secs(failures: u32, base: u64, cap: u64) -> u64 { + base.saturating_mul(2_u64.saturating_pow(failures.saturating_sub(1))) + .min(cap) +} + +fn failure(failures_before: u32, now: u64, base: u64, cap: u64) -> OutcomeChanges { + let failures = failures_before.saturating_add(1); + OutcomeChanges { + state: ProviderStateChange { + n: Some(now.saturating_add(backoff_secs(failures, base, cap))), + f: Some(failures), + ..ProviderStateChange::default() + }, + ..OutcomeChanges::default() + } +} + +/// Whether the provider declares private state. +fn keeps_private_state(descriptor: &ProviderDescriptor) -> bool { + descriptor + .resolves() + .is_some_and(Resolves::has_private_state) +} + +/// How `Resolved(resolution)` becomes changes (spec 4.5 and 8). +fn resolved_changes( + provider: &dyn IdentityProvider, + resolution: &Resolution, + view: &EligibilityView, + context: &OutcomeContext<'_>, +) -> OutcomeChanges { + let descriptor = provider.descriptor(); + let id = descriptor.id(); + let now = context.now.as_secs(); + let stateful = keeps_private_state(descriptor); + + let kept = drop_unclaimed(id, resolution.eids(), context.catalog); + let mut sets = entry_sets(&kept, context.now); + log_entry_sets(id, &sets); + let discarded = discarded_sets(descriptor, &mut sets, view); + let malformed = resolution.state_rejected() + || reject_positive_part(stateful, &discarded) + || all_sets_discarded(&sets, &discarded); + if malformed { + log::warn!("Identity provider '{id}' returned a result the coordinator rejects"); + } + + let positive = if malformed { + Vec::new() + } else { + positive_changes(provider, &sets, &discarded, view, context) + }; + let mut removals = invalidations(id, resolution, view, context.catalog); + let removed_any = !removals.is_empty(); + let mut state = ProviderStateChange::default(); + if resolution.is_opted_out() { + removals.extend(opt_out_removals(id, view, context.catalog)); + state.opt_out = Some(KvOptOut { + at: now, + fp: context.fingerprint.to_owned(), + }); + state.st = StateUpdate::Clear; + } + + let mut next = None; + if malformed { + let failures = view.failures(id).saturating_add(1); + state.f = Some(failures); + next = Some(now.saturating_add(backoff_secs( + failures, + INPUT_BACKOFF_BASE_SECS, + INPUT_BACKOFF_CAP_SECS, + ))); + } else { + if !resolution.eids().is_empty() { + state.f = Some(0); + } + if stateful && state.st == StateUpdate::Keep { + state.st = state_update(id, resolution); + } + if resolution.is_no_match() { + let wait = resolution + .retry_in() + .map_or(NO_MATCH_WAIT_SECS, |delay| delay.as_secs()) + .min(MAX_RETRY_SECS); + next = Some(now.saturating_add(wait)); + } else if removed_any { + next = Some(now); + } + } + if let Some(delay) = resolution.retry_in() { + let earliest = now.saturating_add(delay.as_secs().min(MAX_RETRY_SECS)); + next = Some(next.map_or(earliest, |current| current.max(earliest))); + } + state.n = next; + + if stateful && !malformed { + let result = (!positive.is_empty() || state != ProviderStateChange::default()).then(|| { + ProviderResult { + provider: id.clone(), + ids: positive, + state, + expected_state: context.expected_state.clone(), + } + }); + return OutcomeChanges { + changes: removals, + result, + state: ProviderStateChange::default(), + }; + } + removals.extend(positive); + OutcomeChanges { + changes: removals, + result: None, + state, + } +} + +/// Step 1: drops the entries of sources the provider does not claim after +/// narrowing. Logs each source with the provider ID only. +fn drop_unclaimed( + provider: &IdentityProviderId, + eids: &[ResolvedEid], + catalog: &ProviderCatalog, +) -> Vec { + let claimed = catalog.sources_of(provider); + eids.iter() + .filter(|eid| { + let keep = claimed.contains(eid.source()); + if !keep { + log::debug!( + "Dropping an entry of identity provider '{provider}' for unclaimed source '{}'", + eid.source() + ); + } + keep + }) + .cloned() + .collect() +} + +/// Step 2: logs each dropped part and oversized source, never a value. +fn log_entry_sets(provider: &IdentityProviderId, sets: &EntrySets) { + for drop in &sets.drops { + log::debug!( + "Identity provider '{provider}' source '{}': dropped {}", + drop.source, + drop.part + ); + } + for source in &sets.oversized { + log::warn!( + "Identity provider '{provider}' source '{source}': discarded an oversized entry set" + ); + } +} + +/// Step 3: the sources whose set is discarded. An oversized set is. A set of +/// an expiring source that carries no expiry keeps the stored `exp` when its +/// entries equal the stored `eids`, and is discarded otherwise. +fn discarded_sets( + descriptor: &ProviderDescriptor, + sets: &mut EntrySets, + view: &EligibilityView, +) -> BTreeSet { + let mut discarded: BTreeSet = sets.oversized.iter().cloned().collect(); + for (source, set) in &mut sets.sets { + if !descriptor.is_expiring(source) || set.expires_at.is_some() { + continue; + } + let stored_exp = view + .record(source) + .filter(|record| record.eids == set.entries) + .and_then(|record| record.exp); + match stored_exp { + Some(exp) => set.expires_at = Some(UnixTime::from_secs(exp)), + None => { + log::warn!( + "Identity provider '{}' source '{source}': discarded a set that carries no expiry", + descriptor.id() + ); + discarded.insert(source.clone()); + } + } + } + discarded +} + +/// Step 4: whether a discarded set rejects the whole positive part. It does +/// for a provider with private state, whose IDs and state apply as a unit. +fn reject_positive_part(stateful: bool, discarded: &BTreeSet) -> bool { + stateful && !discarded.is_empty() +} + +/// Step 5: whether the result named a source and every set is discarded. +fn all_sets_discarded(sets: &EntrySets, discarded: &BTreeSet) -> bool { + sets.all_discarded() + || (!sets.sets.is_empty() && sets.sets.keys().all(|source| discarded.contains(source))) +} + +/// Steps 6 to 8: each kept set becomes one record, written whole. +fn positive_changes( + provider: &dyn IdentityProvider, + sets: &EntrySets, + discarded: &BTreeSet, + view: &EligibilityView, + context: &OutcomeContext<'_>, +) -> Vec { + let descriptor = provider.descriptor(); + let id = descriptor.id(); + let stateful = keeps_private_state(descriptor); + sets.sets + .iter() + .filter(|(source, _)| !discarded.contains(*source)) + .map(|(source, set)| { + if set.refresh_until.is_some() && !stateful { + log::warn!( + "Identity provider '{id}' source '{source}': ignoring refresh_until without private state" + ); + } + let record = record_of(id, set, stateful, context.now); + if context.catalog.owner_of(source) == Some(id) { + PartnerIdChange::OwnerWrite { + source: source.clone(), + record, + expected: view.record(source).map(record_revision), + } + } else { + PartnerIdChange::FillMissing { + source: source.clone(), + record, + } + } + }) + .collect() +} + +fn record_of( + provider: &IdentityProviderId, + set: &EntrySet, + stateful: bool, + now: UnixTime, +) -> KvPartnerId { + KvPartnerId { + w: Some(provider.to_string()), + rv: None, + at: Some(now.as_secs()), + exp: set.expires_at.map(UnixTime::as_secs), + ra: set.refresh_at.map(UnixTime::as_secs), + ru: stateful + .then_some(set.refresh_until) + .flatten() + .map(UnixTime::as_secs), + eids: set.entries.clone(), + } +} + +/// Step 9: an `OwnerRemove` for each claimed source the result invalidated +/// that has a record. It carries no hash: applying it hashes the live record. +fn invalidations( + provider: &IdentityProviderId, + resolution: &Resolution, + view: &EligibilityView, + catalog: &ProviderCatalog, +) -> Vec { + let claimed = catalog.sources_of(provider); + resolution + .invalidated() + .iter() + .filter_map(|source| { + if !claimed.contains(source) { + log::debug!( + "Ignoring invalidation by identity provider '{provider}' of unclaimed source '{source}'" + ); + return None; + } + Some(PartnerIdChange::OwnerRemove { + source: source.clone(), + provider: provider.clone(), + expected: record_revision(view.record(source)?), + }) + }) + .collect() +} + +/// Step 10: a `WriterRemove` for every record this provider wrote, and for a +/// capture-only provider every record of its sources that has no writer. +fn opt_out_removals( + provider: &IdentityProviderId, + view: &EligibilityView, + catalog: &ProviderCatalog, +) -> Vec { + let capture_only = catalog.capture_only_sources(provider); + view.records() + .filter(|(source, record)| match record.w.as_deref() { + Some(writer) => writer == provider.as_str(), + None => capture_only.contains(source), + }) + .map(|(source, _)| PartnerIdChange::WriterRemove { + source: source.clone(), + writer: provider.clone(), + }) + .collect() +} + +/// The private-state change the resolution asks for. +fn state_update(provider: &IdentityProviderId, resolution: &Resolution) -> StateUpdate { + match resolution.state_change() { + StateChange::Keep => StateUpdate::Keep, + StateChange::Clear => StateUpdate::Clear, + StateChange::Replace(json) => match serde_json::from_str(json) { + Ok(value) => StateUpdate::Replace(value), + Err(_) => { + log::warn!("Identity provider '{provider}' returned state that is not JSON"); + StateUpdate::Keep + } + }, + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + use std::time::Duration; + + use serde_json::json; + + use super::*; + use crate::consent::types::GppConsent; + use crate::consent::us_privacy::decode_us_privacy; + use crate::ec::identity::descriptor::{ + ConsentPolicy, Input, ProviderDescriptor, ResolveKind, Resolves, UsOptOut, + }; + use crate::ec::identity::resolution::{ResolvedEid, ResolvedUid}; + use crate::ec::identity::revision::{record_revision, uid_hash}; + use crate::ec::identity::test_support::{ + OutcomeFixture, allowing_consent, facts_for_writer, fixed_capture_provider, fixed_provider, + pid, provider_from, record_for_test, source, stateful_fixed_provider, + test_consent_gdpr_without_vendor, + }; + use crate::ec::kv_types::{KvEid, KvUid}; + use crate::settings::ProviderMode; + + /// The six lines every eligibility test repeats: one provider, its + /// catalog and the settings and consent the call reads. + struct EligibilityFixture { + provider: Arc, + catalog: ProviderCatalog, + settings: ProviderSettings, + identity: IdentitySettings, + consent: ConsentContext, + } + + impl EligibilityFixture { + /// A catalog holding `provider` alone, with no source owners. + fn new(provider: &Arc) -> Self { + Self::with_catalog( + provider, + ProviderCatalog::new(vec![provider.clone()], BTreeMap::new()), + ) + } + + /// A catalog holding `provider` alone, owning each of `sources`. + fn owning(provider: &Arc, sources: &[&str]) -> Self { + let owners = sources + .iter() + .map(|name| (source(name), provider.descriptor().id().clone())) + .collect(); + Self::with_catalog( + provider, + ProviderCatalog::new(vec![provider.clone()], owners), + ) + } + + fn with_catalog(provider: &Arc, catalog: ProviderCatalog) -> Self { + Self { + provider: provider.clone(), + catalog, + settings: ProviderSettings::default(), + identity: IdentitySettings::default(), + consent: allowing_consent(), + } + } + + fn with_settings(mut self, settings: ProviderSettings) -> Self { + self.settings = settings; + self + } + + fn with_consent(mut self, consent: ConsentContext) -> Self { + self.consent = consent; + self + } + + fn check(&self, view: &EligibilityView, facts: &RequestFacts, phase: Phase) -> Eligibility { + let input = EligibilityInput { + provider: self.provider.as_ref(), + settings: &self.settings, + identity: &self.identity, + catalog: &self.catalog, + consent: &self.consent, + }; + eligibility(&input, view, facts, phase) + } + + fn is_eligible(&self, view: &EligibilityView, facts: &RequestFacts, phase: Phase) -> bool { + matches!(self.check(view, facts, phase), Eligibility::Eligible { .. }) + } + + /// The kinds of a writer navigation, after send. + fn kinds(&self, view: &EligibilityView) -> BTreeMap { + match self.check(view, &facts_for_writer(), Phase::AfterSend) { + Eligibility::Eligible { kinds } => kinds, + Eligibility::Skip(reason) => panic!("should be eligible, skipped for {reason:?}"), + } + } + + fn skip_reason(&self, view: &EligibilityView, facts: &RequestFacts) -> SkipReason { + match self.check(view, facts, Phase::AfterSend) { + Eligibility::Skip(reason) => reason, + Eligibility::Eligible { .. } => panic!("should be skipped"), + } + } + } + + fn custom_provider( + id: &str, + sources: &[&str], + resolves: Resolves, + policy: ConsentPolicy, + ) -> Arc { + let mut builder = ProviderDescriptor::builder(pid(id)) + .with_resolves(resolves.with_endpoint("https://api.example.com")) + .with_consent_policy(policy); + for name in sources { + builder = builder.with_source(source(name)); + } + provider_from(builder.build().expect("should build the descriptor")) + } + + fn expiring_provider(id: &str, sources: &[&str]) -> Arc { + let mut builder = ProviderDescriptor::builder(pid(id)) + .with_resolves(Resolves::new().with_endpoint("https://api.example.com")) + .with_consent_policy(ConsentPolicy::eid_persistence_only()) + .with_expiring_ids(); + for name in sources { + builder = builder.with_source(source(name)); + } + provider_from(builder.build().expect("should build the descriptor")) + } + + fn positive_record(change: &PartnerIdChange) -> Option<&KvPartnerId> { + match change { + PartnerIdChange::FillMissing { record, .. } + | PartnerIdChange::OwnerWrite { record, .. } => Some(record), + PartnerIdChange::OwnerRemove { .. } | PartnerIdChange::WriterRemove { .. } => None, + } + } + + fn kinds_of(entries: &[(&str, ResolveKind)]) -> BTreeMap { + entries + .iter() + .map(|(name, kind)| (source(name), *kind)) + .collect() + } + + fn one_entry(uid: &str) -> KvEid { + KvEid { + uids: vec![KvUid { + id: uid.to_owned(), + atype: None, + ext: None, + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + } + } + + fn large_entries(source_domain: &SourceDomain) -> Vec { + // Three entries of about 2 KB each: the set is over MAX_ENTRY_SET_BYTES. + let large_ext = json!({ "segments": "x".repeat(2_000) }); + (0..3) + .map(|index| { + ResolvedEid::new(source_domain, ResolvedUid::new(format!("example-{index}"))) + .with_ext(large_ext.clone()) + }) + .collect() + } + + // Eligibility + + #[test] + fn skips_when_not_a_real_browser() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let facts = RequestFacts { + is_real_browser: false, + ..facts_for_writer() + }; + + assert_eq!( + fixture.skip_reason(&EligibilityView::empty(), &facts), + SkipReason::NotRealBrowser, + "should skip a client that does not look like a browser" + ); + assert!( + fixture.is_eligible( + &EligibilityView::empty(), + &facts_for_writer(), + Phase::AfterSend + ), + "control: a real browser should be eligible" + ); + } + + #[test] + fn skips_when_ec_persistence_is_not_allowed() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let facts = RequestFacts { + ec_persistence_allowed: false, + ..facts_for_writer() + }; + + assert_eq!( + fixture.skip_reason(&EligibilityView::empty(), &facts), + SkipReason::EcPersistenceNotAllowed, + "should skip when global consent blocks EID persistence" + ); + } + + #[test] + fn skips_when_the_consent_policy_blocks() { + let provider = custom_provider( + "p", + &["a.example.com"], + Resolves::new(), + ConsentPolicy::strict().with_tcf_vendor(5), + ); + let blocked = + EligibilityFixture::new(&provider).with_consent(test_consent_gdpr_without_vendor(5)); + let allowed = + EligibilityFixture::new(&provider).with_consent(test_consent_gdpr_without_vendor(6)); + let view = EligibilityView::empty(); + + assert_eq!( + blocked.skip_reason(&view, &facts_for_writer()), + SkipReason::ConsentPolicy, + "should skip when the policy's vendor has no consent" + ); + assert!( + allowed.is_eligible(&view, &facts_for_writer(), Phase::AfterSend), + "control: should run when the vendor has consent" + ); + } + + #[test] + fn skips_a_provider_that_does_not_resolve() { + let fixture = EligibilityFixture::new(&fixed_capture_provider("p", &["a.example.com"])); + + assert_eq!( + fixture.skip_reason(&EligibilityView::empty(), &facts_for_writer()), + SkipReason::NotAResolver, + "should never run a capture-only provider" + ); + } + + #[test] + fn skips_new_visitor_in_every_mode() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let view = EligibilityView::empty(); + for phase in [Phase::Hold, Phase::AfterSend] { + let facts = RequestFacts { + new_visitor: true, + ..facts_for_writer() + }; + assert!( + matches!( + fixture.check(&view, &facts, phase), + Eligibility::Skip(SkipReason::NewVisitor) + ), + "should run no provider on a new visitor's first navigation ({phase:?})" + ); + } + let returning = facts_for_writer(); + assert!( + matches!( + fixture.check(&view, &returning, Phase::AfterSend), + Eligibility::Eligible { .. } + ), + "control: a returning writer should be eligible" + ); + } + + #[test] + fn stager_runs_no_provider_in_either_phase() { + let provider = fixed_provider("p", &["a.example.com"]); + let view = EligibilityView::empty(); + let stager = RequestFacts { + writer: false, + runs_auction: true, + ..facts_for_writer() + }; + for mode in [ProviderMode::Hold, ProviderMode::AfterSend] { + let fixture = EligibilityFixture::new(&provider).with_settings(ProviderSettings { + mode, + ..ProviderSettings::default() + }); + for phase in [Phase::Hold, Phase::AfterSend] { + assert!( + matches!( + fixture.check(&view, &stager, phase), + Eligibility::Skip(SkipReason::NotWriter) + ), + "should run no provider on a stager ({mode:?}, {phase:?})" + ); + } + } + } + + #[test] + fn owned_record_written_by_someone_else_is_due_for_refresh() { + let fixture = EligibilityFixture::owning( + &fixed_provider("p", &["a.example.com", "b.example.com"]), + &["a.example.com", "b.example.com"], + ); + let mut view = EligibilityView::empty(); + // Written by the browser, and acquired recently. + view.set_record_for_test( + &source("a.example.com"), + record_for_test("example-a", None, 999), + ); + // Written by this provider, acquired recently. + view.set_record_for_test( + &source("b.example.com"), + record_for_test("example-b", Some("p"), 999), + ); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("a.example.com", ResolveKind::Refresh)]), + "should refresh the browser-seeded record and leave the fresh owned one" + ); + } + + #[test] + fn owned_source_without_a_record_is_initial() { + let fixture = EligibilityFixture::owning( + &fixed_provider("p", &["a.example.com"]), + &["a.example.com"], + ); + + assert_eq!( + fixture.kinds(&EligibilityView::empty()), + kinds_of(&[("a.example.com", ResolveKind::Initial)]), + "should ask for an initial resolution when there is no record" + ); + } + + #[test] + fn owned_record_is_due_by_refresh_anchor_or_age() { + let fixture = EligibilityFixture::owning( + &fixed_provider("p", &["a.example.com", "b.example.com", "c.example.com"]), + &["a.example.com", "b.example.com", "c.example.com"], + ) + .with_settings(ProviderSettings { + refresh_after_sec: 100, + ..ProviderSettings::default() + }); + let mut view = EligibilityView::empty(); + // `ra` has passed although `at` is recent. + let mut passed = record_for_test("example-a", Some("p"), 999); + passed.ra = Some(1_000); + view.set_record_for_test(&source("a.example.com"), passed); + // `ra` is ahead although `at` is old, so `ra` wins. + let mut ahead = record_for_test("example-b", Some("p"), 100); + ahead.ra = Some(5_000); + view.set_record_for_test(&source("b.example.com"), ahead); + // No `ra`, and `at + refresh_after_sec` has passed. + view.set_record_for_test( + &source("c.example.com"), + record_for_test("example-c", Some("p"), 900), + ); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[ + ("a.example.com", ResolveKind::Refresh), + ("c.example.com", ResolveKind::Refresh) + ]), + "should refresh when ra passed, or when there is no ra and the age passed" + ); + } + + #[test] + fn owned_record_without_at_or_ra_is_due() { + let fixture = EligibilityFixture::owning( + &fixed_provider("p", &["a.example.com"]), + &["a.example.com"], + ); + let mut view = EligibilityView::empty(); + let mut record = record_for_test("example-a", Some("p"), 999); + record.at = None; + view.set_record_for_test(&source("a.example.com"), record); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("a.example.com", ResolveKind::Refresh)]), + "should refresh a record with no acquisition time" + ); + } + + #[test] + fn browser_seeded_uid_in_iv_is_never_due() { + let fixture = EligibilityFixture::owning( + &fixed_provider("p", &["a.example.com"]), + &["a.example.com"], + ); + let mut view = EligibilityView::empty(); + view.set_record_for_test( + &source("a.example.com"), + record_for_test("example-bad", None, 10), + ); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("a.example.com", ResolveKind::Refresh)]), + "control: a browser-seeded record should be due" + ); + + view.set_invalidated_for_test("p", vec![uid_hash("example-bad")]); + assert!( + matches!( + fixture.check(&view, &facts_for_writer(), Phase::AfterSend), + Eligibility::Skip(SkipReason::NoSourceDue) + ), + "should never refresh a browser-seeded value the provider invalidated" + ); + } + + #[test] + fn expired_refresh_anchor_stays_refresh() { + let fixture = EligibilityFixture::owning( + &expiring_provider("p", &["a.example.com"]), + &["a.example.com"], + ); + let mut view = EligibilityView::empty(); + let mut anchor = record_for_test("example-a", Some("p"), 100); + anchor.exp = Some(500); + anchor.ru = Some(9_000); + view.set_record_for_test(&source("a.example.com"), anchor); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("a.example.com", ResolveKind::Refresh)]), + "should keep the kind Refresh for an expired record that is still stored" + ); + } + + #[test] + fn unowned_source_with_unexpired_record_is_not_initial_even_when_consent_hides_it() { + let hiding = custom_provider( + "q", + &["a.example.com"], + Resolves::new(), + ConsentPolicy::strict().with_tcf_vendor(5), + ); + let asking = fixed_provider("p", &["a.example.com"]); + let catalog = ProviderCatalog::new(vec![hiding, asking.clone()], BTreeMap::new()); + // Consent withholds vendor 5, so `q`'s record is not servable. + let fixture = EligibilityFixture::with_catalog(&asking, catalog) + .with_consent(test_consent_gdpr_without_vendor(5)); + let mut view = EligibilityView::empty(); + view.set_record_for_test( + &source("a.example.com"), + record_for_test("example-a", Some("q"), 900), + ); + + assert!( + matches!( + fixture.check(&view, &facts_for_writer(), Phase::AfterSend), + Eligibility::Skip(SkipReason::NoSourceDue) + ), + "should not call to fill a source that already holds an unexpired ID" + ); + assert_eq!( + fixture.kinds(&EligibilityView::empty()), + kinds_of(&[("a.example.com", ResolveKind::Initial)]), + "control: an absent record should be Initial" + ); + } + + #[test] + fn expired_unowned_record_counts_as_absent() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let mut view = EligibilityView::empty(); + let mut record = record_for_test("example-a", None, 100); + record.exp = Some(1_000); + view.set_record_for_test(&source("a.example.com"), record); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("a.example.com", ResolveKind::Initial)]), + "should treat a record expired at now as absent" + ); + } + + #[test] + fn record_of_an_unconfigured_writer_counts_as_absent() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let mut view = EligibilityView::empty(); + view.set_record_for_test( + &source("a.example.com"), + record_for_test("example-a", Some("gone"), 900), + ); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("a.example.com", ResolveKind::Initial)]), + "should treat a record written by an unconfigured provider as absent" + ); + + let mut browser_view = EligibilityView::empty(); + browser_view.set_record_for_test( + &source("a.example.com"), + record_for_test("example-a", None, 900), + ); + assert!( + matches!( + fixture.check(&browser_view, &facts_for_writer(), Phase::AfterSend), + Eligibility::Skip(SkipReason::NoSourceDue) + ), + "control: a browser-seeded unexpired record should count as present" + ); + } + + #[test] + fn hold_provider_runs_before_the_auction_and_not_after_send() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])) + .with_settings(ProviderSettings { + mode: ProviderMode::Hold, + ..ProviderSettings::default() + }); + let view = EligibilityView::empty(); + let facts = RequestFacts { + runs_auction: true, + ..facts_for_writer() + }; + + assert!( + fixture.is_eligible(&view, &facts, Phase::Hold), + "should hold before the auction" + ); + assert!( + !fixture.is_eligible(&view, &facts, Phase::AfterSend), + "should not run again after send" + ); + } + + #[test] + fn hold_provider_runs_after_send_when_the_navigation_runs_no_auction() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])) + .with_settings(ProviderSettings { + mode: ProviderMode::Hold, + ..ProviderSettings::default() + }); + let view = EligibilityView::empty(); + + let no_auction = RequestFacts { + runs_auction: false, + ..facts_for_writer() + }; + let auction = RequestFacts { + runs_auction: true, + ..facts_for_writer() + }; + let stager = RequestFacts { + writer: false, + runs_auction: true, + ..facts_for_writer() + }; + + assert!( + fixture.is_eligible(&view, &no_auction, Phase::AfterSend), + "should move after send without an auction" + ); + assert!( + !fixture.is_eligible(&view, &no_auction, Phase::Hold), + "should not hold without an auction" + ); + assert!( + fixture.is_eligible(&view, &auction, Phase::Hold), + "control: should hold before an auction" + ); + assert!( + !fixture.is_eligible(&view, &auction, Phase::AfterSend), + "should not also run after send" + ); + assert!( + !fixture.is_eligible(&view, &stager, Phase::Hold), + "should never run on a stager" + ); + } + + #[test] + fn after_send_provider_never_runs_in_hold() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let view = EligibilityView::empty(); + for runs_auction in [true, false] { + let facts = RequestFacts { + runs_auction, + ..facts_for_writer() + }; + assert!( + !fixture.is_eligible(&view, &facts, Phase::Hold), + "should not hold an after-send provider (auction: {runs_auction})" + ); + assert!( + fixture.is_eligible(&view, &facts, Phase::AfterSend), + "control: should run it after send (auction: {runs_auction})" + ); + } + } + + #[test] + fn narrowed_away_source_is_never_a_kind() { + let provider = fixed_provider("p", &["a.example.com", "b.example.com"]); + let catalog = ProviderCatalog::new(vec![provider.clone()], BTreeMap::new()) + .with_narrowing(BTreeMap::from([(pid("p"), vec![source("a.example.com")])])); + let fixture = EligibilityFixture::with_catalog(&provider, catalog); + + assert_eq!( + fixture.kinds(&EligibilityView::empty()), + kinds_of(&[("a.example.com", ResolveKind::Initial)]), + "should leave out a source the operator narrowed away" + ); + } + + #[test] + fn kinds_the_provider_does_not_list_are_left_out() { + let provider = custom_provider( + "p", + &["a.example.com", "b.example.com"], + Resolves::new().with_kinds([ResolveKind::Initial]), + ConsentPolicy::eid_persistence_only(), + ); + let fixture = EligibilityFixture::owning(&provider, &["a.example.com", "b.example.com"]); + let mut view = EligibilityView::empty(); + view.set_record_for_test( + &source("a.example.com"), + record_for_test("example-a", None, 10), + ); + + assert_eq!( + fixture.kinds(&view), + kinds_of(&[("b.example.com", ResolveKind::Initial)]), + "should leave out the Refresh source of an Initial-only provider" + ); + } + + #[test] + fn source_whose_needs_are_unavailable_is_left_out() { + let provider = custom_provider( + "p", + &["a.example.com"], + Resolves::new() + .with_needs_for(ResolveKind::Initial, [Input::ClientIp]) + .with_needs_for(ResolveKind::Refresh, [Input::HashedEmail]), + ConsentPolicy::eid_persistence_only(), + ); + let fixture = EligibilityFixture::owning(&provider, &["a.example.com"]); + let initial = EligibilityView::empty(); + let mut refresh = EligibilityView::empty(); + refresh.set_record_for_test( + &source("a.example.com"), + record_for_test("example-a", None, 10), + ); + let mut without_email = facts_for_writer(); + without_email.available_inputs.remove(&Input::HashedEmail); + let mut without_address = facts_for_writer(); + without_address.available_inputs.remove(&Input::ClientIp); + + assert_eq!( + fixture.skip_reason(&refresh, &without_email), + SkipReason::NoSourceDue, + "should leave out a Refresh whose needs are missing" + ); + assert_eq!( + fixture.skip_reason(&initial, &without_address), + SkipReason::NoSourceDue, + "should leave out an Initial whose needs are missing" + ); + assert!( + fixture.is_eligible(&initial, &without_email, Phase::AfterSend), + "control: the Initial should not need the email" + ); + assert!( + fixture.is_eligible(&refresh, &facts_for_writer(), Phase::AfterSend), + "control: should run when the inputs are available" + ); + } + + #[test] + fn future_n_and_sticky_opt_out_block() { + let fixture = EligibilityFixture::new(&fixed_provider("p", &["a.example.com"])); + let facts = facts_for_writer(); + + let mut waiting = EligibilityView::empty(); + waiting.set_next_for_test("p", 1_001); + assert_eq!( + fixture.skip_reason(&waiting, &facts), + SkipReason::NotYetEligible, + "should wait for n" + ); + + let mut due = EligibilityView::empty(); + due.set_next_for_test("p", 1_000); + assert!( + fixture.is_eligible(&due, &facts, Phase::AfterSend), + "control: should run once n has passed" + ); + + let mut opted_out = EligibilityView::empty(); + opted_out.set_opt_out_for_test("p", 500, None); + assert_eq!( + fixture.skip_reason(&opted_out, &facts), + SkipReason::OptedOut, + "should not call a provider the visitor opted out of" + ); + } + + #[test] + fn a_later_clear_lifts_the_opt_out() { + let mut view = EligibilityView::empty(); + view.set_opt_out_for_test("p", 500, Some(600)); + assert!( + !view.is_opted_out(&pid("p")), + "should lift the opt-out when oc is later than o.at" + ); + + view.set_opt_out_for_test("p", 500, Some(500)); + assert!( + view.is_opted_out(&pid("p")), + "should keep the opt-out when oc is not strictly later" + ); + + view.set_opt_out_for_test("p", 500, Some(400)); + assert!( + view.is_opted_out(&pid("p")), + "should keep the opt-out when oc is older" + ); + assert!( + !view.is_opted_out(&pid("other")), + "should not opt out a provider with no state" + ); + } + + #[test] + fn view_reads_only_the_kv_row() { + let mut entry = KvEntry::new(&allowing_consent(), None, 10, "example.com"); + entry.ids.insert( + "a.example.com".to_owned(), + record_for_test("example-a", Some("p"), 900), + ); + entry.ps.insert( + "p".to_owned(), + crate::ec::kv_types::KvProviderState { + n: Some(2_000), + f: 2, + o: Some(KvOptOut { + at: 5, + fp: "1".to_owned(), + }), + oc: Some(4), + iv: vec!["0123456789abcdef".to_owned()], + ov: Vec::new(), + st: None, + }, + ); + + let view = EligibilityView::from_kv(&entry); + + assert!( + view.record(&source("a.example.com")).is_some(), + "should hold the source's whole record" + ); + assert_eq!(view.failures(&pid("p")), 2, "should hold f"); + assert_eq!(view.next_eligible(&pid("p")), Some(2_000), "should hold n"); + assert!(view.is_opted_out(&pid("p")), "should hold o and oc"); + } + + // Fingerprint + + #[test] + fn fingerprint_bits_is_a_plain_hex_bitmask_and_changes_with_each_signal() { + let policy = ConsentPolicy::strict() + .with_tcf_purposes([2, 10]) + .with_tcf_vendor(5); + let signals = PolicySignals::default(); + let gdpr = test_consent_gdpr_without_vendor(6); + let bits = |consent: &ConsentContext, signals: PolicySignals| { + let value = fingerprint_bits(&policy, consent, signals); + assert!( + value.len() <= 16 && value == value.to_lowercase(), + "should be at most 16 lowercase hex digits" + ); + u64::from_str_radix(&value, 16).expect("should parse as hex") + }; + + // Purposes 1, 2, 4 and 10 are read and granted: bits 0, 1, 3, 9. + // Vendor 5 is granted: bit 10. + let control = bits(&gdpr, signals); + assert_eq!( + control, + (1 << 0) | (1 << 1) | (1 << 3) | (1 << 9) | (1 << 10), + "should set one bit per granted purpose and the vendor bit" + ); + + let mut without_purpose_two = gdpr.clone(); + if let Some(tcf) = without_purpose_two.tcf.as_mut() { + tcf.purpose_consents[1] = false; + } + assert_eq!( + bits(&without_purpose_two, signals), + control & !(1 << 1), + "should clear only bit 1 when purpose 2 is not granted" + ); + + let without_vendor = test_consent_gdpr_without_vendor(5); + assert_eq!( + bits(&without_vendor, signals), + control & !(1 << 10), + "should clear only the vendor bit when the vendor is not granted" + ); + + let mut sale = gdpr.clone(); + sale.us_privacy = Some(decode_us_privacy("1YYN").expect("should decode us_privacy")); + assert_eq!( + bits(&sale, signals), + control | (1 << 11), + "should set the sale bit for a US Privacy sale opt-out" + ); + + let mut gpp_sale = gdpr.clone(); + gpp_sale.gpp = Some(GppConsent { + version: 1, + section_ids: vec![7], + eu_tcf: None, + us_sale_opt_out: Some(true), + }); + assert_eq!( + bits(&gpp_sale, signals), + control | (1 << 11), + "should set the sale bit for a GPP sale opt-out" + ); + + let mut gpc = gdpr.clone(); + gpc.gpc = true; + assert_eq!( + bits(&gpc, signals), + control | (1 << 12), + "should set the GPC bit" + ); + + assert_eq!( + bits(&gdpr, PolicySignals { dnt: true }), + control | (1 << 13), + "should set the DNT bit" + ); + } + + #[test] + fn fingerprint_bits_leaves_a_relaxed_signal_at_zero_and_never_moves_a_bit() { + let strict = ConsentPolicy::strict(); + let relaxed = ConsentPolicy::strict() + .allowing_gpc() + .allowing_dnt() + .allowing_us_opt_out(UsOptOut::Sale); + let mut consent = allowing_consent(); + consent.gpc = true; + let signals = PolicySignals { dnt: true }; + + assert_eq!( + fingerprint_bits(&strict, &consent, signals), + "3000", + "should set the GPC and DNT bits at positions 12 and 13" + ); + assert_eq!( + fingerprint_bits(&relaxed, &consent, signals), + "0", + "should leave a relaxed signal at zero" + ); + assert_eq!( + fingerprint_bits( + &ConsentPolicy::strict().with_tcf_purposes([7]), + &consent, + PolicySignals::default() + ), + "1000", + "should keep GPC at bit 12 whatever purposes the policy reads" + ); + } + + // Section 8 rows + + #[test] + fn no_match_waits_retry_in_capped_at_seven_days() { + let provider = fixed_provider("p", &["a.example.com"]); + let fixture = OutcomeFixture::new(); + let context = fixture.context_at(1_000); + let default = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::no_match()), + &EligibilityView::empty(), + &context, + ); + assert_eq!( + default.state.n, + Some(1_000 + 86_400), + "should wait the 24 h default" + ); + let long = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved( + Resolution::no_match().with_retry_in(Duration::from_secs(30 * 86_400)), + ), + &EligibilityView::empty(), + &context, + ); + assert_eq!( + long.state.n, + Some(1_000 + 7 * 86_400), + "should cap the wait at 7 days" + ); + } + + #[test] + fn found_resets_failures_and_retry_in_applies_to_every_resolution() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 3); + view.set_record_for_test(&a, record_for_test("example-old", Some("p"), 900)); + let fixture = OutcomeFixture::new(); + let context = fixture.context_at(1_000); + + let found = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved( + Resolution::found([ResolvedEid::new(&a, ResolvedUid::new("u"))]) + .with_retry_in(Duration::from_secs(600)), + ), + &view, + &context, + ); + let invalidated = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved( + Resolution::invalidate(&a).with_retry_in(Duration::from_secs(600)), + ), + &view, + &context, + ); + + assert_eq!( + found.state.f, + Some(0), + "should reset the failure count on found" + ); + assert_eq!( + found.state.n, + Some(1_000 + 600), + "should wait at least retry_in after found" + ); + assert_eq!( + invalidated.changes.len(), + 1, + "control: should remove the stored record" + ); + assert_eq!( + invalidated.state.n, + Some(1_000 + 600), + "should raise invalidate's n = now to retry_in" + ); + } + + #[test] + fn found_without_retry_in_sets_no_n() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("u"), + )])), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.state.n, None, + "should leave the refresh schedule to the record's own timing" + ); + } + + #[test] + fn input_failure_backs_off_exponentially_to_a_day() { + let provider = fixed_provider("p", &["a.example.com"]); + let fixture = OutcomeFixture::new(); + let context = fixture.context_at(1_000); + let cases = [ + (0, 1, 3_600), + (1, 2, 7_200), + (4, 5, 57_600), + (5, 6, 86_400), + (u32::MAX, u32::MAX, 86_400), + ]; + for (before, after, wait) in cases { + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", before); + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::InputFailure, + &view, + &context, + ); + assert_eq!( + changes.state.f, + Some(after), + "should count the failure (f was {before})" + ); + assert_eq!( + changes.state.n, + Some(1_000 + wait), + "should back off min(1 h x 2^(f-1), 24 h) (f was {before})" + ); + } + } + + #[test] + fn transport_failure_backs_off_to_an_hour() { + let provider = fixed_provider("p", &["a.example.com"]); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 5); + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::TransportFailure, + &view, + &fixture.context_at(1_000), + ); + assert_eq!(changes.state.f, Some(6), "should count the failure"); + assert_eq!( + changes.state.n, + Some(1_000 + 3_600), + "should back off min(5 min x 2^5, 1 h) = 1 h" + ); + + let first = outcome_to_changes( + provider.as_ref(), + CallOutcome::TransportFailure, + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + assert_eq!( + first.state.n, + Some(1_000 + 300), + "should start at 5 minutes" + ); + } + + #[test] + fn hold_cut_off_waits_hold_retry_without_counting_a_failure() { + let provider = fixed_provider("p", &["a.example.com"]); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 2); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::HoldCutOff, + &view, + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.state.n, + Some(1_000 + 300), + "should wait hold_retry_sec" + ); + assert_eq!(changes.state.f, None, "should not count a failure"); + assert!(changes.changes.is_empty(), "should change no stored ID"); + } + + #[test] + fn skip_waits_refresh_after() { + let provider = fixed_provider("p", &["a.example.com"]); + let fixture = OutcomeFixture::new().with_settings(ProviderSettings { + refresh_after_sec: 777, + ..ProviderSettings::default() + }); + for outcome in [CallOutcome::Skipped, CallOutcome::InputUnavailable] { + let changes = outcome_to_changes( + provider.as_ref(), + outcome, + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + assert_eq!( + changes.state.n, + Some(1_000 + 777), + "should wait refresh_after_sec" + ); + assert_eq!(changes.state.f, None, "should not count a failure"); + } + } + + #[test] + fn burst_cap_waits_one_window_and_never_counts_a_failure() { + let provider = fixed_provider("p", &["a.example.com"]); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 2); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::BurstCapped, + &view, + &fixture.context_at(1_000), + ); + + assert_eq!(changes.state.n, Some(1_060), "should wait one 60 s window"); + assert_eq!(changes.state.f, None, "should never count a failure"); + } + + #[test] + fn breaker_open_and_not_dispatched_change_nothing() { + let provider = fixed_provider("p", &["a.example.com"]); + let fixture = OutcomeFixture::new(); + for outcome in [CallOutcome::BreakerOpen, CallOutcome::NotDispatched] { + let changes = outcome_to_changes( + provider.as_ref(), + outcome, + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + assert_eq!( + changes, + OutcomeChanges::default(), + "should change nothing when the call was not made" + ); + } + } + + #[test] + fn invalid_state_clears_state_and_keeps_f() { + let provider = stateful_fixed_provider("p", &["a.example.com"]); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 2); + let fixture = OutcomeFixture::new().with_settings(ProviderSettings { + refresh_after_sec: 777, + ..ProviderSettings::default() + }); + let context = fixture.context_at(1_000); + + let bare = outcome_to_changes( + provider.as_ref(), + CallOutcome::InvalidState(None), + &view, + &context, + ); + assert_eq!(bare.state.st, StateUpdate::Clear, "should clear st"); + assert_eq!(bare.state.f, None, "should keep f"); + assert_eq!( + bare.state.n, + Some(1_000 + 777), + "should wait refresh_after_sec" + ); + + let with_result = outcome_to_changes( + provider.as_ref(), + CallOutcome::InvalidState(Some( + Resolution::no_match().with_retry_in(Duration::from_secs(60)), + )), + &view, + &context, + ); + let result = with_result + .result + .expect("should carry the result's own state change"); + assert_eq!(result.state.st, StateUpdate::Clear, "should clear st"); + assert_eq!(result.state.f, None, "should keep f"); + assert_eq!( + result.state.n, + Some(1_000 + 60), + "should keep the n the result set" + ); + } + + #[test] + fn oversized_state_rejects_positive_part_but_keeps_removals() { + let provider = stateful_fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&b, record_for_test("example-old", Some("p"), 900)); + let resolution = Resolution::found([ResolvedEid::new(&a, ResolvedUid::new("example-a"))]) + .also_invalidate(&b) + .replacing_state(&"x".repeat(2_000)); + + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(resolution), + &view, + &fixture.context_at(1_000), + ); + + assert!( + changes.result.is_none(), + "should store no ID and no state change" + ); + assert_eq!( + changes.changes.len(), + 1, + "should keep only the invalidation" + ); + assert!( + matches!(&changes.changes[0], PartnerIdChange::OwnerRemove { source, .. } if *source == b), + "should still apply the invalidation" + ); + assert_eq!( + changes.state.st, + StateUpdate::Keep, + "should keep the old state" + ); + assert_eq!(changes.state.f, Some(1), "should handle it as Malformed"); + assert_eq!( + changes.state.n, + Some(1_000 + 3_600), + "should back off min(1 h x 2^0, 24 h)" + ); + } + + #[test] + fn opted_out_removes_what_it_wrote_and_clears_state() { + let provider = fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let c = source("c.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, record_for_test("example-a", Some("p"), 900)); + view.set_record_for_test(&b, record_for_test("example-b", None, 900)); + view.set_record_for_test(&c, record_for_test("example-c", Some("p"), 900)); + let fixture = OutcomeFixture::new().with_fingerprint("5"); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::opted_out()), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes, + vec![ + PartnerIdChange::WriterRemove { + source: a, + writer: pid("p"), + }, + PartnerIdChange::WriterRemove { + source: c, + writer: pid("p"), + }, + ], + "should remove every record the provider wrote, claimed or not, and no browser record" + ); + assert_eq!( + changes.state.opt_out, + Some(KvOptOut { + at: 1_000, + fp: "5".to_owned() + }), + "should record the opt-out" + ); + assert_eq!(changes.state.st, StateUpdate::Clear, "should clear st"); + } + + #[test] + fn capture_only_opt_out_also_removes_browser_records_of_its_sources() { + let provider = fixed_capture_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let c = source("c.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, record_for_test("example-a", None, 900)); + view.set_record_for_test(&b, record_for_test("example-b", Some("q"), 900)); + view.set_record_for_test(&c, record_for_test("example-c", None, 900)); + let catalog = ProviderCatalog::new(vec![provider.clone()], BTreeMap::new()); + let fixture = OutcomeFixture::new().with_catalog(catalog); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::opted_out()), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes, + vec![PartnerIdChange::WriterRemove { + source: a, + writer: pid("p"), + }], + "should remove the browser record of its own source only" + ); + } + + #[test] + fn opted_out_records_the_fingerprint_of_the_current_consent() { + let provider = fixed_provider("p", &["a.example.com"]); + let policy = ConsentPolicy::strict(); + let mut consent = allowing_consent(); + consent.gpc = true; + let fingerprint = fingerprint_bits(&policy, &consent, PolicySignals::default()); + let fixture = OutcomeFixture::new().with_fingerprint(&fingerprint); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::opted_out()), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.state.opt_out, + Some(KvOptOut { + at: 1_000, + fp: "1000".to_owned() + }), + "should store the fingerprint of the consent in force" + ); + } + + #[test] + fn stateful_opt_out_goes_into_the_result_with_its_removals_outside() { + let provider = stateful_fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, record_for_test("example-a", Some("p"), 900)); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::opted_out()), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!(changes.changes.len(), 1, "should list the removal"); + let result = changes + .result + .expect("should carry the opt-out in one result"); + assert!(result.ids.is_empty(), "should hold no positive change"); + assert!(result.state.opt_out.is_some(), "should record the opt-out"); + assert_eq!(result.state.st, StateUpdate::Clear, "should clear st"); + } + + #[test] + fn invalidate_removes_the_set_and_sets_n_now() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let record = record_for_test("example-a", Some("p"), 900); + let expected = record_revision(&record); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, record); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::invalidate(&a)), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes, + vec![PartnerIdChange::OwnerRemove { + source: a, + provider: pid("p"), + expected, + }], + "should remove the set at the revision read" + ); + assert_eq!(changes.state.n, Some(1_000), "should set n = now"); + assert_eq!(changes.state.f, None, "should not touch f"); + } + + #[test] + fn invalidating_a_source_without_a_record_makes_no_change() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::invalidate(&a)), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert_eq!( + changes, + OutcomeChanges::default(), + "should change nothing for a source with no record" + ); + } + + #[test] + fn invalidating_an_unclaimed_source_makes_no_change() { + let provider = fixed_provider("p", &["a.example.com"]); + let c = source("c.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&c, record_for_test("example-c", Some("q"), 900)); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::invalidate(&c)), + &view, + &fixture.context_at(1_000), + ); + + assert!( + changes.changes.is_empty(), + "should not let a provider remove a source it does not claim" + ); + } + + // Entry sets + + #[test] + fn found_with_several_entries_becomes_one_whole_set_per_source() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let resolution = Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a").with_atype(1)) + .with_uid( + ResolvedUid::new("example-b") + .with_atype(3) + .with_ext(json!({ "stype": "ppuid" })), + ) + .with_inserter("example.com") + .with_mm(3), + ResolvedEid::new(&a, ResolvedUid::new("example-c")).with_matcher("partner.example.com"), + ]); + + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(resolution), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes.len(), + 1, + "should make one change for the one source" + ); + let record = positive_record(&changes.changes[0]).expect("should be a positive change"); + assert_eq!(record.w.as_deref(), Some("p"), "should name the writer"); + assert_eq!(record.at, Some(1_000), "should stamp the write time"); + assert_eq!( + record.eids, + vec![ + KvEid { + uids: vec![ + KvUid { + id: "example-a".to_owned(), + atype: Some(1), + ext: None + }, + KvUid { + id: "example-b".to_owned(), + atype: Some(3), + ext: Some(json!({ "stype": "ppuid" })) + }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + KvEid { + uids: vec![KvUid { + id: "example-c".to_owned(), + atype: None, + ext: None + }], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ], + "should store the whole set, in the order the provider returned it" + ); + } + + #[test] + fn one_uid_result_without_eid_fields_becomes_a_one_entry_record() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-a"), + )])), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + let record = positive_record(&changes.changes[0]).expect("should be a positive change"); + assert_eq!( + record.eids, + KvPartnerId::new("example-a").eids, + "should write the shape browser ingest writes" + ); + assert_eq!(record.rv, None, "should leave the revision to the merge"); + } + + #[test] + fn owned_source_becomes_owner_write_and_unowned_becomes_fill_missing() { + let provider = fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let catalog = ProviderCatalog::new( + vec![provider.clone()], + BTreeMap::from([(a.clone(), pid("p"))]), + ); + let stored = record_for_test("example-old", Some("p"), 900); + let expected = record_revision(&stored); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, stored); + let fixture = OutcomeFixture::new().with_catalog(catalog); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a")), + ResolvedEid::new(&b, ResolvedUid::new("example-b")), + ])), + &view, + &fixture.context_at(1_000), + ); + + assert!( + changes.changes.iter().any(|change| matches!( + change, + PartnerIdChange::OwnerWrite { source, expected: Some(revision), .. } + if *source == a && *revision == expected + )), + "should write the owned source at the revision read" + ); + assert!( + changes.changes.iter().any(|change| matches!( + change, + PartnerIdChange::FillMissing { source, .. } if *source == b + )), + "should only fill the unowned source" + ); + + let fresh = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-a"), + )])), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + assert!( + matches!( + &fresh.changes[0], + PartnerIdChange::OwnerWrite { expected: None, .. } + ), + "should expect no record when the owned source had none" + ); + } + + #[test] + fn stateful_found_goes_into_one_provider_result_with_expected_state() { + let provider = stateful_fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let expected_state = StateRevision::of(Some(&json!({ "token": "example-old" }))); + let fixture = OutcomeFixture::new().with_expected_state(expected_state.clone()); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved( + Resolution::found([ResolvedEid::new(&a, ResolvedUid::new("example-a"))]) + .replacing_state(&json!({ "token": "example-new" })), + ), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert!( + changes.changes.is_empty(), + "should hold no stand-alone change" + ); + assert_eq!( + changes.state, + ProviderStateChange::default(), + "should hold no stand-alone state" + ); + let result = changes.result.expect("should produce one result"); + assert_eq!(result.provider, pid("p"), "should name the provider"); + assert_eq!(result.ids.len(), 1, "should carry the positive change"); + assert_eq!( + result.state.st, + StateUpdate::Replace(json!({ "token": "example-new" })), + "should replace st" + ); + assert_eq!(result.state.f, Some(0), "should reset f in the result"); + assert_eq!( + result.expected_state, + Some(expected_state), + "should expect the state revision the call read" + ); + } + + #[test] + fn unclaimed_source_entries_are_discarded() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let c = source("c.example.com"); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a")), + ResolvedEid::new(&c, ResolvedUid::new("example-c")), + ])), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes.len(), + 1, + "should keep only the claimed source" + ); + assert!( + matches!(&changes.changes[0], PartnerIdChange::FillMissing { source, .. } if *source == a), + "should drop the source the provider does not claim" + ); + } + + #[test] + fn found_naming_only_unclaimed_sources_stores_nothing_and_resets_failures() { + let provider = fixed_provider("p", &["a.example.com"]); + let c = source("c.example.com"); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 3); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &c, + ResolvedUid::new("example-c"), + )])), + &view, + &fixture.context_at(1_000), + ); + + assert!(changes.changes.is_empty(), "should store nothing"); + assert_eq!(changes.state.f, Some(0), "should still reset f on found"); + } + + #[test] + fn oversized_set_from_a_stateless_provider_leaves_only_that_source_unchanged() { + let provider = fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let mut entries = large_entries(&a); + entries.push(ResolvedEid::new(&b, ResolvedUid::new("example-b"))); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found(entries)), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + assert_eq!(changes.changes.len(), 1, "should keep the other source"); + assert!( + matches!(&changes.changes[0], PartnerIdChange::FillMissing { source, .. } if *source == b), + "should leave the oversized source as it is" + ); + assert_eq!(changes.state.f, Some(0), "should still count as found"); + } + + #[test] + fn oversized_set_from_a_stateful_provider_rejects_the_positive_part_and_keeps_removals() { + let provider = stateful_fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&b, record_for_test("example-old", Some("p"), 900)); + let resolution = Resolution::found(large_entries(&a)) + .also_invalidate(&b) + .replacing_state(&json!({ "token": "example-token" })); + + let fixture = OutcomeFixture::new(); + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(resolution), + &view, + &fixture.context_at(1_000), + ); + + assert!( + changes.result.is_none(), + "should store no ID and no state change" + ); + assert!( + changes.changes.iter().any( + |change| matches!(change, PartnerIdChange::OwnerRemove { source, .. } if *source == b) + ), + "should still apply the invalidation" + ); + assert!( + changes + .changes + .iter() + .all(|change| positive_record(change).is_none()), + "should store no positive change" + ); + assert!( + matches!(changes.state.st, StateUpdate::Keep), + "should keep the old state" + ); + assert_eq!(changes.state.f, Some(1), "should handle it as Malformed"); + assert_eq!( + changes.state.n, + Some(1_000 + 3_600), + "should back off min(1 h x 2^0, 24 h)" + ); + } + + #[test] + fn result_whose_every_set_is_discarded_is_malformed() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 1); + view.set_record_for_test( + &source("b.example.com"), + record_for_test("example-b", Some("p"), 900), + ); + let fixture = OutcomeFixture::new(); + + let oversized = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved( + Resolution::found(large_entries(&a)).also_invalidate(&source("b.example.com")), + ), + &view, + &fixture.context_at(1_000), + ); + let dropped = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new(" "), + )])), + &view, + &fixture.context_at(1_000), + ); + + for (name, changes) in [("oversized", &oversized), ("dropped", &dropped)] { + assert_eq!(changes.state.f, Some(2), "should count a failure ({name})"); + assert_eq!( + changes.state.n, + Some(1_000 + 7_200), + "should back off min(1 h x 2^1, 24 h) ({name})" + ); + assert!( + changes + .changes + .iter() + .all(|change| positive_record(change).is_none()), + "should store no positive change ({name})" + ); + } + assert_eq!( + oversized.changes.len(), + 1, + "should still apply the removal of an oversized result" + ); + } + + #[test] + fn invalidating_a_set_carries_no_hash_until_it_applies() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + let mut record = record_for_test("example-0-0", Some("p"), 900); + record.eids = (0..4) + .map(|chunk| KvEid { + uids: (0..8) + .map(|index| KvUid { + id: format!("example-{chunk}-{index}"), + atype: None, + ext: None, + }) + .collect(), + inserter: None, + matcher: None, + mm: None, + ext: None, + }) + .collect(); + let expected = record_revision(&record); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, record); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::invalidate(&a)), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes.len(), + 1, + "should make one removal for the 32-UID set" + ); + assert!( + matches!( + &changes.changes[0], + PartnerIdChange::OwnerRemove { source, provider, expected: revision } + if *source == a && provider.as_str() == "p" && *revision == expected + ), + "should remove the whole set at the revision it read" + ); + assert_eq!(changes.state.n, Some(1_000), "should set n = now"); + } + + #[test] + fn expiring_set_without_expiry_keeps_stored_exp_only_when_unchanged() { + let provider = expiring_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let catalog = ProviderCatalog::new( + vec![provider.clone()], + BTreeMap::from([(a.clone(), pid("p"))]), + ); + let fixture = OutcomeFixture::new().with_catalog(catalog); + let mut stored = record_for_test("example-a", Some("p"), 900); + stored.exp = Some(5_000); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&a, stored); + + let unchanged = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-a"), + )])), + &view, + &fixture.context_at(1_000), + ); + let changed = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-other"), + )])), + &view, + &fixture.context_at(1_000), + ); + let with_expiry = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-other"), + ) + .with_expires_in(Duration::from_secs(60))])), + &view, + &fixture.context_at(1_000), + ); + + let kept = positive_record(&unchanged.changes[0]).expect("should store the unchanged set"); + assert_eq!(kept.exp, Some(5_000), "should keep the stored exp"); + assert_eq!(unchanged.state.f, Some(0), "should count as found"); + + assert!( + changed.changes.is_empty(), + "should discard a changed set that carries no expiry" + ); + assert_eq!( + changed.state.f, + Some(1), + "should handle a result whose every set is discarded as Malformed" + ); + + let fresh = positive_record(&with_expiry.changes[0]).expect("should store the set"); + assert_eq!(fresh.exp, Some(1_060), "should use the result's own expiry"); + } + + #[test] + fn result_without_a_source_leaves_its_set_alone() { + let provider = fixed_provider("p", &["a.example.com", "b.example.com"]); + let a = source("a.example.com"); + let b = source("b.example.com"); + let mut view = EligibilityView::empty(); + view.set_record_for_test(&b, record_for_test("example-b", Some("p"), 900)); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-a"), + )])), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!( + changes.changes.len(), + 1, + "should change only the named source" + ); + assert!( + matches!(&changes.changes[0], PartnerIdChange::FillMissing { source, .. } if *source == a), + "should leave the source the result did not name" + ); + } + + #[test] + fn record_carries_the_earliest_times_of_its_set() { + let provider = stateful_fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + let resolution = Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a")) + .with_expires_in(Duration::from_secs(600)) + .with_refresh_in(Duration::from_secs(300)) + .with_refresh_until(UnixTime::from_secs(9_000)), + ResolvedEid::new(&a, ResolvedUid::new("example-b")) + .with_expires_in(Duration::from_secs(60)) + .with_refresh_in(Duration::from_secs(900)) + .with_refresh_until(UnixTime::from_secs(8_000)), + ]); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(resolution), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + let result = changes.result.expect("should produce one result"); + let record = positive_record(&result.ids[0]).expect("should be a positive change"); + assert_eq!(record.exp, Some(1_060), "should keep the earliest expiry"); + assert_eq!(record.ra, Some(1_300), "should keep the earliest refresh"); + assert_eq!( + record.ru, + Some(8_000), + "should keep the earliest refresh limit" + ); + } + + #[test] + fn refresh_until_is_ignored_for_a_stateless_provider() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ResolvedEid::new( + &a, + ResolvedUid::new("example-a"), + ) + .with_refresh_until(UnixTime::from_secs(9_000))])), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + let record = positive_record(&changes.changes[0]).expect("should be a positive change"); + assert_eq!( + record.ru, None, + "should ignore refresh_until without private state" + ); + } + + #[test] + fn duplicate_entries_collapse_into_one_entry() { + // A duplicate entry is dropped, so the kept set has one entry. + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a")), + ResolvedEid::new(&a, ResolvedUid::new("example-a")), + ])), + &EligibilityView::empty(), + &fixture.context_at(1_000), + ); + + let record = positive_record(&changes.changes[0]).expect("should be a positive change"); + assert_eq!( + record.eids, + vec![one_entry("example-a")], + "should drop the duplicate" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index aed5a6cf9d..87b57812c7 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -370,25 +370,23 @@ impl Resolves { } /// Gives the provider the full EC ID. Only generic pull may use this. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(in crate::ec::identity) fn with_ec_id(mut self) -> Self { self.ec_id = true; self } /// Hosts the provider may call. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn hosts(&self) -> &[String] { &self.hosts } /// Inputs required for `kind`: the common needs plus that kind's. - // Temporary: first used by a later identity task. - #[allow(dead_code)] #[must_use] pub(crate) fn needs(&self, kind: ResolveKind) -> BTreeSet { let mut needs = self.needs.clone(); @@ -399,9 +397,9 @@ impl Resolves { } /// Every input declared for any kind, needed or wanted. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn declared_inputs(&self) -> BTreeSet { let mut all = self.needs.clone(); all.extend(self.wants.iter().copied()); @@ -412,33 +410,29 @@ impl Resolves { } /// Inputs passed when available. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn wants(&self) -> &BTreeSet { &self.wants } /// The kinds this provider handles. - // Temporary: first used by a later identity task. - #[allow(dead_code)] #[must_use] pub(crate) fn kinds(&self) -> &[ResolveKind] { &self.kinds } /// Whether the provider keeps private state. - // Temporary: first used by a later identity task. - #[allow(dead_code)] #[must_use] pub(crate) fn has_private_state(&self) -> bool { self.private_state } /// Whether the provider receives the full EC ID. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn gets_ec_id(&self) -> bool { self.ec_id } @@ -476,9 +470,9 @@ impl Captures { } /// Whether a proxied request matches a rule. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn matches(&self, method: &Method, path: &str) -> bool { self.paths .iter() @@ -490,17 +484,17 @@ impl Captures { } /// Exact path rules. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn paths(&self) -> &[(Method, String)] { &self.paths } /// Prefix rules. + #[must_use] // Temporary: first used by a later identity task. #[allow(dead_code)] - #[must_use] pub(crate) fn prefixes(&self) -> &[String] { &self.prefixes } diff --git a/crates/trusted-server-core/src/ec/identity/exchange.rs b/crates/trusted-server-core/src/ec/identity/exchange.rs index ae493fe0de..6c5d1650a2 100644 --- a/crates/trusted-server-core/src/ec/identity/exchange.rs +++ b/crates/trusted-server-core/src/ec/identity/exchange.rs @@ -185,17 +185,17 @@ impl ProviderRequest { } /// The call context JSON. - // Temporary: the coordinator, the first non-test caller, arrives in a later task. - #[allow(dead_code)] #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn call_context_json(&self) -> Option<&str> { self.call_context.as_deref() } /// Whether the provider set a `Cookie` header. - // Temporary: the coordinator, the first non-test caller, arrives in a later task. - #[allow(dead_code)] #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn has_cookie_header(&self) -> bool { self.headers.contains_key(COOKIE) } @@ -222,7 +222,7 @@ impl fmt::Debug for ProviderResponse { impl ProviderResponse { /// Built by the coordinator after buffering and decoding. - // Temporary: the coordinator, the first non-test caller, arrives in a later task. + // Temporary: the dispatcher, the first non-test caller, arrives in a later task. #[allow(dead_code)] #[must_use] pub(crate) fn new( diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs index 79f944d7dc..4ca12bae43 100644 --- a/crates/trusted-server-core/src/ec/identity/input.rs +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -1,9 +1,5 @@ //! What a provider may read for one call. -// Temporary: the coordinator, the first non-test caller, arrives in a later -// task. Remove this when it lands. -#![allow(dead_code)] - use core::cell::RefCell; use core::fmt; use std::collections::{BTreeMap, BTreeSet}; @@ -59,6 +55,8 @@ impl fmt::Debug for ProviderConsent { impl ProviderConsent { /// The view of a request's consent context. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn from_context(context: &ConsentContext) -> Self { Self { gdpr_applies: context.gdpr_applies, @@ -111,6 +109,8 @@ impl Region { /// Normalizes platform geo: upper case, and an ISO 3166-2 `US-CA` style /// region loses its country prefix. `None` without a two-letter country. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn from_geo(country: &str, region: Option<&str>) -> Option { let country = country.trim().to_ascii_uppercase(); if country.len() != 2 || !country.bytes().all(|byte| byte.is_ascii_alphabetic()) { @@ -221,6 +221,8 @@ impl StoredId { /// treated as none. `None` for a record with no entry, which /// `KvEntry::validate` never lets through. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn from_record(source: &SourceDomain, record: &KvPartnerId) -> Option { let uid = record.first_uid()?.to_owned(); let eids = record @@ -323,6 +325,8 @@ pub(crate) trait RandomSource: Send + Sync { } /// The platform generator EC generation uses (`rand::thread_rng`). +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) struct ThreadRandom; impl RandomSource for ThreadRandom { @@ -371,6 +375,8 @@ pub(crate) struct ResolveSignals<'a> { pub(crate) publisher_domain: &'a str, pub(crate) now: UnixTime, pub(crate) random: &'a dyn RandomSource, + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) ec_id: Option, pub(crate) stored: BTreeMap, pub(crate) state_json: Option, @@ -404,6 +410,8 @@ impl<'a> ResolveInput<'a> { /// declared, so reading it is not an error of the provider, but it is /// unavailable (spec 4.4). #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn new( descriptor: &'a ProviderDescriptor, kinds: BTreeMap, @@ -454,12 +462,16 @@ impl<'a> ResolveInput<'a> { /// Undeclared reads recorded so far. The coordinator logs them as /// `Config`, and the harness fails the test. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn recorded_errors(&self) -> Vec { self.undeclared.borrow().clone() } /// Whether a `state()` call failed to decode. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn state_decode_failed(&self) -> bool { *self.state_failed.borrow() } @@ -606,6 +618,8 @@ impl<'a> ResolveInput<'a> { /// `UndeclaredInput { access: EcId }` otherwise. `Config` when the /// request has no EC ID, which the coordinator never allows: it does not /// dispatch generic pull without one. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report> { if !self.descriptor.resolves().is_some_and(Resolves::gets_ec_id) { return Err(self.record(Access::EcId)); @@ -640,6 +654,8 @@ impl fmt::Debug for CaptureInput<'_> { impl<'a> CaptureInput<'a> { /// The input for one capture. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn new( descriptor: &'a ProviderDescriptor, path: &'a str, diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 7f163dd652..525cf99ea9 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -7,6 +7,7 @@ //! write. See `docs/superpowers/specs/2026-10-20-server-side-identity-providers-design.md`. pub(crate) mod catalog; +pub(crate) mod coordinator; pub mod descriptor; pub mod exchange; pub mod ids; diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs index 23de933244..71db219cf9 100644 --- a/crates/trusted-server-core/src/ec/identity/mutation.rs +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -1,10 +1,6 @@ //! One request's identity changes as a single mutation, and its merge into //! the live KV row (spec 5.4). -// Temporary: the coordinator, the first non-test caller, arrives in a later -// task. Remove this when it lands. -#![allow(dead_code)] - use std::collections::BTreeMap; use super::catalog::ProviderCatalog; @@ -53,6 +49,8 @@ pub(crate) enum PartnerIdChange { } impl PartnerIdChange { + // Temporary: first used by a later identity task. + #[allow(dead_code)] fn is_removal(&self) -> bool { matches!(self, Self::OwnerRemove { .. } | Self::WriterRemove { .. }) } @@ -82,14 +80,20 @@ pub(crate) struct ProviderStateChange { } impl ProviderStateChange { + // Temporary: first used by a later identity task. + #[allow(dead_code)] fn is_noop(&self) -> bool { *self == Self::default() } + // Temporary: first used by a later identity task. + #[allow(dead_code)] fn has_removal(&self) -> bool { self.opt_out.is_some() || self.opt_out_clear.is_some() } + // Temporary: first used by a later identity task. + #[allow(dead_code)] fn removals(&self) -> Self { Self { opt_out: self.opt_out.clone(), @@ -98,6 +102,8 @@ impl ProviderStateChange { } } + // Temporary: first used by a later identity task. + #[allow(dead_code)] fn merge(&mut self, later: Self) { self.n = self.n.max(later.n); self.f = later.f.or(self.f); @@ -125,6 +131,8 @@ pub(crate) struct ProviderResult { /// Everything one request changes in the identity row. #[derive(Debug, Clone, Default, PartialEq)] +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) struct IdentityMutation { partner_ids: Vec, provider_state: BTreeMap, @@ -134,6 +142,8 @@ pub(crate) struct IdentityMutation { impl IdentityMutation { /// Adds a change that stands alone. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn add_change(&mut self, change: PartnerIdChange) { self.partner_ids.push(change); } @@ -143,6 +153,8 @@ impl IdentityMutation { /// A removal in `result.ids` is a provider-side mistake: removals go /// through `partner_ids` and `provider_state`. It is logged by provider /// and source, never by UID. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn add_result(&mut self, result: ProviderResult) { for change in result.ids.iter().filter(|change| change.is_removal()) { let (PartnerIdChange::OwnerRemove { source, .. } @@ -162,6 +174,8 @@ impl IdentityMutation { /// Merges a state change into any earlier one for the same provider: /// later `n`, last `f`, the later opt-out and clear, last non-`Keep` /// `st`. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn merge_state( &mut self, provider: IdentityProviderId, @@ -174,12 +188,16 @@ impl IdentityMutation { } /// Sets the cluster size to write. + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn set_cluster_size(&mut self, size: u32) { self.cluster_size = Some(size); } /// Whether nothing would change. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn is_empty(&self) -> bool { self.partner_ids.is_empty() && self.results.is_empty() @@ -192,6 +210,8 @@ impl IdentityMutation { /// Whether any removal, opt-out or clear is present. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn has_removals(&self) -> bool { self.partner_ids.iter().any(PartnerIdChange::is_removal) || self @@ -205,6 +225,8 @@ impl IdentityMutation { /// de-duplicated. An opt-out or clear has no source. Holds no UID, so it /// is safe to log. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn removal_labels(&self) -> Vec<(IdentityProviderId, Option)> { let mut labels: Vec<_> = self .partner_ids @@ -232,6 +254,8 @@ impl IdentityMutation { /// A copy holding only the removals, opt-outs and clears, for the /// removal write and the removal-only retry. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn removals(&self) -> Self { let mut copy = Self { partner_ids: self @@ -257,6 +281,8 @@ impl IdentityMutation { /// Every provider state change that applies regardless of its result: /// the standalone ones, and each result's. + // Temporary: first used by a later identity task. + #[allow(dead_code)] fn all_states(&self) -> impl Iterator { self.provider_state.iter().chain( self.results @@ -267,6 +293,8 @@ impl IdentityMutation { } /// What the merge needs besides the mutation and the live row. +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) struct ApplyContext<'a> { pub(crate) now: u64, pub(crate) random: &'a dyn RandomSource, @@ -275,6 +303,8 @@ pub(crate) struct ApplyContext<'a> { /// The merged row. #[derive(Debug, Clone, PartialEq)] +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) struct Applied { pub(crate) entry: KvEntry, /// Whether the result differs from the live row. A caller skips the @@ -347,6 +377,8 @@ pub(crate) struct Applied { /// breaks one is dropped whole here and never trimmed. /// 6. `changed` is whether the result differs from `live`. #[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) fn apply( mutation: &IdentityMutation, live: &KvEntry, @@ -382,6 +414,8 @@ pub(crate) fn apply( /// for the capture removal write and the removal-only retry. It ignores every /// positive change, `n`, `f`, `st` and `cluster_size`. #[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) fn apply_removals_only( mutation: &IdentityMutation, live: &KvEntry, @@ -407,12 +441,16 @@ pub(crate) fn apply_removals_only( }) } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn state_mut<'a>(entry: &'a mut KvEntry, provider: &IdentityProviderId) -> &'a mut KvProviderState { entry.ps.entry(provider.to_string()).or_default() } /// Removes the record at `source` when `matches` accepts it, and hashes its /// set into the list `list` picks from `provider`'s state. +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn remove_record( entry: &mut KvEntry, source: &str, @@ -429,14 +467,20 @@ fn remove_record( } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn invalidated(state: &mut KvProviderState) -> &mut Vec { &mut state.iv } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn opted_out_hashes(state: &mut KvProviderState) -> &mut Vec { &mut state.ov } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn apply_removals(entry: &mut KvEntry, mutation: &IdentityMutation, context: &ApplyContext<'_>) { for change in &mutation.partner_ids { match change { @@ -483,6 +527,8 @@ fn apply_removals(entry: &mut KvEntry, mutation: &IdentityMutation, context: &Ap } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn apply_opt_out( entry: &mut KvEntry, provider: &IdentityProviderId, @@ -518,6 +564,8 @@ fn apply_opt_out( } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn apply_clear(entry: &mut KvEntry, provider: &IdentityProviderId, clear: u64) { let Some(state) = entry.ps.get_mut(provider.as_str()) else { return; @@ -528,6 +576,8 @@ fn apply_clear(entry: &mut KvEntry, provider: &IdentityProviderId, clear: u64) { } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn is_opted_out(entry: &KvEntry, provider: &str) -> bool { entry .ps @@ -536,10 +586,14 @@ fn is_opted_out(entry: &KvEntry, provider: &str) -> bool { .is_some_and(|(opt_out, cleared)| cleared.is_none_or(|cleared| cleared <= opt_out.at)) } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn is_expired(record: &KvPartnerId, now: u64) -> bool { record.exp.is_some_and(|exp| exp <= now) } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn change_applies(entry: &KvEntry, change: &PartnerIdChange, context: &ApplyContext<'_>) -> bool { match change { PartnerIdChange::FillMissing { source, record } => { @@ -569,6 +623,8 @@ fn change_applies(entry: &KvEntry, change: &PartnerIdChange, context: &ApplyCont } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn fill_applies( entry: &KvEntry, source: &SourceDomain, @@ -612,6 +668,8 @@ fn fill_applies( }) } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn write_change(entry: &mut KvEntry, change: &PartnerIdChange, context: &ApplyContext<'_>) { let (PartnerIdChange::FillMissing { source, record } | PartnerIdChange::OwnerWrite { source, record, .. }) = change @@ -626,6 +684,8 @@ fn write_change(entry: &mut KvEntry, change: &PartnerIdChange, context: &ApplyCo /// Applies the positive changes and returns the state changes that go with /// them: every standalone one, and those of the results that applied. +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn apply_positive<'a>( entry: &mut KvEntry, mutation: &'a IdentityMutation, @@ -657,6 +717,8 @@ fn apply_positive<'a>( accepted } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn apply_state( entry: &mut KvEntry, mutation: &IdentityMutation, @@ -683,12 +745,16 @@ fn apply_state( } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn is_configured(entry_key: &str, context: &ApplyContext<'_>) -> bool { entry_key .parse::() .is_ok_and(|id| context.catalog.is_configured(&id)) } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn finish_removal_fields(entry: &mut KvEntry, context: &ApplyContext<'_>) { for state in entry.ps.values_mut() { for list in [&mut state.iv, &mut state.ov] { @@ -725,6 +791,8 @@ fn finish_removal_fields(entry: &mut KvEntry, context: &ApplyContext<'_>) { } } +// Temporary: first used by a later identity task. +#[allow(dead_code)] fn drop_expired_records(entry: &mut KvEntry, context: &ApplyContext<'_>) { let ps = &entry.ps; entry.ids.retain(|source, record| { diff --git a/crates/trusted-server-core/src/ec/identity/policy.rs b/crates/trusted-server-core/src/ec/identity/policy.rs index 49531faf02..8ca993ce69 100644 --- a/crates/trusted-server-core/src/ec/identity/policy.rs +++ b/crates/trusted-server-core/src/ec/identity/policy.rs @@ -37,7 +37,7 @@ pub(crate) fn tcf_applies(consent: &ConsentContext) -> bool { /// Whether the request carries a decoded US sale opt-out, from GPP or from /// the US Privacy string. -fn us_sale_opt_out(consent: &ConsentContext) -> bool { +pub(crate) fn us_sale_opt_out(consent: &ConsentContext) -> bool { consent.gpp.as_ref().and_then(|gpp| gpp.us_sale_opt_out) == Some(true) || consent .us_privacy diff --git a/crates/trusted-server-core/src/ec/identity/resolution.rs b/crates/trusted-server-core/src/ec/identity/resolution.rs index d5fa6d89dc..ae58a0d245 100644 --- a/crates/trusted-server-core/src/ec/identity/resolution.rs +++ b/crates/trusted-server-core/src/ec/identity/resolution.rs @@ -1,9 +1,5 @@ //! Provider results and errors. -// Temporary: the coordinator, the first non-test caller, arrives in a later -// task. Remove this when it lands. -#![allow(dead_code)] - use std::collections::BTreeMap; use std::iter; use std::time::Duration; @@ -91,9 +87,9 @@ impl IdentityProviderError { } /// Whether this is the [`Self::not_implemented`] marker. - // Temporary: first used by the provider test harness in a later task. - #[allow(dead_code)] #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn is_not_implemented(&self) -> bool { matches!(self, Self::Config { message } if message.starts_with(NOT_IMPLEMENTED_MARKER)) } @@ -111,8 +107,6 @@ pub enum When { impl When { /// Resolves against `now`. #[must_use] - // Temporary: the coordinator, the first non-test caller, arrives in a later task. - #[allow(dead_code)] pub(crate) fn resolve(self, now: UnixTime) -> UnixTime { match self { Self::In(delay) => now + delay, @@ -548,6 +542,8 @@ impl Resolution { /// Whether this result changes state at all. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn changes_state(&self) -> bool { self.state != StateOp::Keep } diff --git a/crates/trusted-server-core/src/ec/identity/revision.rs b/crates/trusted-server-core/src/ec/identity/revision.rs index cca095aee9..c0ea5ef6e6 100644 --- a/crates/trusted-server-core/src/ec/identity/revision.rs +++ b/crates/trusted-server-core/src/ec/identity/revision.rs @@ -7,6 +7,8 @@ use super::input::RandomSource; use crate::ec::kv_types::{KvEid, KvPartnerId, RECORD_REVISION_HEX_LEN}; use crate::settings::Settings; +// Temporary: first used by a later identity task. +#[allow(dead_code)] type HmacSha256 = Hmac; /// The visitor key, derived from `ec.passphrase` once per settings load. @@ -101,6 +103,8 @@ impl StateRevision { /// the empty string when there is none. Compact JSON is never empty, so /// absent and present state never share a revision. #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] pub(crate) fn of(st: Option<&serde_json::Value>) -> Self { let bytes = st .and_then(|value| serde_json::to_vec(value).ok()) @@ -118,6 +122,8 @@ pub(crate) fn uid_hash(uid: &str) -> String { /// One hash per UID of a removed set, in set order, without duplicates. #[must_use] +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) fn set_hashes(set: &[KvEid]) -> Vec { let mut hashes = Vec::new(); for uid in set.iter().flat_map(|entry| &entry.uids) { @@ -131,6 +137,8 @@ pub(crate) fn set_hashes(set: &[KvEid]) -> Vec { /// Appends hashes to a `ps.iv` or `ps.ov` list, oldest first. A hash /// already present moves to the end. The caller truncates. +// Temporary: first used by a later identity task. +#[allow(dead_code)] pub(crate) fn push_hashes(list: &mut Vec, hashes: Vec) { for hash in hashes { list.retain(|existing| *existing != hash); diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index b9090804de..25c572a6a6 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -8,6 +8,7 @@ ) )] +use std::collections::BTreeSet; use std::sync::Arc; use serde_json::json; @@ -15,9 +16,18 @@ use serde_json::json; use crate::consent::ConsentContext; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::TcfConsent; -use crate::ec::identity::descriptor::{Captures, ConsentPolicy, ProviderDescriptor, Resolves}; +use crate::ec::identity::catalog::ProviderCatalog; +use crate::ec::identity::coordinator::{OutcomeContext, RequestFacts}; +use crate::ec::identity::descriptor::{ + Captures, ConsentPolicy, Input, ProviderDescriptor, Resolves, +}; +use crate::ec::identity::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use crate::ec::identity::input::SeededRandom; +use crate::ec::identity::policy::PolicySignals; use crate::ec::identity::provider::IdentityProvider; +use crate::ec::identity::revision::{StateRevision, new_revision}; use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; +use crate::settings::{IdentitySettings, ProviderSettings}; /// A request with no consent signal and no regulated jurisdiction. #[must_use] @@ -138,6 +148,162 @@ pub fn fixed_capture_provider(id: &str, sources: &[&str]) -> Arc Arc { + Arc::new(DescriptorOnly(descriptor)) +} + +/// A `fixed_provider` declared `with_private_state()`. +/// +/// # Panics +/// +/// Panics when `id` or a source is not valid. +#[must_use] +pub fn stateful_fixed_provider(id: &str, sources: &[&str]) -> Arc { + let mut builder = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_resolves( + Resolves::new() + .with_endpoint("https://api.example.com") + .with_private_state(), + ) + .with_consent_policy(ConsentPolicy::eid_persistence_only()); + for source in sources { + builder = builder.with_source(source.parse().expect("should parse the source")); + } + provider_from(builder.build().expect("should build the descriptor")) +} + +/// A request with no consent signal and no regulated jurisdiction. +#[must_use] +pub fn allowing_consent() -> ConsentContext { + test_consent_allowing_everything() +} + +/// A source domain. +/// +/// # Panics +/// +/// Panics when `domain` is not valid. +#[must_use] +pub fn source(domain: &str) -> SourceDomain { + domain.parse().expect("should parse the source") +} + +/// A provider ID. +/// +/// # Panics +/// +/// Panics when `id` is not valid. +#[must_use] +pub fn pid(id: &str) -> IdentityProviderId { + id.parse().expect("should parse the provider ID") +} + +/// A one-UID record with a fresh `rv`, written by `writer` at `at`. +#[must_use] +pub(crate) fn record_for_test(uid: &str, writer: Option<&str>, at: u64) -> KvPartnerId { + KvPartnerId { + w: writer.map(str::to_owned), + rv: Some(new_revision(&SeededRandom::new(at)).as_str().to_owned()), + at: Some(at), + ..KvPartnerId::new(uid) + } +} + +/// Facts for a returning visitor's writer navigation that runs no auction: +/// a real browser, persistence allowed and every input available. +#[must_use] +pub(crate) fn facts_for_writer() -> RequestFacts { + RequestFacts { + is_real_browser: true, + ec_persistence_allowed: true, + policy_signals: PolicySignals::default(), + writer: true, + new_visitor: false, + runs_auction: false, + now: UnixTime::from_secs(1_000), + available_inputs: BTreeSet::from([ + Input::ClientIp, + Input::UserAgent, + Input::PageUrl, + Input::Region, + Input::HashedEmail, + ]), + } +} + +/// An owned fixture for [`OutcomeContext`], which borrows from it. +/// +/// `new` holds a catalog with provider `p` claiming `a.example.com` and +/// `b.example.com`, no source owners and default settings. +pub(crate) struct OutcomeFixture { + catalog: ProviderCatalog, + settings: ProviderSettings, + identity: IdentitySettings, + expected_state: Option, + fingerprint: String, +} + +impl OutcomeFixture { + #[must_use] + pub(crate) fn new() -> Self { + Self { + catalog: ProviderCatalog::new( + vec![fixed_provider("p", &["a.example.com", "b.example.com"])], + std::collections::BTreeMap::new(), + ), + settings: ProviderSettings::default(), + identity: IdentitySettings::default(), + expected_state: None, + fingerprint: "0".to_owned(), + } + } + + /// Replaces the catalog. + #[must_use] + pub(crate) fn with_catalog(mut self, catalog: ProviderCatalog) -> Self { + self.catalog = catalog; + self + } + + /// Replaces the provider's settings. + #[must_use] + pub(crate) fn with_settings(mut self, settings: ProviderSettings) -> Self { + self.settings = settings; + self + } + + /// Sets the consent fingerprint an opt-out records. + #[must_use] + pub(crate) fn with_fingerprint(mut self, fingerprint: &str) -> Self { + fingerprint.clone_into(&mut self.fingerprint); + self + } + + /// Sets the state revision the call read. + #[must_use] + pub(crate) fn with_expected_state(mut self, expected_state: StateRevision) -> Self { + self.expected_state = Some(expected_state); + self + } + + /// The context for an outcome at `now`. + #[must_use] + pub(crate) fn context_at(&self, now: u64) -> OutcomeContext<'_> { + OutcomeContext { + now: UnixTime::from_secs(now), + catalog: &self.catalog, + settings: &self.settings, + identity: &self.identity, + expected_state: self.expected_state.clone(), + fingerprint: &self.fingerprint, + } + } +} + struct DescriptorOnly(ProviderDescriptor); impl IdentityProvider for DescriptorOnly { diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index c8f256da0b..d212fee216 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -87,6 +87,10 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { "ec/identity/catalog.rs", include_str!("ec/identity/catalog.rs"), ), + ( + "ec/identity/coordinator.rs", + include_str!("ec/identity/coordinator.rs"), + ), ( "ec/identity/descriptor.rs", include_str!("ec/identity/descriptor.rs"), From b0d9ca2981f6fdb3fb0e62f2efb9ab3248b7614e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 21:11:25 +0530 Subject: [PATCH 28/89] Handle a result naming only unclaimed sources as Malformed drop_unclaimed now reports whether it dropped anything, so a result whose every entry was for an unclaimed source backs off instead of resetting the failure count. Signed-off-by: dhruv8sh --- .../src/ec/identity/coordinator.rs | 59 +++++++++++++++---- .../src/ec/identity/test_support.rs | 2 +- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-core/src/ec/identity/coordinator.rs b/crates/trusted-server-core/src/ec/identity/coordinator.rs index 123efff7aa..bee23eeb10 100644 --- a/crates/trusted-server-core/src/ec/identity/coordinator.rs +++ b/crates/trusted-server-core/src/ec/identity/coordinator.rs @@ -634,13 +634,13 @@ fn resolved_changes( let now = context.now.as_secs(); let stateful = keeps_private_state(descriptor); - let kept = drop_unclaimed(id, resolution.eids(), context.catalog); + let (kept, dropped_unclaimed) = drop_unclaimed(id, resolution.eids(), context.catalog); let mut sets = entry_sets(&kept, context.now); log_entry_sets(id, &sets); let discarded = discarded_sets(descriptor, &mut sets, view); let malformed = resolution.state_rejected() || reject_positive_part(stateful, &discarded) - || all_sets_discarded(&sets, &discarded); + || all_sets_discarded(&sets, &discarded, dropped_unclaimed); if malformed { log::warn!("Identity provider '{id}' returned a result the coordinator rejects"); } @@ -718,17 +718,21 @@ fn resolved_changes( } /// Step 1: drops the entries of sources the provider does not claim after -/// narrowing. Logs each source with the provider ID only. +/// narrowing. Logs each source with the provider ID only. Also reports +/// whether it dropped anything, which step 5 needs. fn drop_unclaimed( provider: &IdentityProviderId, eids: &[ResolvedEid], catalog: &ProviderCatalog, -) -> Vec { +) -> (Vec, bool) { let claimed = catalog.sources_of(provider); - eids.iter() + let mut dropped = false; + let kept = eids + .iter() .filter(|eid| { let keep = claimed.contains(eid.source()); if !keep { + dropped = true; log::debug!( "Dropping an entry of identity provider '{provider}' for unclaimed source '{}'", eid.source() @@ -737,7 +741,8 @@ fn drop_unclaimed( keep }) .cloned() - .collect() + .collect(); + (kept, dropped) } /// Step 2: logs each dropped part and oversized source, never a value. @@ -793,9 +798,15 @@ fn reject_positive_part(stateful: bool, discarded: &BTreeSet) -> b stateful && !discarded.is_empty() } -/// Step 5: whether the result named a source and every set is discarded. -fn all_sets_discarded(sets: &EntrySets, discarded: &BTreeSet) -> bool { +/// Step 5: whether the result named a source and every set is discarded, +/// including by dropping entries of unclaimed sources (spec 4.5). +fn all_sets_discarded( + sets: &EntrySets, + discarded: &BTreeSet, + dropped_unclaimed: bool, +) -> bool { sets.all_discarded() + || (sets.sets.is_empty() && dropped_unclaimed) || (!sets.sets.is_empty() && sets.sets.keys().all(|source| discarded.contains(source))) } @@ -2568,7 +2579,7 @@ mod tests { } #[test] - fn found_naming_only_unclaimed_sources_stores_nothing_and_resets_failures() { + fn found_naming_only_unclaimed_sources_is_malformed() { let provider = fixed_provider("p", &["a.example.com"]); let c = source("c.example.com"); let mut view = EligibilityView::empty(); @@ -2586,7 +2597,35 @@ mod tests { ); assert!(changes.changes.is_empty(), "should store nothing"); - assert_eq!(changes.state.f, Some(0), "should still reset f on found"); + assert_eq!(changes.state.f, Some(4), "should count a failure"); + assert_eq!( + changes.state.n, + Some(1_000 + 28_800), + "should back off min(1 h x 2^3, 24 h)" + ); + } + + #[test] + fn one_claimed_and_one_unclaimed_source_still_resets_failures() { + let provider = fixed_provider("p", &["a.example.com"]); + let a = source("a.example.com"); + let c = source("c.example.com"); + let mut view = EligibilityView::empty(); + view.set_failures_for_test("p", 3); + let fixture = OutcomeFixture::new(); + + let changes = outcome_to_changes( + provider.as_ref(), + CallOutcome::Resolved(Resolution::found([ + ResolvedEid::new(&a, ResolvedUid::new("example-a")), + ResolvedEid::new(&c, ResolvedUid::new("example-c")), + ])), + &view, + &fixture.context_at(1_000), + ); + + assert_eq!(changes.changes.len(), 1, "should store the claimed source"); + assert_eq!(changes.state.f, Some(0), "should reset f on found"); } #[test] diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 25c572a6a6..31e74eab88 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -204,7 +204,7 @@ pub fn pid(id: &str) -> IdentityProviderId { /// A one-UID record with a fresh `rv`, written by `writer` at `at`. #[must_use] -pub(crate) fn record_for_test(uid: &str, writer: Option<&str>, at: u64) -> KvPartnerId { +pub fn record_for_test(uid: &str, writer: Option<&str>, at: u64) -> KvPartnerId { KvPartnerId { w: writer.map(str::to_owned), rv: Some(new_revision(&SeededRandom::new(at)).as_str().to_owned()), From 2c0052975abcd91b3b2cb487eed3d114be55cb74 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 21:22:11 +0530 Subject: [PATCH 29/89] Add coordinator dispatch with request checks and byte caps Dispatch builds each provider call, checks it is https to a declared host with no Cookie header, applies the burst cap and sends it with at most max_concurrency in flight. Responses are bounded by max_response_bytes before and after decoding, and each result maps to a CallOutcome. The Content-Length check, body buffering and in-flight drain move out of pull_sync into the coordinator so both share them. The module-wide dead_code allow is narrowed to the entry points that have no non-test caller yet, and the allows on the items dispatch consumes are removed. Signed-off-by: dhruv8sh --- .../src/ec/identity/coordinator.rs | 1061 ++++++++++++++++- .../src/ec/identity/descriptor.rs | 8 - .../src/ec/identity/exchange.rs | 4 - .../src/ec/identity/input.rs | 6 - .../src/ec/identity/revision.rs | 2 - .../src/ec/identity/test_support.rs | 314 +++++ .../trusted-server-core/src/ec/pull_sync.rs | 90 +- 7 files changed, 1377 insertions(+), 108 deletions(-) diff --git a/crates/trusted-server-core/src/ec/identity/coordinator.rs b/crates/trusted-server-core/src/ec/identity/coordinator.rs index bee23eeb10..b0a5382582 100644 --- a/crates/trusted-server-core/src/ec/identity/coordinator.rs +++ b/crates/trusted-server-core/src/ec/identity/coordinator.rs @@ -1,28 +1,39 @@ -//! Coordinator eligibility and outcomes (spec 4.7 and 8). +//! Coordinator eligibility, dispatch and outcomes (spec 4.7 and 8). //! -//! Pure logic: which of a provider's sources need a call, and what stored -//! changes a call's outcome becomes. Dispatch, the breaker and the burst cap -//! live elsewhere. - -// Temporary: the dispatcher, the first non-test caller, arrives in a later -// task. Remove this when it lands. -#![allow(dead_code)] +//! Which of a provider's sources need a call, how the calls are sent under +//! the transport limits, and what stored changes a call's outcome becomes. use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::Report; +use http::{StatusCode, header}; +use web_time::Instant; use super::catalog::ProviderCatalog; use super::descriptor::{ ConsentPolicy, Input, ProviderDescriptor, ResolveKind, Resolves, UsOptOut, }; +use super::exchange::{ProviderRequest, ProviderResponse}; use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::input::{RandomSource, ResolveInput, ResolveSignals}; use super::mutation::{PartnerIdChange, ProviderResult, ProviderStateChange, StateUpdate}; use super::policy::{PolicySignals, policy_allows, tcf_applies, us_sale_opt_out}; use super::provider::IdentityProvider; -use super::resolution::{EntrySet, EntrySets, Resolution, ResolvedEid, StateChange, entry_sets}; -use super::revision::{StateRevision, record_revision, uid_hash}; +use super::resolution::{ + EntrySet, EntrySets, IdentityProviderError, Resolution, ResolvedEid, StateChange, entry_sets, +}; +use super::revision::{IdentityKeys, StateRevision, record_revision, uid_hash, visitor_key}; use crate::consent::{ConsentContext, effective_tcf}; use crate::ec::kv_types::{KvEntry, KvOptOut, KvPartnerId}; -use crate::settings::{IdentitySettings, ProviderMode, ProviderSettings}; +use crate::ec::rate_limiter::RateLimiter; +use crate::platform::{ + PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, + RuntimeServices, +}; +use crate::settings::{IdentitySettings, ProviderMode, ProviderSettings, Settings}; +use crate::streaming_processor::{BodyStreamDecoder, Compression}; /// The highest TCF purpose the fingerprint has a bit for. const FINGERPRINT_PURPOSES: u8 = 10; @@ -51,6 +62,8 @@ const TRANSPORT_BACKOFF_BASE_SECS: u64 = 300; const TRANSPORT_BACKOFF_CAP_SECS: u64 = 3_600; /// One provider's `ps` entry, as the request read it. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] #[derive(Debug, Clone, Default)] struct ProviderView { n: Option, @@ -65,6 +78,8 @@ struct ProviderView { /// What eligibility and outcomes read from the KV row the request read /// (spec 4.7), and nothing else. It never reads `ts-eids-s`: every removal is /// already in KV (spec 5.3). +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] #[derive(Debug, Clone, Default)] pub(crate) struct EligibilityView { records: BTreeMap, @@ -76,6 +91,8 @@ impl EligibilityView { /// source domain or provider ID cannot name a configured one and is /// skipped. #[must_use] + // Temporary: the request wiring, the first non-test caller, arrives in a later task. + #[allow(dead_code)] pub(crate) fn from_kv(entry: &KvEntry) -> Self { let records = entry .ids @@ -105,6 +122,8 @@ impl EligibilityView { /// A view of a row with no records and no provider state. #[must_use] + // Temporary: the request wiring, the first non-test caller, arrives in a later task. + #[allow(dead_code)] pub(crate) fn empty() -> Self { Self::default() } @@ -134,6 +153,8 @@ impl EligibilityView { /// The provider's private state `st`, which a call reads. #[must_use] + // Temporary: the request wiring, the first non-test caller, arrives in a later task. + #[allow(dead_code)] pub(crate) fn state(&self, provider: &IdentityProviderId) -> Option<&serde_json::Value> { self.providers .get(provider) @@ -142,6 +163,8 @@ impl EligibilityView { /// Whether the provider's opt-out removed a UID with this hash, `ps.ov`. #[must_use] + // Temporary: the request wiring, the first non-test caller, arrives in a later task. + #[allow(dead_code)] pub(crate) fn has_opt_out_hash(&self, provider: &IdentityProviderId, hash: &str) -> bool { self.providers .get(provider) @@ -204,6 +227,8 @@ impl EligibilityView { } /// What the request is, as eligibility reads it. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] #[derive(Debug, Clone)] pub(crate) struct RequestFacts { /// The client looks like a real browser (step 1). @@ -225,6 +250,8 @@ pub(crate) struct RequestFacts { } /// When a provider is dispatched, relative to the navigation response. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Phase { /// Before the auction, within the hold budget. @@ -272,6 +299,8 @@ pub(crate) enum Eligibility { } /// What [`eligibility`] reads about one provider. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] pub(crate) struct EligibilityInput<'a> { pub(crate) provider: &'a dyn IdentityProvider, pub(crate) settings: &'a ProviderSettings, @@ -282,6 +311,8 @@ pub(crate) struct EligibilityInput<'a> { /// Whether the provider is eligible for this request, steps 1 to 7 of spec /// 4.7 in order, and which sources need which kind of call. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] #[must_use] pub(crate) fn eligibility( input: &EligibilityInput<'_>, @@ -445,6 +476,8 @@ fn has_unexpired_id(catalog: &ProviderCatalog, record: &KvPartnerId, now: UnixTi /// A signal the policy does not read or relaxes stays 0. Later parts append /// bits after these and keep these where they are. #[must_use] +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] pub(crate) fn fingerprint_bits( policy: &ConsentPolicy, consent: &ConsentContext, @@ -501,6 +534,8 @@ pub(crate) enum CallOutcome { /// A transport failure, timeout, 429 or 5xx. TransportFailure, /// The circuit breaker was open, so nothing was sent. + // Temporary: the breaker, the only producer, arrives in a later task. + #[allow(dead_code)] BreakerOpen, /// The burst cap stopped the call. BurstCapped, @@ -509,6 +544,8 @@ pub(crate) enum CallOutcome { } /// What an outcome needs besides the provider and the view. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] pub(crate) struct OutcomeContext<'a> { pub(crate) now: UnixTime, pub(crate) catalog: &'a ProviderCatalog, @@ -534,6 +571,8 @@ pub(crate) struct OutcomeChanges { } /// The stored changes of an outcome: the table of spec 8, row by row. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] #[must_use] pub(crate) fn outcome_to_changes( provider: &dyn IdentityProvider, @@ -930,12 +969,527 @@ fn state_update(provider: &IdentityProviderId, resolution: &Resolution) -> State } } +/// Everything a dispatch reads that is not per call. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] +pub(crate) struct IdentityDeps<'a> { + pub(crate) settings: &'a Settings, + pub(crate) catalog: &'a ProviderCatalog, + pub(crate) keys: &'a IdentityKeys, + /// Draws record revisions when a mutation applies. + // Temporary: first read when the mutations are applied. + #[allow(dead_code)] + pub(crate) random: &'a dyn RandomSource, + pub(crate) services: &'a RuntimeServices, + pub(crate) rate_limiter: Option<&'a dyn RateLimiter>, +} + +/// One provider call to dispatch: the provider and the kind each of its +/// sources needs. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub(crate) struct Call { + pub(crate) provider: IdentityProviderId, + pub(crate) kinds: BTreeMap, +} + +/// A call that was sent and is waiting for its answer. +struct InFlightCall<'a> { + provider: &'a dyn IdentityProvider, + input: ResolveInput<'a>, + call_context: Option, +} + +/// What preparing one call came to. +enum Prepared<'a> { + /// The call is settled without being sent. + Settled(CallOutcome), + /// The call was sent. + Sent(InFlightCall<'a>, PlatformPendingRequest), +} + +/// Whether the provider is a generic pull provider, the only kind given the +/// full EC ID. +fn is_generic_pull(provider: &dyn IdentityProvider) -> bool { + provider + .descriptor() + .resolves() + .is_some_and(Resolves::gets_ec_id) +} + +/// The order calls are dispatched in. Generic pull providers keep the +/// fairness rule of spec 10.4: sorted by source domain, then rotated left by +/// `(now / 3600) % len`, so the starting partner changes each hour when +/// `max_concurrency` limits a batch. Every other provider follows, by ID. +fn dispatch_order(calls: Vec, catalog: &ProviderCatalog, now: UnixTime) -> Vec { + let (mut generic, mut others): (Vec, Vec) = calls.into_iter().partition(|call| { + catalog + .get(&call.provider) + .is_some_and(|provider| is_generic_pull(provider.as_ref())) + }); + generic.sort_by(|left, right| { + let source = |call: &Call| { + catalog + .get(&call.provider) + .map(|provider| provider.descriptor().primary_source().clone()) + }; + source(left) + .cmp(&source(right)) + .then_with(|| left.provider.cmp(&right.provider)) + }); + if !generic.is_empty() { + let hour = usize::try_from(now.as_secs() / 3600).unwrap_or(0); + let offset = hour % generic.len(); + generic.rotate_left(offset); + } + others.sort_by(|left, right| left.provider.cmp(&right.provider)); + generic.extend(others); + generic +} + +/// Allows an `https` request to a declared host, with no `Cookie` header. +/// +/// # Errors +/// +/// [`IdentityProviderError::Config`] for any other request. +pub(crate) fn check_request( + request: &ProviderRequest, + descriptor: &ProviderDescriptor, +) -> Result<(), Report> { + if request.url().scheme() != "https" { + return Err(IdentityProviderError::config( + "a provider request must use https", + )); + } + let host = request.url().host_str().unwrap_or_default(); + let declared = descriptor.resolves().is_some_and(|resolves| { + resolves + .hosts() + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(host)) + }); + if !declared { + return Err(IdentityProviderError::config( + "a provider request must go to a declared host", + )); + } + if request.has_cookie_header() { + return Err(IdentityProviderError::config( + "a provider request must not set a Cookie header", + )); + } + Ok(()) +} + +/// Decodes a response body to at most `cap` bytes: identity, `gzip` or `br`. +/// +/// `None` when the decoded body is over `cap`, does not decode, or the +/// encoding is unknown. The cap applies while decoding, so a small body +/// cannot expand past it. +pub(crate) fn decode_body(encoding: Option<&str>, body: Vec, cap: usize) -> Option> { + let compression = match encoding.map(str::trim) { + None | Some("") => Compression::None, + Some(name) if name.eq_ignore_ascii_case("identity") => Compression::None, + Some(name) if name.eq_ignore_ascii_case("gzip") => Compression::Gzip, + Some(name) if name.eq_ignore_ascii_case("br") => Compression::Brotli, + Some(_) => return None, + }; + if compression == Compression::None { + return (body.len() <= cap).then_some(body); + } + let mut decoder = BodyStreamDecoder::new(compression, cap); + let mut decoded = decoder + .decode_chunk(bytes::Bytes::from(body)) + .ok()? + .to_vec(); + decoded.extend(decoder.finish().ok()?); + (decoded.len() <= cap).then_some(decoded) +} + +/// Whether the response declares a body over `cap`, or an unreadable length. +pub(crate) fn content_length_exceeds_limit( + response: &PlatformResponse, + cap: usize, + label: &str, +) -> bool { + let Some(value) = response.response.headers().get(header::CONTENT_LENGTH) else { + return false; + }; + let Some(length) = value + .to_str() + .ok() + .and_then(|value| value.parse::().ok()) + else { + log::warn!("'{label}' returned an unreadable Content-Length header, rejecting"); + return true; + }; + if length > cap { + log::warn!("'{label}' returned an oversized Content-Length ({length} bytes), rejecting"); + return true; + } + false +} + +/// Buffers a response body of at most `cap` bytes, as sent (still encoded). +/// `None` when the declared or actual length is over `cap`. +pub(crate) fn buffered_body( + response: PlatformResponse, + cap: usize, + label: &str, +) -> Option<(http::response::Parts, bytes::Bytes)> { + if content_length_exceeds_limit(&response, cap, label) { + return None; + } + let (parts, body) = response.response.into_parts(); + let body = body.into_bytes().unwrap_or_default(); + if body.len() > cap { + log::warn!( + "'{label}' returned an oversized response ({} bytes), rejecting", + body.len() + ); + return None; + } + Some((parts, body)) +} + +/// What waiting for one in-flight request came to. +pub(crate) enum Waited { + /// The response arrived. + Response(PlatformResponse), + /// The request failed or timed out. + Failed, + /// The caller stopped waiting, so the request was dropped. + Dropped, +} + +/// Waits for each in-flight request in turn and empties `in_flight`. All were +/// sent up front, so waiting in turn does not change concurrency. +/// +/// `keep_waiting` is asked before each wait. Once it returns `false`, every +/// request still pending is dropped without waiting. +pub(crate) fn drain_in_flight( + in_flight: &mut Vec<(T, PlatformPendingRequest)>, + services: &RuntimeServices, + mut keep_waiting: impl FnMut() -> bool, +) -> Vec<(T, Waited)> { + let mut waiting = true; + in_flight + .drain(..) + .map(|(label, pending)| { + waiting = waiting && keep_waiting(); + if !waiting { + return (label, Waited::Dropped); + } + match futures::executor::block_on(services.http_client().wait(pending)) { + Ok(response) => (label, Waited::Response(response)), + Err(error) => { + log::warn!("Request failed for '{label}': {error:?}"); + (label, Waited::Failed) + } + } + }) + .collect() +} + +/// Logs a provider `Config` failure. Only the error's own context is logged, +/// never its attachments. +fn log_config(provider: &IdentityProviderId, error: &Report) { + log::error!( + "Identity provider '{provider}' configuration error: {}", + error.current_context() + ); +} + +/// The outcome a `request` error becomes. +fn request_error_outcome( + provider: &IdentityProviderId, + error: &Report, +) -> CallOutcome { + if matches!( + error.current_context(), + IdentityProviderError::InputUnavailable { .. } + ) { + return CallOutcome::InputUnavailable; + } + log_config(provider, error); + CallOutcome::InputFailure +} + +/// The outcome a `response` error becomes. +fn response_error_outcome(error: &Report) -> CallOutcome { + match error.current_context() { + IdentityProviderError::Rejected { status } + if *status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error() => + { + CallOutcome::TransportFailure + } + IdentityProviderError::InvalidState => CallOutcome::InvalidState(None), + _ => CallOutcome::InputFailure, + } +} + +/// The first-byte timeout of a call, or `None` when a hold dispatch has no +/// budget left. +fn call_timeout( + deps: &IdentityDeps<'_>, + settings: &ProviderSettings, + remaining: Option, +) -> Option { + let policy = deps.services.backend().naming_policy(); + let millis = |value: u64| u32::try_from(value).unwrap_or(u32::MAX); + let timeout_ms = match remaining { + None => { + let configured = millis(settings.after_send_timeout_ms); + policy.canonicalize_transport_timeout_ms(configured, configured) + } + Some(remaining) => { + let configured = millis( + settings + .timeout_ms + .unwrap_or(deps.settings.ec.identity.hold_budget_ms), + ); + let remaining_ms = u32::try_from(remaining.as_millis()).unwrap_or(u32::MAX); + policy.canonicalize_transport_timeout_ms(remaining_ms, configured) + } + }; + (timeout_ms > 0).then(|| Duration::from_millis(u64::from(timeout_ms))) +} + +/// Whether the burst cap stops the call. `Err` when it could not be read. +fn burst_capped( + deps: &IdentityDeps<'_>, + provider: &IdentityProviderId, + ec_id: Option<&str>, + settings: &ProviderSettings, +) -> Result { + let Some(limiter) = deps.rate_limiter else { + return Ok(false); + }; + let Some(ec_id) = ec_id else { + log::warn!("Identity provider '{provider}' not dispatched: the request has no EC ID"); + return Err(()); + }; + let key = format!("identity:{provider}:{}", visitor_key(deps.keys, ec_id)); + limiter + .exceeded_per_minute(&key, settings.max_calls_per_minute) + .map_err(|error| { + log::warn!( + "Identity provider '{provider}' not dispatched, burst cap unreadable: {error:?}" + ); + }) +} + +/// Builds, checks and sends one call. +fn prepare_call<'a>( + deps: &IdentityDeps<'a>, + call: &'a Call, + signals: &'a ResolveSignals<'a>, + started: Instant, + budget: Option, +) -> Prepared<'a> { + let Some(provider) = deps.catalog.get(&call.provider) else { + log::error!( + "Identity provider '{}' is not in the catalog, not dispatched", + call.provider + ); + return Prepared::Settled(CallOutcome::InputFailure); + }; + let provider: &dyn IdentityProvider = provider.as_ref(); + let descriptor = provider.descriptor(); + let id = descriptor.id(); + let settings = deps.settings.ec.identity.provider(id); + + let input = ResolveInput::new(descriptor, call.kinds.clone(), signals); + let request = provider.request(&input); + if !input.recorded_errors().is_empty() { + log::error!("Identity provider '{id}' read an input it did not declare"); + return Prepared::Settled(CallOutcome::InputFailure); + } + let request = match request { + Ok(Some(request)) => request, + Ok(None) => return Prepared::Settled(CallOutcome::Skipped), + Err(error) => return Prepared::Settled(request_error_outcome(id, &error)), + }; + if let Err(error) = check_request(&request, descriptor) { + log_config(id, &error); + return Prepared::Settled(CallOutcome::InputFailure); + } + + let remaining = budget.map(|budget| budget.saturating_sub(started.elapsed())); + let Some(timeout) = call_timeout(deps, &settings, remaining) else { + return Prepared::Settled(CallOutcome::HoldCutOff); + }; + match burst_capped(deps, id, signals.ec_id.as_deref(), &settings) { + Ok(true) => return Prepared::Settled(CallOutcome::BurstCapped), + Ok(false) => {} + Err(()) => return Prepared::Settled(CallOutcome::NotDispatched), + } + + let url = request.url(); + let backend = deps.services.backend().ensure(&PlatformBackendSpec { + scheme: url.scheme().to_owned(), + host: url.host_str().unwrap_or_default().to_owned(), + port: url.port(), + host_header_override: None, + certificate_check: deps.settings.proxy.certificate_check, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(id.to_string()), + }); + let backend = match backend { + Ok(name) => name, + Err(error) => { + log::warn!("Identity provider '{id}': failed to resolve the backend: {error:?}"); + return Prepared::Settled(CallOutcome::TransportFailure); + } + }; + + let mut builder = http::Request::builder() + .method(request.method().clone()) + .uri(url.as_str()); + for (name, value) in request.headers() { + builder = builder.header(name, value); + } + let body = if request.body().is_empty() { + EdgeBody::empty() + } else { + EdgeBody::from(request.body().to_vec()) + }; + let Ok(outbound) = builder.body(body) else { + log::error!("Identity provider '{id}': the request could not be built"); + return Prepared::Settled(CallOutcome::InputFailure); + }; + let call_context = request.call_context_json().map(str::to_owned); + + match futures::executor::block_on( + deps.services + .http_client() + .send_async(PlatformHttpRequest::new(outbound, backend)), + ) { + Ok(pending) => Prepared::Sent( + InFlightCall { + provider, + input, + call_context, + }, + pending, + ), + Err(error) => { + log::warn!("Identity provider '{id}': failed to send: {error:?}"); + Prepared::Settled(CallOutcome::TransportFailure) + } + } +} + +/// The outcome of a call whose response arrived. +fn finish_call( + call: &InFlightCall<'_>, + response: PlatformResponse, + max_response_bytes: usize, +) -> CallOutcome { + let id = call.provider.descriptor().id(); + let Some((parts, raw)) = buffered_body(response, max_response_bytes, id.as_str()) else { + return CallOutcome::TransportFailure; + }; + let encoding = parts + .headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()); + let Some(body) = decode_body(encoding, raw.to_vec(), max_response_bytes) else { + log::warn!("Identity provider '{id}' returned a body that is over the cap or undecodable"); + return CallOutcome::TransportFailure; + }; + let response = + ProviderResponse::new(parts.status, parts.headers, body, call.call_context.clone()); + let result = call.provider.response(&response, &call.input); + if !call.input.recorded_errors().is_empty() { + log::error!("Identity provider '{id}' read an input it did not declare"); + return CallOutcome::InputFailure; + } + match result { + Ok(resolution) if call.input.state_decode_failed() => { + CallOutcome::InvalidState(Some(resolution)) + } + Ok(resolution) => CallOutcome::Resolved(resolution), + Err(error) => { + if matches!( + error.current_context(), + IdentityProviderError::Config { .. } + ) { + log_config(id, &error); + } + response_error_outcome(&error) + } + } +} + +/// Dispatches `calls` and returns each provider's outcome (spec 4.7). +/// +/// At most `max_concurrency` calls are in flight. `budget` is `Some(the +/// remaining hold budget)` for a hold dispatch before the auction, and +/// `None` after the response is sent: there, a call that misses its timeout is +/// an ordinary transport failure. +// Temporary: the request wiring, the first non-test caller, arrives in a later task. +#[allow(dead_code)] +pub(crate) fn dispatch( + deps: &IdentityDeps<'_>, + calls: Vec, + signals: &ResolveSignals<'_>, + budget: Option, +) -> Vec<(IdentityProviderId, CallOutcome)> { + let started = Instant::now(); + let identity = &deps.settings.ec.identity; + let max_concurrency = identity.max_concurrency.max(1); + let calls = dispatch_order(calls, deps.catalog, signals.now); + let mut outcomes = Vec::with_capacity(calls.len()); + let mut in_flight: Vec<(IdentityProviderId, PlatformPendingRequest)> = Vec::new(); + let mut sent: BTreeMap> = BTreeMap::new(); + + let drain = |in_flight: &mut Vec<(IdentityProviderId, PlatformPendingRequest)>, + sent: &mut BTreeMap>, + outcomes: &mut Vec<(IdentityProviderId, CallOutcome)>| { + let within_budget = || budget.is_none_or(|budget| started.elapsed() < budget); + for (id, waited) in drain_in_flight(in_flight, deps.services, within_budget) { + let Some(call) = sent.remove(&id) else { + continue; + }; + let outcome = match waited { + Waited::Response(response) => { + finish_call(&call, response, identity.max_response_bytes) + } + Waited::Failed if budget.is_some() => CallOutcome::HoldCutOff, + Waited::Failed => CallOutcome::TransportFailure, + Waited::Dropped => CallOutcome::HoldCutOff, + }; + outcomes.push((id, outcome)); + } + }; + + for call in &calls { + match prepare_call(deps, call, signals, started, budget) { + Prepared::Settled(outcome) => outcomes.push((call.provider.clone(), outcome)), + Prepared::Sent(sent_call, pending) => { + sent.insert(call.provider.clone(), sent_call); + in_flight.push((call.provider.clone(), pending)); + if in_flight.len() >= max_concurrency { + drain(&mut in_flight, &mut sent, &mut outcomes); + } + } + } + } + drain(&mut in_flight, &mut sent, &mut outcomes); + outcomes +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; + use std::io::Write as _; + use serde_json::json; use super::*; @@ -944,8 +1498,12 @@ mod tests { use crate::ec::identity::descriptor::{ ConsentPolicy, Input, ProviderDescriptor, ResolveKind, Resolves, UsOptOut, }; + use crate::ec::identity::input::{ProviderConsent, SeededRandom, ThreadRandom}; use crate::ec::identity::resolution::{ResolvedEid, ResolvedUid}; use crate::ec::identity::revision::{record_revision, uid_hash}; + use crate::ec::identity::test_support::fake_http::{ + FakeBackend, FakeHttpClient, ScriptedProvider, fake_services, + }; use crate::ec::identity::test_support::{ OutcomeFixture, allowing_consent, facts_for_writer, fixed_capture_provider, fixed_provider, pid, provider_from, record_for_test, source, stateful_fixed_provider, @@ -953,6 +1511,7 @@ mod tests { }; use crate::ec::kv_types::{KvEid, KvUid}; use crate::settings::ProviderMode; + use crate::test_support::tests::create_test_settings; /// The six lines every eligibility test repeats: one provider, its /// catalog and the settings and consent the call reads. @@ -2973,4 +3532,484 @@ mod tests { "should drop the duplicate" ); } + + // Dispatch + + struct RecordingLimiter { + exceeded: bool, + fail: bool, + keys: std::sync::Mutex>, + } + + impl RecordingLimiter { + fn new(exceeded: bool, fail: bool) -> Self { + Self { + exceeded, + fail, + keys: std::sync::Mutex::new(Vec::new()), + } + } + } + + impl RateLimiter for RecordingLimiter { + fn exceeded( + &self, + key: &str, + _hourly_limit: u32, + ) -> Result> { + self.keys + .lock() + .expect("should lock the keys") + .push(key.to_owned()); + if self.fail { + return Err(Report::new(crate::error::TrustedServerError::Proxy { + message: "limiter unavailable".to_owned(), + })); + } + Ok(self.exceeded) + } + } + + /// Providers, a fake client and everything `dispatch` reads. + struct DispatchFixture { + settings: Settings, + catalog: ProviderCatalog, + keys: IdentityKeys, + client: Arc, + backend: Arc, + services: RuntimeServices, + } + + impl DispatchFixture { + fn new(providers: Vec>) -> Self { + let settings = create_test_settings(); + let client = Arc::new(FakeHttpClient::new()); + let backend = Arc::new(FakeBackend::new()); + let services = fake_services(&client, &backend); + Self { + keys: IdentityKeys::from_settings(&settings), + catalog: ProviderCatalog::new(providers, BTreeMap::new()), + settings, + client, + backend, + services, + } + } + + fn run( + &self, + ids: &[&str], + now: u64, + budget: Option, + limiter: Option<&dyn RateLimiter>, + ) -> Vec<(IdentityProviderId, CallOutcome)> { + let deps = IdentityDeps { + settings: &self.settings, + catalog: &self.catalog, + keys: &self.keys, + random: &SeededRandom::new(1), + services: &self.services, + rate_limiter: limiter, + }; + let calls = ids + .iter() + .map(|id| { + let provider = self + .catalog + .get(&pid(id)) + .expect("should hold the provider"); + Call { + provider: pid(id), + kinds: BTreeMap::from([( + provider.descriptor().primary_source().clone(), + ResolveKind::Initial, + )]), + } + }) + .collect(); + let signals = ResolveSignals { + client_ip: None, + user_agent: Some("ExampleBrowser/1.0"), + page_url: None, + region: None, + hashed_email: None, + consent: ProviderConsent::default(), + publisher_domain: "www.example.com", + now: UnixTime::from_secs(now), + random: &ThreadRandom, + ec_id: Some(format!("{}.ABC123", "a".repeat(64))), + stored: BTreeMap::new(), + state_json: None, + }; + dispatch(&deps, calls, &signals, budget) + } + } + + fn outcome_of<'a>( + outcomes: &'a [(IdentityProviderId, CallOutcome)], + id: &str, + ) -> &'a CallOutcome { + &outcomes + .iter() + .find(|(provider, _)| *provider == pid(id)) + .expect("should hold an outcome for the provider") + .1 + } + + fn scripted(id: &str, host: &str, source: &str) -> Arc { + ScriptedProvider::new(id, host, source, false).into_provider() + } + + #[test] + fn check_request_enforces_https_host_and_no_cookie() { + let descriptor = fixed_provider("p", &["a.example.com"]).descriptor().clone(); + let good = ProviderRequest::get( + &"https://api.example.com/x" + .parse() + .expect("should parse the URL"), + ); + check_request(&good, &descriptor) + .expect("should allow an https request to a declared host (control)"); + for (name, request) in [ + ( + "http", + ProviderRequest::get( + &"http://api.example.com/x" + .parse() + .expect("should parse the URL"), + ), + ), + ( + "host", + ProviderRequest::get( + &"https://other.example.net/x" + .parse() + .expect("should parse the URL"), + ), + ), + ( + "cookie", + good.clone() + .try_with_header("cookie", "a=1") + .expect("should add the header"), + ), + ] { + assert!( + check_request(&request, &descriptor).is_err(), + "should reject the {name} case" + ); + } + } + + #[test] + fn decode_body_caps_decoded_size() { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(&vec![b'a'; 10_000]) + .expect("should write to the encoder"); + let gzipped = encoder.finish().expect("should finish the gzip stream"); + assert_eq!( + decode_body(Some("gzip"), gzipped.clone(), 10_000).map(|body| body.len()), + Some(10_000), + "should decode a body that fits" + ); + assert!( + decode_body(Some("gzip"), gzipped, 9_999).is_none(), + "should reject a body over the cap after decoding" + ); + assert!( + decode_body(Some("zstd"), vec![1], 10).is_none(), + "should reject an unknown encoding" + ); + assert_eq!( + decode_body(None, b"ok".to_vec(), 10).as_deref(), + Some(&b"ok"[..]), + "should pass an identity body through" + ); + assert!( + decode_body(None, vec![0; 11], 10).is_none(), + "should reject an identity body over the cap" + ); + } + + #[test] + fn dispatch_maps_status_codes_to_outcomes() { + let mut fixture = DispatchFixture::new(vec![ + scripted("ok", "ok.example.com", "ok.example.com"), + scripted("busy", "busy.example.com", "busy.example.com"), + scripted("bad", "bad.example.com", "bad.example.com"), + scripted("big", "big.example.com", "big.example.com"), + scripted("long", "long.example.com", "long.example.com"), + ]); + fixture.settings.ec.identity.max_response_bytes = 8; + fixture.client.respond("ok.example.com", 200, b"{}"); + fixture.client.respond("busy.example.com", 503, b"{}"); + fixture.client.respond("bad.example.com", 400, b"{}"); + fixture + .client + .respond("big.example.com", 200, b"0123456789"); + fixture.client.respond_with_headers( + "long.example.com", + 200, + b"{}", + &[("content-length", "9999")], + ); + + let outcomes = fixture.run(&["ok", "busy", "bad", "big", "long"], 1_000, None, None); + + assert!( + matches!(outcome_of(&outcomes, "ok"), CallOutcome::Resolved(_)), + "should resolve a 200" + ); + assert!( + matches!(outcome_of(&outcomes, "busy"), CallOutcome::TransportFailure), + "should count a 503 as a transport failure" + ); + assert!( + matches!(outcome_of(&outcomes, "bad"), CallOutcome::InputFailure), + "should count a 400 as an input failure" + ); + assert!( + matches!(outcome_of(&outcomes, "big"), CallOutcome::TransportFailure), + "should count a body over the cap as a transport failure" + ); + assert!( + matches!(outcome_of(&outcomes, "long"), CallOutcome::TransportFailure), + "should count a Content-Length over the cap as a transport failure" + ); + } + + #[test] + fn dispatch_rejects_a_body_that_decodes_over_the_cap() { + let mut fixture = + DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); + fixture.settings.ec.identity.max_response_bytes = 4_096; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(&vec![b'a'; 100_000]) + .expect("should write to the encoder"); + let gzipped = encoder.finish().expect("should finish the gzip stream"); + assert!( + gzipped.len() < 4_096, + "should compress under the cap (control)" + ); + fixture.client.respond_with_headers( + "p.example.com", + 200, + &gzipped, + &[("content-encoding", "gzip")], + ); + + let outcomes = fixture.run(&["p"], 1_000, None, None); + + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::TransportFailure), + "should count a decompression bomb as a transport failure" + ); + } + + #[test] + fn after_send_timeout_is_a_transport_failure_not_a_cut_off() { + let fixture = DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); + fixture.client.time_out("p.example.com"); + + let outcomes = fixture.run(&["p"], 1_000, None, None); + + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::TransportFailure), + "should count a missed after-send timeout as a transport failure" + ); + assert_eq!( + fixture.backend.first_byte_timeouts(), + vec![Duration::from_millis( + ProviderSettings::default_after_send_timeout_ms() + )], + "should wait the after-send timeout for the first byte" + ); + } + + #[test] + fn hold_dispatch_cuts_off_a_call_the_budget_ran_out_on() { + let fixture = DispatchFixture::new(vec![ + scripted("a", "a.example.com", "a.example.com"), + scripted("b", "b.example.com", "b.example.com"), + scripted("c", "c.example.com", "c.example.com"), + ]); + fixture.client.respond("a.example.com", 200, b"{}"); + fixture.client.respond("b.example.com", 200, b"{}"); + fixture.client.respond("c.example.com", 200, b"{}"); + fixture + .client + .delay("a.example.com", Duration::from_millis(60)); + fixture.client.time_out("c.example.com"); + + let outcomes = fixture.run( + &["a", "b", "c"], + 1_000, + Some(Duration::from_millis(20)), + None, + ); + + assert!( + matches!(outcome_of(&outcomes, "a"), CallOutcome::Resolved(_)), + "should keep a response that arrived in flight" + ); + assert!( + matches!(outcome_of(&outcomes, "b"), CallOutcome::HoldCutOff), + "should drop a call still pending when the budget ran out" + ); + assert!( + matches!(outcome_of(&outcomes, "c"), CallOutcome::HoldCutOff), + "should drop a later call too, and a hold timeout is a cut off" + ); + assert_eq!( + fixture.backend.first_byte_timeouts().len(), + 3, + "should have sent all three before the budget ran out" + ); + } + + #[test] + fn hold_dispatch_with_no_budget_sends_nothing() { + let fixture = DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); + fixture.client.respond("p.example.com", 200, b"{}"); + + let outcomes = fixture.run(&["p"], 1_000, Some(Duration::ZERO), None); + + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::HoldCutOff), + "should cut off a hold call with no budget left" + ); + assert!( + fixture.client.sent_hosts().is_empty(), + "should send nothing" + ); + } + + #[test] + fn hold_dispatch_waits_the_smaller_of_the_hold_timeout_and_the_budget() { + let mut fixture = + DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); + fixture.settings.ec.identity.hold_budget_ms = 80; + fixture.client.respond("p.example.com", 200, b"{}"); + + fixture.run(&["p"], 1_000, Some(Duration::from_secs(10)), None); + fixture.run(&["p"], 1_000, Some(Duration::from_millis(30)), None); + + let timeouts = fixture.backend.first_byte_timeouts(); + assert_eq!( + timeouts[0], + Duration::from_millis(80), + "should wait the hold timeout when the budget is larger" + ); + assert!( + timeouts[1] <= Duration::from_millis(30), + "should wait no longer than the remaining budget" + ); + } + + #[test] + fn burst_cap_skips_without_counting_a_failure() { + let fixture = DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); + fixture.client.respond("p.example.com", 200, b"{}"); + let capped = RecordingLimiter::new(true, false); + let broken = RecordingLimiter::new(false, true); + + let outcomes = fixture.run(&["p"], 1_000, None, Some(&capped)); + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::BurstCapped), + "should report a burst-capped call" + ); + assert!( + fixture.client.sent_hosts().is_empty(), + "should not send a burst-capped call" + ); + let keys = capped.keys.lock().expect("should lock the keys"); + assert!( + keys[0].starts_with("identity:p:") && !keys[0].contains(&"a".repeat(64)), + "should key the cap by provider and the keyed visitor hash, not the EC ID" + ); + drop(keys); + + let outcomes = fixture.run(&["p"], 1_000, None, Some(&broken)); + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::NotDispatched), + "should not dispatch when the limiter cannot be read" + ); + assert!( + fixture.client.sent_hosts().is_empty(), + "should not send when the limiter cannot be read" + ); + + let allowing = RecordingLimiter::new(false, false); + let outcomes = fixture.run(&["p"], 1_000, None, Some(&allowing)); + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::Resolved(_)), + "should dispatch under the cap (control)" + ); + } + + #[test] + fn undeclared_read_is_an_input_failure_even_when_ignored() { + let fixture = DispatchFixture::new(vec![ + ScriptedProvider::new("p", "p.example.com", "p.example.com", false) + .reading_undeclared_user_agent() + .into_provider(), + ]); + fixture.client.respond("p.example.com", 200, b"{}"); + + let outcomes = fixture.run(&["p"], 1_000, None, None); + + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::InputFailure), + "should fail a provider that read an undeclared input, though it ignored the error" + ); + assert!( + fixture.client.sent_hosts().is_empty(), + "should not send the call" + ); + } + + #[test] + fn generic_pull_dispatch_order_rotates_hourly() { + let generic = + |id: &str, host: &str| ScriptedProvider::new(id, host, host, true).into_provider(); + let mut fixture = DispatchFixture::new(vec![ + generic("gamma", "gamma.example.com"), + generic("alpha", "alpha.example.com"), + generic("beta", "beta.example.com"), + scripted("other", "other.example.com", "other.example.com"), + ]); + fixture.settings.ec.identity.max_concurrency = 1; + for host in ["alpha", "beta", "gamma", "other"] { + fixture + .client + .respond(&format!("{host}.example.com"), 200, b"{}"); + } + let ids = ["other", "gamma", "beta", "alpha"]; + + fixture.run(&ids, 100, None, None); + fixture.run(&ids, 3_600, None, None); + fixture.run(&ids, 7_200, None, None); + + let sent = fixture.client.sent_hosts(); + assert_eq!( + sent, + [ + "alpha.example.com", + "beta.example.com", + "gamma.example.com", + "other.example.com", + "beta.example.com", + "gamma.example.com", + "alpha.example.com", + "other.example.com", + "gamma.example.com", + "alpha.example.com", + "beta.example.com", + "other.example.com", + ], + "should sort generic pull by source, rotate the start each hour, and send other providers after" + ); + } } diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index 87b57812c7..25c3d760fc 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -380,8 +380,6 @@ impl Resolves { /// Hosts the provider may call. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn hosts(&self) -> &[String] { &self.hosts } @@ -398,8 +396,6 @@ impl Resolves { /// Every input declared for any kind, needed or wanted. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn declared_inputs(&self) -> BTreeSet { let mut all = self.needs.clone(); all.extend(self.wants.iter().copied()); @@ -411,8 +407,6 @@ impl Resolves { /// Inputs passed when available. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn wants(&self) -> &BTreeSet { &self.wants } @@ -431,8 +425,6 @@ impl Resolves { /// Whether the provider receives the full EC ID. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn gets_ec_id(&self) -> bool { self.ec_id } diff --git a/crates/trusted-server-core/src/ec/identity/exchange.rs b/crates/trusted-server-core/src/ec/identity/exchange.rs index 6c5d1650a2..87911bd704 100644 --- a/crates/trusted-server-core/src/ec/identity/exchange.rs +++ b/crates/trusted-server-core/src/ec/identity/exchange.rs @@ -194,8 +194,6 @@ impl ProviderRequest { /// Whether the provider set a `Cookie` header. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn has_cookie_header(&self) -> bool { self.headers.contains_key(COOKIE) } @@ -222,8 +220,6 @@ impl fmt::Debug for ProviderResponse { impl ProviderResponse { /// Built by the coordinator after buffering and decoding. - // Temporary: the dispatcher, the first non-test caller, arrives in a later task. - #[allow(dead_code)] #[must_use] pub(crate) fn new( status: StatusCode, diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs index 4ca12bae43..9865fab331 100644 --- a/crates/trusted-server-core/src/ec/identity/input.rs +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -410,8 +410,6 @@ impl<'a> ResolveInput<'a> { /// declared, so reading it is not an error of the provider, but it is /// unavailable (spec 4.4). #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn new( descriptor: &'a ProviderDescriptor, kinds: BTreeMap, @@ -462,16 +460,12 @@ impl<'a> ResolveInput<'a> { /// Undeclared reads recorded so far. The coordinator logs them as /// `Config`, and the harness fails the test. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn recorded_errors(&self) -> Vec { self.undeclared.borrow().clone() } /// Whether a `state()` call failed to decode. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn state_decode_failed(&self) -> bool { *self.state_failed.borrow() } diff --git a/crates/trusted-server-core/src/ec/identity/revision.rs b/crates/trusted-server-core/src/ec/identity/revision.rs index c0ea5ef6e6..cae88de048 100644 --- a/crates/trusted-server-core/src/ec/identity/revision.rs +++ b/crates/trusted-server-core/src/ec/identity/revision.rs @@ -148,8 +148,6 @@ pub(crate) fn push_hashes(list: &mut Vec, hashes: Vec) { /// The per-visitor rate-limit key component, from the full EC ID. #[must_use] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) fn visitor_key(keys: &IdentityKeys, ec_id: &str) -> String { let mut mac = HmacSha256::new_from_slice(&keys.visitor).expect("should accept a 32-byte key"); mac.update(ec_id.as_bytes()); diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 31e74eab88..57f683520c 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -311,3 +311,317 @@ impl IdentityProvider for DescriptorOnly { &self.0 } } + +/// A fake HTTP client, backend and provider for dispatch tests. +#[cfg(test)] +pub(crate) mod fake_http { + use std::collections::BTreeMap; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use error_stack::Report; + + use crate::ec::identity::descriptor::{ConsentPolicy, ProviderDescriptor, Resolves}; + use crate::ec::identity::exchange::{ProviderRequest, ProviderResponse}; + use crate::ec::identity::input::ResolveInput; + use crate::ec::identity::provider::IdentityProvider; + use crate::ec::identity::resolution::{IdentityProviderError, Resolution}; + use crate::platform::test_support::{NoopConfigStore, NoopGeo, NoopSecretStore}; + use crate::platform::{ + BackendNamingPolicy, ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, + PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, + PlatformSelectResult, RuntimeServices, + }; + + /// What the fake does for one host. + #[derive(Clone)] + enum Route { + Respond { + status: u16, + headers: Vec<(String, String)>, + body: Vec, + }, + Fail, + } + + struct FakePending { + host: String, + route: Route, + delay: Duration, + } + + /// An HTTP client that answers per host, and records what it was sent. + pub(crate) struct FakeHttpClient { + routes: Mutex>, + delays: Mutex>, + sent: Mutex>, + } + + impl FakeHttpClient { + pub(crate) fn new() -> Self { + Self { + routes: Mutex::new(BTreeMap::new()), + delays: Mutex::new(BTreeMap::new()), + sent: Mutex::new(Vec::new()), + } + } + + /// Answers `host` with `status` and `body`. + pub(crate) fn respond(&self, host: &str, status: u16, body: &[u8]) { + self.respond_with_headers(host, status, body, &[]); + } + + /// Answers `host` with `status`, `headers` and `body`. + pub(crate) fn respond_with_headers( + &self, + host: &str, + status: u16, + body: &[u8], + headers: &[(&str, &str)], + ) { + let route = Route::Respond { + status, + headers: headers + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(), + body: body.to_vec(), + }; + self.routes + .lock() + .expect("should lock the routes") + .insert(host.to_owned(), route); + } + + /// Makes the wait for `host` fail, as a first-byte timeout does. + pub(crate) fn time_out(&self, host: &str) { + self.routes + .lock() + .expect("should lock the routes") + .insert(host.to_owned(), Route::Fail); + } + + /// Makes the wait for `host` take `delay` of wall-clock time. + pub(crate) fn delay(&self, host: &str, delay: Duration) { + self.delays + .lock() + .expect("should lock the delays") + .insert(host.to_owned(), delay); + } + + /// The hosts requests were sent to, in order. + pub(crate) fn sent_hosts(&self) -> Vec { + self.sent + .lock() + .expect("should lock the sent hosts") + .clone() + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for FakeHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let host = request.request.uri().host().unwrap_or_default().to_owned(); + self.sent + .lock() + .expect("should lock the sent hosts") + .push(host.clone()); + let route = self + .routes + .lock() + .expect("should lock the routes") + .get(&host) + .cloned() + .ok_or_else(|| Report::new(PlatformError::HttpClient))?; + let delay = self + .delays + .lock() + .expect("should lock the delays") + .get(&host) + .copied() + .unwrap_or_default(); + Ok(PlatformPendingRequest::new(FakePending { + host, + route, + delay, + })) + } + + async fn select( + &self, + mut pending_requests: Vec, + ) -> Result> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient)); + } + let first = pending_requests.remove(0); + let ready = self.wait(first).await; + Ok(PlatformSelectResult { + ready, + remaining: pending_requests, + failed_backend_name: None, + }) + } + + async fn wait( + &self, + pending: PlatformPendingRequest, + ) -> Result> { + let pending = pending + .downcast::() + .map_err(|_| Report::new(PlatformError::HttpClient))?; + if !pending.delay.is_zero() { + std::thread::sleep(pending.delay); + } + match pending.route { + Route::Fail => Err(Report::new(PlatformError::HttpClient)), + Route::Respond { + status, + headers, + body, + } => { + let mut builder = edgezero_core::http::response_builder().status(status); + for (name, value) in headers { + builder = builder.header(name, value); + } + let response = builder + .body(edgezero_core::body::Body::from(body)) + .map_err(|_| Report::new(PlatformError::HttpClient))?; + Ok(PlatformResponse::new(response).with_backend_name(pending.host)) + } + } + } + } + + /// A backend named after its host, that records each spec it is given. + pub(crate) struct FakeBackend { + specs: Mutex>, + } + + impl FakeBackend { + pub(crate) fn new() -> Self { + Self { + specs: Mutex::new(Vec::new()), + } + } + + /// The first-byte timeouts the backends were registered with, in order. + pub(crate) fn first_byte_timeouts(&self) -> Vec { + self.specs + .lock() + .expect("should lock the specs") + .iter() + .map(|spec| spec.first_byte_timeout) + .collect() + } + } + + impl PlatformBackend for FakeBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name( + &self, + spec: &PlatformBackendSpec, + ) -> Result> { + Ok(spec.host.clone()) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.specs + .lock() + .expect("should lock the specs") + .push(spec.clone()); + Ok(spec.host.clone()) + } + } + + /// Services that send through `client` and register backends in `backend`. + pub(crate) fn fake_services( + client: &Arc, + backend: &Arc, + ) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(backend.clone()) + .http_client(client.clone()) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build() + } + + /// A resolver that calls its endpoint and answers `no_match` to a 2xx. + pub(crate) struct ScriptedProvider { + descriptor: ProviderDescriptor, + reads_user_agent: bool, + } + + impl ScriptedProvider { + /// A provider `id` calling `host`, claiming `source`. With `generic` + /// it is a generic pull provider. + pub(crate) fn new(id: &str, host: &str, source: &str, generic: bool) -> Self { + let mut resolves = Resolves::new().with_endpoint(&format!("https://{host}/resolve")); + if generic { + resolves = resolves.with_ec_id(); + } + let descriptor = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_source(source.parse().expect("should parse the source")) + .with_resolves(resolves) + .with_consent_policy(ConsentPolicy::eid_persistence_only()) + .build() + .expect("should build the descriptor"); + Self { + descriptor, + reads_user_agent: false, + } + } + + /// Reads `user_agent()` without declaring it, and ignores the error. + pub(crate) fn reading_undeclared_user_agent(mut self) -> Self { + self.reads_user_agent = true; + self + } + + pub(crate) fn into_provider(self) -> Arc { + Arc::new(self) + } + } + + impl IdentityProvider for ScriptedProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + if self.reads_user_agent { + let _ = input.user_agent().ok(); + } + Ok(Some(ProviderRequest::get(input.endpoint()?))) + } + + fn response( + &self, + response: &ProviderResponse, + _input: &ResolveInput<'_>, + ) -> Result> { + response.success_text().map(|_| Resolution::no_match()) + } + } +} diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index a72accc9ea..f304c5b530 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -9,7 +9,7 @@ //! the entry no longer stores per-partner sync timestamps. use edgezero_core::body::Body as EdgeBody; -use http::{Method, StatusCode, header}; +use http::{Method, StatusCode}; use serde::Deserialize; use url::Url; @@ -21,6 +21,7 @@ use crate::platform::{ use crate::settings::Settings; use super::generation::{ec_hash, is_valid_ec_id}; +use super::identity::coordinator::{Waited, buffered_body, drain_in_flight}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; use super::rate_limiter::RateLimiter; @@ -46,11 +47,6 @@ impl PullSyncContext { } } -struct InFlightPull { - source_domain: String, - pending: PlatformPendingRequest, -} - #[derive(Debug, Deserialize)] struct PullSyncResponse { uid: Option, @@ -182,7 +178,7 @@ pub fn dispatch_pull_sync( } let max_concurrency = settings.ec.identity.max_concurrency.max(1); - let mut in_flight: Vec = Vec::new(); + let mut in_flight: Vec<(String, PlatformPendingRequest)> = Vec::new(); let mut updates = Vec::new(); for partner in pull_partners { @@ -271,10 +267,7 @@ pub fn dispatch_pull_sync( } }; - in_flight.push(InFlightPull { - source_domain: partner.source_domain.clone(), - pending, - }); + in_flight.push((partner.source_domain.clone(), pending)); if in_flight.len() >= max_concurrency { drain_pull_batch(&mut in_flight, services, &mut updates); @@ -375,25 +368,16 @@ fn pull_rate_limit_key(source_domain: &str, ec_id: &str) -> String { } fn drain_pull_batch( - in_flight: &mut Vec, + in_flight: &mut Vec<(String, PlatformPendingRequest)>, services: &RuntimeServices, updates: &mut Vec, ) { - for pending in in_flight.drain(..) { - let source_domain = pending.source_domain; - // All requests were dispatched up front via send_async, so waiting on - // each in turn does not change concurrency. - let response = - match futures::executor::block_on(services.http_client().wait(pending.pending)) { - Ok(response) => response, - Err(err) => { - log::warn!( - "Pull sync: request failed for partner '{}': {err:?}", - source_domain - ); - continue; - } - }; + // All requests were dispatched up front via send_async, so waiting on + // each in turn does not change concurrency. + for (source_domain, waited) in drain_in_flight(in_flight, services, || true) { + let Waited::Response(response) = waited else { + continue; + }; let Some(uid) = extract_pull_uid(response, &source_domain) else { continue; @@ -409,39 +393,6 @@ fn drain_pull_batch( /// This prevents a misbehaving partner from exhausting WASM memory. const MAX_PULL_RESPONSE_BYTES: usize = 64 * 1024; -fn response_content_length_exceeds_limit(response: &PlatformResponse, source_domain: &str) -> bool { - let Some(value) = response.response.headers().get(header::CONTENT_LENGTH) else { - return false; - }; - - let Some(value) = value.to_str().ok() else { - log::warn!( - "Pull sync: partner '{}' returned invalid Content-Length header, rejecting", - source_domain - ); - return true; - }; - - let Ok(length) = value.parse::() else { - log::warn!( - "Pull sync: partner '{}' returned malformed Content-Length header, rejecting", - source_domain - ); - return true; - }; - - if length > MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized Content-Length ({} bytes), rejecting", - source_domain, - length - ); - return true; - } - - false -} - fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option { let status = response.response.status(); @@ -462,23 +413,7 @@ fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized response ({} bytes), rejecting", - source_domain, - body.len() - ); - return None; - } + let (_, body) = buffered_body(response, MAX_PULL_RESPONSE_BYTES, source_domain)?; let payload = match serde_json::from_slice::(&body) { Ok(payload) => payload, Err(err) => { @@ -522,6 +457,7 @@ mod tests { use crate::platform::PlatformResponse; use crate::redacted::Redacted; use crate::test_support::tests::purpose_one_only_consent; + use http::header; fn make_response(status: u16, body: &[u8]) -> PlatformResponse { PlatformResponse::new( From 0c6f2aa34c056214ca8a0852d504f4564a359680 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 21:27:15 +0530 Subject: [PATCH 30/89] Give each dispatched call only its own provider state Provider private state moves off the shared signals onto Call and the call's ResolveInput. A failed hold-mode wait is a cut off only when the hold budget was spent, and otherwise a transport failure. Signed-off-by: dhruv8sh --- .../src/ec/identity/coordinator.rs | 140 ++++++++++++++++-- .../src/ec/identity/input.rs | 27 ++-- .../src/ec/identity/test_support.rs | 44 +++++- .../trusted-server-core/src/ec/pull_sync.rs | 3 +- 4 files changed, 190 insertions(+), 24 deletions(-) diff --git a/crates/trusted-server-core/src/ec/identity/coordinator.rs b/crates/trusted-server-core/src/ec/identity/coordinator.rs index b0a5382582..65497036fd 100644 --- a/crates/trusted-server-core/src/ec/identity/coordinator.rs +++ b/crates/trusted-server-core/src/ec/identity/coordinator.rs @@ -992,6 +992,9 @@ pub(crate) struct IdentityDeps<'a> { pub(crate) struct Call { pub(crate) provider: IdentityProviderId, pub(crate) kinds: BTreeMap, + /// This provider's stored private state `ps.st`, as JSON. Only this + /// call's `ResolveInput` sees it. + pub(crate) state_json: Option, } /// A call that was sent and is waiting for its answer. @@ -999,6 +1002,13 @@ struct InFlightCall<'a> { provider: &'a dyn IdentityProvider, input: ResolveInput<'a>, call_context: Option, + /// When the request was sent. + sent_at: Instant, + /// How long the backend waits for the first byte. + timeout: Duration, + /// The remaining hold budget, not the provider's `timeout_ms`, set the + /// timeout. + budget_limited: bool, } /// What preparing one call came to. @@ -1006,7 +1016,7 @@ enum Prepared<'a> { /// The call is settled without being sent. Settled(CallOutcome), /// The call was sent. - Sent(InFlightCall<'a>, PlatformPendingRequest), + Sent(Box>, PlatformPendingRequest), } /// Whether the provider is a generic pull provider, the only kind given the @@ -1235,9 +1245,10 @@ fn call_timeout( deps: &IdentityDeps<'_>, settings: &ProviderSettings, remaining: Option, -) -> Option { +) -> Option<(Duration, bool)> { let policy = deps.services.backend().naming_policy(); let millis = |value: u64| u32::try_from(value).unwrap_or(u32::MAX); + let mut budget_limited = false; let timeout_ms = match remaining { None => { let configured = millis(settings.after_send_timeout_ms); @@ -1250,10 +1261,11 @@ fn call_timeout( .unwrap_or(deps.settings.ec.identity.hold_budget_ms), ); let remaining_ms = u32::try_from(remaining.as_millis()).unwrap_or(u32::MAX); + budget_limited = remaining_ms < configured; policy.canonicalize_transport_timeout_ms(remaining_ms, configured) } }; - (timeout_ms > 0).then(|| Duration::from_millis(u64::from(timeout_ms))) + (timeout_ms > 0).then(|| (Duration::from_millis(u64::from(timeout_ms)), budget_limited)) } /// Whether the burst cap stops the call. `Err` when it could not be read. @@ -1267,7 +1279,7 @@ fn burst_capped( return Ok(false); }; let Some(ec_id) = ec_id else { - log::warn!("Identity provider '{provider}' not dispatched: the request has no EC ID"); + log::debug!("Identity provider '{provider}' not dispatched: the request has no EC ID"); return Err(()); }; let key = format!("identity:{provider}:{}", visitor_key(deps.keys, ec_id)); @@ -1300,7 +1312,8 @@ fn prepare_call<'a>( let id = descriptor.id(); let settings = deps.settings.ec.identity.provider(id); - let input = ResolveInput::new(descriptor, call.kinds.clone(), signals); + let input = ResolveInput::new(descriptor, call.kinds.clone(), signals) + .with_state_json(call.state_json.clone()); let request = provider.request(&input); if !input.recorded_errors().is_empty() { log::error!("Identity provider '{id}' read an input it did not declare"); @@ -1317,7 +1330,7 @@ fn prepare_call<'a>( } let remaining = budget.map(|budget| budget.saturating_sub(started.elapsed())); - let Some(timeout) = call_timeout(deps, &settings, remaining) else { + let Some((timeout, budget_limited)) = call_timeout(deps, &settings, remaining) else { return Prepared::Settled(CallOutcome::HoldCutOff); }; match burst_capped(deps, id, signals.ec_id.as_deref(), &settings) { @@ -1368,11 +1381,14 @@ fn prepare_call<'a>( .send_async(PlatformHttpRequest::new(outbound, backend)), ) { Ok(pending) => Prepared::Sent( - InFlightCall { + Box::new(InFlightCall { provider, input, call_context, - }, + sent_at: Instant::now(), + timeout, + budget_limited, + }), pending, ), Err(error) => { @@ -1444,10 +1460,10 @@ pub(crate) fn dispatch( let calls = dispatch_order(calls, deps.catalog, signals.now); let mut outcomes = Vec::with_capacity(calls.len()); let mut in_flight: Vec<(IdentityProviderId, PlatformPendingRequest)> = Vec::new(); - let mut sent: BTreeMap> = BTreeMap::new(); + let mut sent: BTreeMap>> = BTreeMap::new(); let drain = |in_flight: &mut Vec<(IdentityProviderId, PlatformPendingRequest)>, - sent: &mut BTreeMap>, + sent: &mut BTreeMap>>, outcomes: &mut Vec<(IdentityProviderId, CallOutcome)>| { let within_budget = || budget.is_none_or(|budget| started.elapsed() < budget); for (id, waited) in drain_in_flight(in_flight, deps.services, within_budget) { @@ -1458,7 +1474,17 @@ pub(crate) fn dispatch( Waited::Response(response) => { finish_call(&call, response, identity.max_response_bytes) } - Waited::Failed if budget.is_some() => CallOutcome::HoldCutOff, + // A failed wait is a cut off only when the hold budget was + // spent: the whole budget, or a timeout the remaining budget + // set. Any other failure counts against the provider. + Waited::Failed + if budget.is_some_and(|budget| started.elapsed() >= budget) + || (budget.is_some() + && call.budget_limited + && call.sent_at.elapsed() >= call.timeout) => + { + CallOutcome::HoldCutOff + } Waited::Failed => CallOutcome::TransportFailure, Waited::Dropped => CallOutcome::HoldCutOff, }; @@ -3602,6 +3628,17 @@ mod tests { now: u64, budget: Option, limiter: Option<&dyn RateLimiter>, + ) -> Vec<(IdentityProviderId, CallOutcome)> { + self.run_with_state(ids, &[], now, budget, limiter) + } + + fn run_with_state( + &self, + ids: &[&str], + states: &[(&str, &str)], + now: u64, + budget: Option, + limiter: Option<&dyn RateLimiter>, ) -> Vec<(IdentityProviderId, CallOutcome)> { let deps = IdentityDeps { settings: &self.settings, @@ -3619,6 +3656,10 @@ mod tests { .get(&pid(id)) .expect("should hold the provider"); Call { + state_json: states + .iter() + .find(|(owner, _)| owner == id) + .map(|(_, state)| (*state).to_owned()), provider: pid(id), kinds: BTreeMap::from([( provider.descriptor().primary_source().clone(), @@ -3639,7 +3680,6 @@ mod tests { random: &ThreadRandom, ec_id: Some(format!("{}.ABC123", "a".repeat(64))), stored: BTreeMap::new(), - state_json: None, }; dispatch(&deps, calls, &signals, budget) } @@ -3860,7 +3900,7 @@ mod tests { ); assert!( matches!(outcome_of(&outcomes, "c"), CallOutcome::HoldCutOff), - "should drop a later call too, and a hold timeout is a cut off" + "should drop a call the budget ran out before waiting on" ); assert_eq!( fixture.backend.first_byte_timeouts().len(), @@ -3869,6 +3909,80 @@ mod tests { ); } + #[test] + fn hold_failure_before_the_budget_is_spent_is_a_transport_failure() { + let mut fixture = + DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); + fixture.settings.ec.identity.providers.insert( + "p".to_owned(), + ProviderSettings { + timeout_ms: Some(10), + ..ProviderSettings::default() + }, + ); + fixture.client.time_out("p.example.com"); + + let outcomes = fixture.run(&["p"], 1_000, Some(Duration::from_secs(60)), None); + + assert!( + matches!(outcome_of(&outcomes, "p"), CallOutcome::TransportFailure), + "should count a failure under timeout_ms as a transport failure, not a cut off" + ); + } + + #[test] + fn each_call_reads_only_its_own_providers_state() { + let fixture = DispatchFixture::new(vec![ + ScriptedProvider::new("one", "one.example.com", "one.example.com", false) + .stateful() + .into_provider(), + ScriptedProvider::new("two", "two.example.com", "two.example.com", false) + .stateful() + .into_provider(), + ScriptedProvider::new("three", "three.example.com", "three.example.com", false) + .stateful() + .into_provider(), + ]); + for host in ["one", "two", "three"] { + fixture + .client + .respond(&format!("{host}.example.com"), 200, b"{}"); + } + + fixture.run_with_state( + &["one", "two", "three"], + &[ + ("one", r#"{"token":"first-secret"}"#), + ("two", r#"{"token":"second-secret"}"#), + ], + 1_000, + None, + None, + ); + + let uris = fixture.client.sent_uris(); + let uri_for = |host: &str| { + uris.iter() + .find(|uri| uri.contains(host)) + .expect("should have sent to the host") + .clone() + }; + let first = uri_for("one.example.com"); + assert!( + first.contains("first-secret") && !first.contains("second-secret"), + "should give the first provider only its own state" + ); + let second = uri_for("two.example.com"); + assert!( + second.contains("second-secret") && !second.contains("first-secret"), + "should give the second provider only its own state" + ); + assert!( + uri_for("three.example.com").contains("state=none"), + "should give a provider with no stored state none" + ); + } + #[test] fn hold_dispatch_with_no_budget_sends_nothing() { let fixture = DispatchFixture::new(vec![scripted("p", "p.example.com", "p.example.com")]); diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs index 9865fab331..bfe58221ce 100644 --- a/crates/trusted-server-core/src/ec/identity/input.rs +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -379,7 +379,6 @@ pub(crate) struct ResolveSignals<'a> { #[allow(dead_code)] pub(crate) ec_id: Option, pub(crate) stored: BTreeMap, - pub(crate) state_json: Option, } /// What one provider may read for one call. @@ -389,6 +388,7 @@ pub struct ResolveInput<'a> { signals: &'a ResolveSignals<'a>, declared: BTreeSet, available: BTreeSet, + state_json: Option, undeclared: RefCell>, state_failed: RefCell, } @@ -431,11 +431,21 @@ impl<'a> ResolveInput<'a> { signals, declared, available, + state_json: None, undeclared: RefCell::new(Vec::new()), state_failed: RefCell::new(false), } } + /// Gives the call its own provider's stored private state `ps.st`, as + /// JSON. State is per provider and sensitive, so it never rides on the + /// signals every call shares. + #[must_use] + pub(crate) fn with_state_json(mut self, state_json: Option) -> Self { + self.state_json = state_json; + self + } + fn record(&self, access: Access) -> Report { self.undeclared.borrow_mut().push(access); Report::new(IdentityProviderError::UndeclaredInput { access }) @@ -514,7 +524,7 @@ impl<'a> ResolveInput<'a> { { return Err(self.record(Access::PrivateState)); } - let Some(raw) = self.signals.state_json.as_deref() else { + let Some(raw) = self.state_json.as_deref() else { return Ok(None); }; serde_json::from_str(raw).map(Some).map_err(|_| { @@ -751,7 +761,6 @@ mod tests { random: &ThreadRandom, ec_id: Some("a".repeat(64) + ".ABC123"), stored: BTreeMap::new(), - state_json: None, } } @@ -876,9 +885,9 @@ mod tests { .with_endpoint("https://a.example.com") .with_private_state(), ); - let mut good = signals(); - good.state_json = Some(r#"{"signature":"s"}"#.to_owned()); - let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &good); + let good = signals(); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &good) + .with_state_json(Some(r#"{"signature":"s"}"#.to_owned())); let state: serde_json::Value = input .state() .expect("should decode the state") @@ -889,9 +898,9 @@ mod tests { "should not record a good state as a failure" ); - let mut bad = signals(); - bad.state_json = Some(r#""a bare string""#.to_owned()); - let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &bad); + let bad = signals(); + let input = ResolveInput::new(&descriptor, kinds(&descriptor, ResolveKind::Refresh), &bad) + .with_state_json(Some(r#""a bare string""#.to_owned())); let error = input .state::() .expect_err("should not decode a bare string"); diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 57f683520c..a47f1cff36 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -355,6 +355,7 @@ pub(crate) mod fake_http { routes: Mutex>, delays: Mutex>, sent: Mutex>, + uris: Mutex>, } impl FakeHttpClient { @@ -363,6 +364,7 @@ pub(crate) mod fake_http { routes: Mutex::new(BTreeMap::new()), delays: Mutex::new(BTreeMap::new()), sent: Mutex::new(Vec::new()), + uris: Mutex::new(Vec::new()), } } @@ -409,6 +411,11 @@ pub(crate) mod fake_http { .insert(host.to_owned(), delay); } + /// The URIs requests were sent to, in order. + pub(crate) fn sent_uris(&self) -> Vec { + self.uris.lock().expect("should lock the sent URIs").clone() + } + /// The hosts requests were sent to, in order. pub(crate) fn sent_hosts(&self) -> Vec { self.sent @@ -436,6 +443,10 @@ pub(crate) mod fake_http { .lock() .expect("should lock the sent hosts") .push(host.clone()); + self.uris + .lock() + .expect("should lock the sent URIs") + .push(request.request.uri().to_string()); let route = self .routes .lock() @@ -567,6 +578,7 @@ pub(crate) mod fake_http { pub(crate) struct ScriptedProvider { descriptor: ProviderDescriptor, reads_user_agent: bool, + stateful: bool, } impl ScriptedProvider { @@ -587,6 +599,7 @@ pub(crate) mod fake_http { Self { descriptor, reads_user_agent: false, + stateful: false, } } @@ -596,6 +609,27 @@ pub(crate) mod fake_http { self } + /// Declares private state, and sends what it read in a `state` query. + pub(crate) fn stateful(mut self) -> Self { + self.stateful = true; + self.descriptor = ProviderDescriptor::builder(self.descriptor.id().clone()) + .with_source(self.descriptor.primary_source().clone()) + .with_resolves( + Resolves::new() + .with_endpoint( + self.descriptor + .endpoint() + .expect("should have an endpoint") + .as_str(), + ) + .with_private_state(), + ) + .with_consent_policy(ConsentPolicy::eid_persistence_only()) + .build() + .expect("should build the descriptor"); + self + } + pub(crate) fn into_provider(self) -> Arc { Arc::new(self) } @@ -613,7 +647,15 @@ pub(crate) mod fake_http { if self.reads_user_agent { let _ = input.user_agent().ok(); } - Ok(Some(ProviderRequest::get(input.endpoint()?))) + let mut request = ProviderRequest::get(input.endpoint()?); + if self.stateful { + let state = input.state::()?; + request = request.with_query( + "state", + &state.map_or_else(|| "none".to_owned(), |state| state.to_string()), + ); + } + Ok(Some(request)) } fn response( diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index f304c5b530..6f714a9452 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -413,7 +413,8 @@ fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option(&body) { Ok(payload) => payload, Err(err) => { From 9cc246d4ac646a5bcbaf5d2e87da092eea663e41 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 21:45:51 +0530 Subject: [PATCH 31/89] Model pull sync partners as GenericPullProvider Adds the provider harness basics and a KV store that counts writes by mode. Generic pull results stay one UID per source, a 404 or null UID stays a no-op, and one catalog validator rejects provider tables it cannot apply. pull_sync_ttl_sec and pull_sync_rate_limit are removed from [[ec.partners]]: pacing comes from each pull partner's [ec.identity.providers.] table, and a config that sets either key fails validation with an error naming its replacement. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 4 +- crates/trusted-server-core/src/config.rs | 63 ++ crates/trusted-server-core/src/ec/admin.rs | 4 +- crates/trusted-server-core/src/ec/auth.rs | 4 +- .../trusted-server-core/src/ec/batch_sync.rs | 4 +- crates/trusted-server-core/src/ec/eids.rs | 4 +- crates/trusted-server-core/src/ec/finalize.rs | 4 +- crates/trusted-server-core/src/ec/identify.rs | 4 +- .../src/ec/identity/catalog.rs | 257 +++++++- .../src/ec/identity/descriptor.rs | 2 - .../src/ec/identity/generic_pull.rs | 565 ++++++++++++++++++ .../src/ec/identity/input.rs | 4 - .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/test_support.rs | 504 +++++++++++++++- crates/trusted-server-core/src/ec/kv.rs | 25 + .../trusted-server-core/src/ec/kv_backend.rs | 201 ++++--- .../trusted-server-core/src/ec/prebid_eids.rs | 4 +- .../trusted-server-core/src/ec/pull_sync.rs | 83 +-- .../src/ec/pull_sync_marker.rs | 4 +- crates/trusted-server-core/src/ec/registry.rs | 62 +- crates/trusted-server-core/src/publisher.rs | 4 +- crates/trusted-server-core/src/settings.rs | 47 +- 22 files changed, 1601 insertions(+), 253 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/generic_pull.rs diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index f415815de7..13c5846438 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -643,8 +643,8 @@ mod tests { pull_sync_enabled: false, pull_sync_url: None, pull_sync_allowed_domains: vec![], - pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: None, } } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index abb81971b4..d4c9a67ece 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -268,6 +268,11 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Report { + Report::new(TrustedServerError::Configuration { message }) +} + +/// Validates the catalog rules of spec 10.2 against every provider +/// descriptor. +/// +/// It rejects a duplicate provider ID, more than 32 providers, an +/// `[ec.identity.providers.]` table that names no enabled provider, and +/// `revoke_scope` on a provider whose descriptor declares no `captures`. It +/// is the one place these rules live. It reads descriptors only, so it runs +/// on unresolved deploy-time settings. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming the first rule +/// that fails. +pub(crate) fn validate_identity_catalog( + descriptors: &[ProviderDescriptor], + partners: &[EcPartner], + identity: &IdentitySettings, +) -> Result<(), Report> { + let mut ids = BTreeSet::new(); + for descriptor in descriptors { + if !ids.insert(descriptor.id().as_str()) { + return Err(catalog_error(format!( + "identity provider `{}` is declared more than once", + descriptor.id() + ))); + } + } + if descriptors.len() > MAX_PROVIDERS { + return Err(catalog_error(format!( + "at most {MAX_PROVIDERS} identity providers may be enabled, got {}", + descriptors.len() + ))); + } + + for (id, settings) in &identity.providers { + let Some(descriptor) = descriptors + .iter() + .find(|descriptor| descriptor.id().as_str() == id) + else { + let configured = if ids.is_empty() { + "none".to_owned() + } else { + ids.iter().copied().collect::>().join(", ") + }; + let hint = partners + .iter() + .find(|partner| { + !partner.pull_sync_enabled + && generic_pull_id(&partner.source_domain).as_str() == id + }) + .map_or_else(String::new, |partner| { + format!( + "; partner `{}` has pull_sync_enabled = false", + partner.source_domain + ) + }); + return Err(catalog_error(format!( + "ec.identity.providers.{id} names no enabled identity provider{hint}; \ + configured providers: {configured}" + ))); + }; + if settings.revoke_scope.is_some() && descriptor.captures().is_none() { + return Err(catalog_error(format!( + "ec.identity.providers.{id}.revoke_scope is set, but provider `{id}` declares no captures" + ))); + } + } + Ok(()) +} /// Enabled providers, their narrowed sources and source owners. Built once /// per settings load. @@ -16,6 +99,45 @@ pub(crate) struct ProviderCatalog { } impl ProviderCatalog { + /// Builds the catalog from settings, without resolving any secret. + /// + /// It reads `settings.ec.partners` directly and never builds a + /// `PartnerRegistry`, so a deploy-time `api_token` that holds a secret + /// store key name is fine. In Part 1 it holds generic pull providers + /// only, with no source owners. It validates their descriptors with + /// [`validate_identity_catalog`] before building. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when a pull partner + /// cannot be described or the catalog rules fail. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn from_settings(settings: &Settings) -> Result> { + let providers = settings + .ec + .partners + .iter() + .filter(|partner| partner.pull_sync_enabled) + .map(|partner| { + GenericPullProvider::from_partner(partner) + .map(|provider| Arc::new(provider) as Arc) + .change_context(TrustedServerError::Configuration { + message: format!( + "ec.partners: pull sync of '{}' cannot be described as an identity provider", + partner.source_domain + ), + }) + }) + .collect::, _>>()?; + let descriptors: Vec<_> = providers + .iter() + .map(|provider| provider.descriptor().clone()) + .collect(); + validate_identity_catalog(&descriptors, &settings.ec.partners, &settings.ec.identity)?; + Ok(Self::new(providers, BTreeMap::new())) + } + /// A catalog with no providers. #[must_use] pub(crate) fn empty() -> Self { @@ -125,3 +247,136 @@ impl ProviderCatalog { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::descriptor::Captures; + use crate::ec::identity::generic_pull::generic_pull_descriptors; + use crate::ec::identity::test_support::{ec_partner, fixed_capture_provider}; + use crate::settings::{ProviderSettings, RevokeScope}; + use crate::test_support::tests::create_test_settings; + + const TOKEN: &str = "example-token-value-0123456789abcdef"; + + fn partner() -> EcPartner { + ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull", + TOKEN, + ) + } + + fn identity_with(id: &str, settings: ProviderSettings) -> IdentitySettings { + let mut identity = IdentitySettings::default(); + identity.providers.insert(id.to_owned(), settings); + identity + } + + fn validate(partners: &[EcPartner], identity: &IdentitySettings) -> Result<(), String> { + let descriptors = generic_pull_descriptors(partners).expect("should describe"); + validate_identity_catalog(&descriptors, partners, identity).map_err(|e| format!("{e:?}")) + } + + #[test] + fn validate_identity_catalog_rejects_unknown_provider_tables() { + let partners = [partner()]; + + let error = validate( + &partners, + &identity_with("pull-other-example-com", ProviderSettings::default()), + ) + .expect_err("should reject a table naming no provider"); + assert!( + error.contains("pull-other-example-com") && error.contains("pull-ssp-example-com"), + "should name the table and list the configured IDs, got: {error}" + ); + + validate( + &partners, + &identity_with("pull-ssp-example-com", ProviderSettings::default()), + ) + .expect("control: a table for the real pull ID should pass"); + } + + #[test] + fn validate_identity_catalog_rejects_revoke_scope_without_captures() { + let partners = [partner()]; + let with_scope = ProviderSettings { + revoke_scope: Some(RevokeScope::Provider), + ..ProviderSettings::default() + }; + + validate( + &partners, + &identity_with("pull-ssp-example-com", with_scope.clone()), + ) + .expect_err("should reject revoke_scope on a provider without captures"); + validate( + &partners, + &identity_with("pull-ssp-example-com", ProviderSettings::default()), + ) + .expect("control: the same table without revoke_scope should pass"); + + let capturing = fixed_capture_provider("capturer", &["capture.example.com"]); + assert!( + capturing + .descriptor() + .captures() + .is_some_and(|c| *c != Captures::new()), + "should hold a capturing descriptor for the control" + ); + validate_identity_catalog( + &[capturing.descriptor().clone()], + &[], + &identity_with("capturer", with_scope), + ) + .expect("control: revoke_scope on a capturing provider should pass"); + } + + #[test] + fn validate_identity_catalog_rejects_duplicate_ids_and_more_than_32_providers() { + let one = partner(); + let descriptors = + generic_pull_descriptors(std::slice::from_ref(&one)).expect("should describe"); + let duplicated = [descriptors[0].clone(), descriptors[0].clone()]; + let _ = validate_identity_catalog(&duplicated, &[one], &IdentitySettings::default()) + .expect_err("should reject a duplicate ID"); + + let many = |count: usize| -> Vec { + (0..count) + .map(|n| { + let source = format!("ssp{n}.example.com"); + ec_partner(&source, &format!("https://sync.{source}/pull"), TOKEN) + }) + .collect() + }; + validate(&many(32), &IdentitySettings::default()) + .expect("control: 32 providers should pass"); + validate(&many(33), &IdentitySettings::default()).expect_err("should reject 33 providers"); + } + + #[test] + fn from_settings_builds_from_unresolved_settings() { + let mut settings = create_test_settings(); + let mut unresolved = partner(); + unresolved.api_token = Some(crate::redacted::Redacted::new("key-name".to_owned())); + settings.ec.partners = vec![unresolved]; + + assert!( + crate::ec::registry::PartnerRegistry::from_config(&settings.ec.partners).is_err(), + "should hold settings the registry rejects" + ); + let catalog = ProviderCatalog::from_settings(&settings).expect("should build"); + + let id = "pull-ssp-example-com".parse().expect("should parse the ID"); + assert!(catalog.is_configured(&id), "should hold the pull provider"); + assert_eq!(catalog.sources_of(&id).len(), 1, "should claim its source"); + assert!( + catalog + .owner_of(&"ssp.example.com".parse().expect("should parse")) + .is_none(), + "should have no owners in Part 1" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/descriptor.rs b/crates/trusted-server-core/src/ec/identity/descriptor.rs index 25c3d760fc..c49da6d47d 100644 --- a/crates/trusted-server-core/src/ec/identity/descriptor.rs +++ b/crates/trusted-server-core/src/ec/identity/descriptor.rs @@ -371,8 +371,6 @@ impl Resolves { /// Gives the provider the full EC ID. Only generic pull may use this. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(in crate::ec::identity) fn with_ec_id(mut self) -> Self { self.ec_id = true; self diff --git a/crates/trusted-server-core/src/ec/identity/generic_pull.rs b/crates/trusted-server-core/src/ec/identity/generic_pull.rs new file mode 100644 index 0000000000..9c34b05ba7 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/generic_pull.rs @@ -0,0 +1,565 @@ +//! Today's pull sync, modelled as an identity provider. +//! +//! A pull-enabled `[[ec.partners]]` entry becomes one [`GenericPullProvider`]: +//! a `GET` of the partner's `pull_sync_url` with the EC ID, answered with one +//! UID. The coordinator owns eligibility, pacing, consent and write-back. + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize; +use sha2::{Digest as _, Sha256}; +use url::Url; + +use super::descriptor::{ConsentPolicy, ProviderDescriptor, Resolves}; +use super::exchange::{ProviderRequest, ProviderResponse}; +use super::ids::{IdentityProviderId, SourceDomain}; +use super::input::ResolveInput; +use super::provider::IdentityProvider; +use super::resolution::{IdentityProviderError, Resolution, ResolvedEid, ResolvedUid}; +use crate::ec::partner::normalize_partner_source_domain; +use crate::ec::prebid_eids::is_valid_eid_uid; +use crate::ec::pull_sync::{build_pull_request_url, validated_pull_sync_url}; +use crate::error::TrustedServerError; +use crate::redacted::Redacted; +use crate::settings::EcPartner; + +/// Longest provider ID, from `^[a-z][a-z0-9_-]{0,62}$`. +const MAX_PROVIDER_ID_LENGTH: usize = 63; + +/// Readable characters kept when the ID is cut, leaving room for `-` and the +/// 8 hash characters. +const CUT_READABLE_LENGTH: usize = MAX_PROVIDER_ID_LENGTH - 9; + +/// Derives the provider ID of the pull partner with `source_domain`. +/// +/// It is `pull-` plus the normalized domain with `.` replaced by `-`. A +/// domain that already contains `-` also gets `-` and the first 8 hex digits +/// of the SHA-256 of the domain, so `a-b.example.com` and `a.b.example.com` +/// differ. An ID over 63 characters is cut to 54 readable characters, then +/// `-` and the hash. +#[must_use] +pub(crate) fn generic_pull_id(source_domain: &str) -> IdentityProviderId { + let domain = normalize_partner_source_domain(source_domain) + .unwrap_or_else(|_| source_domain.to_ascii_lowercase()); + let readable: String = format!("pull-{domain}") + .chars() + .map(|character| { + if character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-' { + character + } else { + '-' + } + }) + .collect(); + let digest = Sha256::digest(domain.as_bytes()); + let hash = hex::encode(&digest[..4]); + + let id = if domain.contains('-') { + format!("{readable}-{hash}") + } else { + readable.clone() + }; + let id = if id.len() > MAX_PROVIDER_ID_LENGTH { + format!("{}-{hash}", &readable[..CUT_READABLE_LENGTH]) + } else { + id + }; + id.parse() + .expect("should derive a valid provider ID from a normalized domain") +} + +/// The descriptors of every pull-enabled partner. +/// +/// Reads no secret, so it works on unresolved deploy-time settings. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when a pull-enabled partner +/// cannot be described, for example a `pull_sync_allowed_domains` entry that +/// is not a hostname. +pub(crate) fn generic_pull_descriptors( + partners: &[EcPartner], +) -> Result, Report> { + partners + .iter() + .filter(|partner| partner.pull_sync_enabled) + .map(|partner| { + describe(partner).change_context(TrustedServerError::Configuration { + message: format!( + "ec.partners: pull sync of '{}' cannot be described as an identity provider", + partner.source_domain + ), + }) + }) + .collect() +} + +fn describe(partner: &EcPartner) -> Result> { + let mut resolves = Resolves::new().with_ec_id(); + for host in &partner.pull_sync_allowed_domains { + resolves = resolves.with_host(&host.trim().trim_end_matches('.').to_ascii_lowercase()); + } + let source: SourceDomain = partner.source_domain.parse()?; + ProviderDescriptor::builder(generic_pull_id(&partner.source_domain)) + .with_source(source) + .with_resolves(resolves) + .with_consent_policy(ConsentPolicy::eid_persistence_only()) + .build() +} + +/// The answer a pull partner gives. +#[derive(Debug, Deserialize)] +struct PullAnswer { + uid: Option, +} + +/// One pull-enabled partner, called as an identity provider. +pub(crate) struct GenericPullProvider { + descriptor: ProviderDescriptor, + source: SourceDomain, + pull_sync_url: Option, + allowed_domains: Vec, + token: Option>, +} + +impl GenericPullProvider { + /// Builds the provider of `partner`. + /// + /// The descriptor reads only `source_domain`, `pull_sync_url` and + /// `pull_sync_allowed_domains`, which are not secret, so this works on + /// unresolved deploy-time settings. The token is kept for + /// [`request`](IdentityProvider::request) only. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the partner cannot be + /// described. + pub(crate) fn from_partner(partner: &EcPartner) -> Result> { + let descriptor = describe(partner)?; + let source = descriptor.primary_source().clone(); + Ok(Self { + descriptor, + source, + pull_sync_url: partner.pull_sync_url.clone(), + allowed_domains: partner.pull_sync_allowed_domains.clone(), + token: partner.ts_pull_token.clone(), + }) + } +} + +impl IdentityProvider for GenericPullProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let Some(mut url) = + validated_pull_sync_url(self.pull_sync_url.as_deref(), &self.allowed_domains) + else { + return Ok(None); + }; + let Some(token) = self + .token + .as_ref() + .filter(|token| !token.expose().trim().is_empty()) + else { + log::warn!( + "Pull sync: partner '{}' enabled but missing ts_pull_token", + self.source + ); + return Ok(None); + }; + + // The allowlist match ignores a trailing dot, and the descriptor's + // hosts carry none, so call the dotless spelling. + let host = url + .host_str() + .unwrap_or_default() + .trim_end_matches('.') + .to_owned(); + if url.host_str() != Some(host.as_str()) && url.set_host(Some(&host)).is_err() { + return Ok(None); + } + let url: Url = build_pull_request_url(url, input.ec_id()?); + Ok(Some(ProviderRequest::get(&url).with_bearer(token))) + } + + fn response( + &self, + response: &ProviderResponse, + _input: &ResolveInput<'_>, + ) -> Result> { + // A 404 is the partner's "no user" answer, as pull sync treats it. + if response.status() == http::StatusCode::NOT_FOUND { + return Ok(Resolution::no_match()); + } + let answer: PullAnswer = response.success_json()?; + match answer.uid { + Some(uid) if is_valid_eid_uid(&uid) => Ok(Resolution::found([ResolvedEid::new( + &self.source, + ResolvedUid::new(uid), + )])), + _ => Ok(Resolution::no_match()), + } + } +} + +#[cfg(test)] +mod tests { + use http::StatusCode; + use serde_json::json; + + use super::*; + use crate::ec::identity::test_support::{ProviderHarness, TEST_EC_ID, ec_partner}; + use crate::settings::{ProviderSettings, Settings}; + use crate::test_support::tests::create_test_settings; + + const TOKEN: &str = "example-token-value-0123456789abcdef"; + + fn partner() -> crate::settings::EcPartner { + ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull", + TOKEN, + ) + } + + #[test] + fn calls_the_partner_url_with_the_ec_id_and_bearer() { + let provider = GenericPullProvider::from_partner(&partner()).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + + let request = harness + .request() + .expect("should build") + .expect("should not skip"); + + assert_eq!( + request.url().host_str(), + Some("sync.ssp.example.com"), + "should call the partner host" + ); + assert_eq!( + request + .url() + .query_pairs() + .find(|(name, _)| name == "ec_id") + .map(|(_, value)| value.into_owned()), + Some(TEST_EC_ID.to_owned()), + "should pass the EC ID" + ); + assert_eq!( + request.headers()["authorization"], + format!("Bearer {TOKEN}"), + "should send the bearer token" + ); + assert_eq!(request.method(), http::Method::GET, "should use GET"); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "uid": "partner-uid" })) + .expect("should parse the answer"); + assert_eq!(resolution.eids().len(), 1, "should return one entry"); + let eid = &resolution.eids()[0]; + assert_eq!( + eid.source().as_str(), + "ssp.example.com", + "should name the source" + ); + assert_eq!(eid.uids().len(), 1, "should return one UID"); + assert_eq!( + eid.uids()[0].id(), + "partner-uid", + "should return the partner's UID" + ); + assert_eq!( + ( + eid.uids()[0].atype(), + eid.uids()[0].ext(), + eid.inserter(), + eid.matcher(), + eid.mm(), + eid.ext() + ), + (None, None, None, None, None, None), + "should set nothing beyond the UID, so it is stored as one entry with one UID" + ); + } + + #[test] + fn keeps_the_query_the_partner_url_already_has() { + let partner = ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull?x=1", + TOKEN, + ); + let provider = GenericPullProvider::from_partner(&partner).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + + let request = harness + .request() + .expect("should build") + .expect("should not skip"); + + let query = request.url().query().expect("should have a query"); + assert!(query.contains("x=1"), "should keep the existing query"); + assert!(query.contains("ec_id="), "should append the EC ID"); + assert!(!query.contains("ip="), "should not forward the client IP"); + } + + #[test] + fn not_found_and_null_uid_are_no_match_as_today() { + let provider = GenericPullProvider::from_partner(&partner()).expect("should build"); + let oversized = "x".repeat(513); + for (name, status, body) in [ + ("404", StatusCode::NOT_FOUND, json!({})), + ( + "404 with a body", + StatusCode::NOT_FOUND, + json!({ "uid": "u" }), + ), + ("null uid", StatusCode::OK, json!({ "uid": null })), + ("missing uid", StatusCode::OK, json!({})), + ("empty uid", StatusCode::OK, json!({ "uid": "" })), + ("blank uid", StatusCode::OK, json!({ "uid": " " })), + ("oversized uid", StatusCode::OK, json!({ "uid": oversized })), + ] { + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + harness + .request() + .expect("should build") + .expect("should not skip"); + let resolution = harness + .respond(status, &body) + .expect("should not be an error"); + assert!(resolution.is_no_match(), "should be no_match for {name}"); + } + } + + #[test] + fn a_uid_at_the_size_limit_is_kept() { + let provider = GenericPullProvider::from_partner(&partner()).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + harness + .request() + .expect("should build") + .expect("should not skip"); + + let resolution = harness + .respond(StatusCode::OK, &json!({ "uid": "x".repeat(512) })) + .expect("should parse"); + + assert_eq!( + resolution.eids()[0].uids()[0].id().len(), + 512, + "should keep a UID of MAX_UID_LENGTH" + ); + } + + #[test] + fn other_failures_are_errors_not_no_match() { + let provider = GenericPullProvider::from_partner(&partner()).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + harness + .request() + .expect("should build") + .expect("should not skip"); + + let rejected = harness + .respond(StatusCode::INTERNAL_SERVER_ERROR, &json!({})) + .expect_err("should reject a 500"); + assert!( + matches!( + rejected.current_context(), + IdentityProviderError::Rejected { status } if *status == StatusCode::INTERNAL_SERVER_ERROR + ), + "should report Rejected" + ); + let malformed = harness + .respond_raw(StatusCode::OK, &[], b"not json") + .expect_err("should reject a body that is not JSON"); + assert!( + matches!( + malformed.current_context(), + IdentityProviderError::Malformed + ), + "should report Malformed" + ); + } + + #[test] + fn skips_without_a_valid_url_or_token() { + let mut http_url = partner(); + http_url.pull_sync_url = Some("http://sync.ssp.example.com/pull".to_owned()); + let mut other_host = partner(); + other_host.pull_sync_url = Some("https://evil.example.net/pull".to_owned()); + let mut no_url = partner(); + no_url.pull_sync_url = None; + let mut no_token = partner(); + no_token.ts_pull_token = None; + + for (name, partner) in [ + ("http scheme", http_url), + ("host outside the allowlist", other_host), + ("no URL", no_url), + ("no token", no_token), + ] { + let provider = GenericPullProvider::from_partner(&partner).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + + let request = harness.request().expect("should not be an error"); + + assert!(request.is_none(), "should skip for {name}"); + } + } + + #[test] + fn accepts_an_allowlist_match_after_normalization() { + let mut partner = partner(); + partner.pull_sync_url = Some("https://SYNC.SSP.EXAMPLE.COM./pull".to_owned()); + partner.pull_sync_allowed_domains = vec!["Sync.Ssp.Example.Com.".to_owned()]; + let provider = GenericPullProvider::from_partner(&partner).expect("should build"); + let mut harness = ProviderHarness::new(&provider).with_ec_id(TEST_EC_ID); + + let request = harness.request().expect("should build"); + + assert!(request.is_some(), "should call a host the allowlist names"); + } + + #[test] + fn a_stored_record_makes_the_source_a_refresh_and_the_call_still_goes_out() { + let provider = GenericPullProvider::from_partner(&partner()).expect("should build"); + let source = provider.descriptor().primary_source().clone(); + let stored = crate::ec::identity::test_support::stored_for_test(&source, "stored-uid"); + let mut harness = ProviderHarness::new(&provider) + .with_ec_id(TEST_EC_ID) + .with_stored(stored); + + let request = harness.request().expect("should build"); + + assert!(request.is_some(), "should refresh a stored source"); + } + + #[test] + fn the_descriptor_reads_no_secret() { + let mut partner = partner(); + partner.ts_pull_token = Some(crate::redacted::Redacted::new("secret-key-name".to_owned())); + let with_key_name = GenericPullProvider::from_partner(&partner).expect("should build"); + let with_token = GenericPullProvider::from_partner(&self::partner()).expect("should build"); + + assert_eq!( + with_key_name.descriptor(), + with_token.descriptor(), + "should describe the same provider whatever the token holds" + ); + let descriptor = with_token.descriptor(); + assert_eq!( + descriptor.id().as_str(), + "pull-ssp-example-com", + "should derive the ID" + ); + assert_eq!(descriptor.sources().len(), 1, "should claim one source"); + assert_eq!( + descriptor.primary_source().as_str(), + "ssp.example.com", + "should claim the partner's source domain" + ); + let resolves = descriptor.resolves().expect("should resolve"); + assert_eq!( + resolves.hosts(), + ["sync.ssp.example.com".to_owned()], + "should allow only the allowlisted hosts" + ); + assert!(resolves.gets_ec_id(), "should get the EC ID"); + assert!(descriptor.captures().is_none(), "should not capture"); + assert!( + descriptor.consent_policy().is_eid_persistence_only(), + "should require the EID persistence consent pull sync requires today" + ); + } + + #[test] + fn descriptors_cover_only_pull_enabled_partners() { + let mut disabled = ec_partner("off.example.com", "https://sync.off.example.com/p", TOKEN); + disabled.pull_sync_enabled = false; + let partners = [partner(), disabled]; + + let descriptors = generic_pull_descriptors(&partners).expect("should build"); + + assert_eq!( + descriptors.len(), + 1, + "should skip a partner without pull sync" + ); + assert_eq!( + descriptors[0].id().as_str(), + "pull-ssp-example-com", + "should describe the pull-enabled partner" + ); + } + + #[test] + fn provider_ids_are_derived_and_bounded() { + assert_eq!( + generic_pull_id("ssp.example.com").as_str(), + "pull-ssp-example-com", + "should keep a dot-only domain readable" + ); + let long = format!("{}.example.com", "a".repeat(70)); + let id = generic_pull_id(&long); + assert!( + id.as_str().len() <= 63, + "should bound the ID to 63 characters" + ); + assert_ne!( + id, + generic_pull_id(&format!("{}.example.com", "a".repeat(71))), + "should keep long IDs unique by hash" + ); + assert_ne!( + generic_pull_id("a-b.example.com"), + generic_pull_id("a.b.example.com"), + "should not collide when a domain already contains a hyphen" + ); + assert_eq!( + generic_pull_id("SSP.Example.Com."), + generic_pull_id("ssp.example.com"), + "should derive the ID from the normalized domain" + ); + let hyphenated_long = format!("{}-b.example.com", "a".repeat(70)); + assert!( + generic_pull_id(&hyphenated_long).as_str().len() <= 63, + "should bound a long hyphenated domain too" + ); + } + + #[test] + fn generic_pull_pacing_comes_from_its_provider_table() { + let mut settings: Settings = create_test_settings(); + let id = generic_pull_id("ssp.example.com"); + + let defaults = settings.ec.identity.provider(&id); + assert_eq!( + defaults.refresh_after_sec, 86_400, + "should default the refresh" + ); + assert_eq!( + defaults.max_calls_per_minute, 1, + "should default the pacing" + ); + + settings.ec.identity.providers.insert( + id.as_str().to_owned(), + ProviderSettings { + max_calls_per_minute: 2, + ..ProviderSettings::default() + }, + ); + let configured = settings.ec.identity.provider(&id); + assert_eq!( + configured.max_calls_per_minute, 2, + "should read the pacing from the pull provider's table" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/input.rs b/crates/trusted-server-core/src/ec/identity/input.rs index bfe58221ce..b54ab7012b 100644 --- a/crates/trusted-server-core/src/ec/identity/input.rs +++ b/crates/trusted-server-core/src/ec/identity/input.rs @@ -375,8 +375,6 @@ pub(crate) struct ResolveSignals<'a> { pub(crate) publisher_domain: &'a str, pub(crate) now: UnixTime, pub(crate) random: &'a dyn RandomSource, - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) ec_id: Option, pub(crate) stored: BTreeMap, } @@ -622,8 +620,6 @@ impl<'a> ResolveInput<'a> { /// `UndeclaredInput { access: EcId }` otherwise. `Config` when the /// request has no EC ID, which the coordinator never allows: it does not /// dispatch generic pull without one. - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(in crate::ec::identity) fn ec_id(&self) -> Result<&str, Report> { if !self.descriptor.resolves().is_some_and(Resolves::gets_ec_id) { return Err(self.record(Access::EcId)); diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 525cf99ea9..7ccd217584 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod catalog; pub(crate) mod coordinator; pub mod descriptor; pub mod exchange; +pub(crate) mod generic_pull; pub mod ids; pub mod input; pub(crate) mod mirror; diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index a47f1cff36..785e28c346 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -8,10 +8,17 @@ ) )] -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use error_stack::Report; +use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use serde::Serialize; +use serde::de::DeserializeOwned; use serde_json::json; +use url::Url; use crate::consent::ConsentContext; use crate::consent::jurisdiction::Jurisdiction; @@ -19,15 +26,23 @@ use crate::consent::types::TcfConsent; use crate::ec::identity::catalog::ProviderCatalog; use crate::ec::identity::coordinator::{OutcomeContext, RequestFacts}; use crate::ec::identity::descriptor::{ - Captures, ConsentPolicy, Input, ProviderDescriptor, Resolves, + Captures, ConsentPolicy, Input, ProviderDescriptor, ResolveKind, Resolves, }; +use crate::ec::identity::exchange::{ProviderRequest, ProviderResponse}; use crate::ec::identity::ids::{IdentityProviderId, SourceDomain, UnixTime}; -use crate::ec::identity::input::SeededRandom; +use crate::ec::identity::input::{ + HashedEmail, ProviderConsent, Region, ResolveInput, ResolveSignals, SeededRandom, StoredId, +}; use crate::ec::identity::policy::PolicySignals; use crate::ec::identity::provider::IdentityProvider; +use crate::ec::identity::resolution::{IdentityProviderError, Resolution, When}; use crate::ec::identity::revision::{StateRevision, new_revision}; +use crate::ec::kv_backend::in_memory::InMemoryEcKv; +use crate::ec::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; -use crate::settings::{IdentitySettings, ProviderSettings}; +use crate::error::TrustedServerError; +use crate::redacted::Redacted; +use crate::settings::{EcPartner, IdentitySettings, ProviderSettings}; /// A request with no consent signal and no regulated jurisdiction. #[must_use] @@ -304,6 +319,444 @@ impl OutcomeFixture { } } +/// A valid EC ID in `{64-hex}.{6-alnum}` format. +pub const TEST_EC_ID: &str = + "0000000000000000000000000000000000000000000000000000000000000000.ABC123"; + +/// A partner with pull sync enabled, calling `pull_sync_url` with `token`. +/// +/// The URL's host is the only allowed domain. +/// +/// # Panics +/// +/// Panics when `source` is not a domain or `pull_sync_url` is not a URL with a host. +#[must_use] +pub fn ec_partner(source: &str, pull_sync_url: &str, token: &str) -> EcPartner { + let host = Url::parse(pull_sync_url) + .expect("should parse the pull URL") + .host_str() + .expect("should have a pull URL host") + .to_owned(); + EcPartner { + name: format!("Partner {source}"), + source_domain: source.to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: None, + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some(pull_sync_url.to_owned()), + pull_sync_allowed_domains: vec![host], + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, + ts_pull_token: Some(Redacted::new(token.to_owned())), + } +} + +/// A stored one-UID record of `source`, as a provider reads it. +/// +/// # Panics +/// +/// Panics when the record is not valid. +#[must_use] +pub(crate) fn stored_for_test(source: &SourceDomain, uid: &str) -> StoredId { + StoredId::from_record(source, &KvPartnerId::new(uid)) + .expect("should build a stored ID from a record with a UID") +} + +/// Write and lookup counts of a [`WriteCountingEcKv`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct WriteCounts { + /// Writes in [`EcKvWriteMode::Add`] mode. + pub add: usize, + /// Writes in [`EcKvWriteMode::IfGenerationMatch`] mode. + pub if_generation_match: usize, + /// Writes in [`EcKvWriteMode::Overwrite`] mode. + pub overwrite: usize, + /// Point reads. + pub lookups: usize, +} + +impl WriteCounts { + /// Every write, whatever its mode. + #[must_use] + pub fn total(&self) -> usize { + self.add + self.if_generation_match + self.overwrite + } +} + +/// An in-memory KV store that counts lookups and writes by mode. +/// +/// Every write is applied or refused as the in-memory store does, so a +/// write that loses its precondition is still counted and reports +/// [`EcKvWriteOutcome::PreconditionFailed`]. +pub struct WriteCountingEcKv { + inner: InMemoryEcKv, + add: AtomicUsize, + if_generation_match: AtomicUsize, + overwrite: AtomicUsize, + lookups: AtomicUsize, +} + +impl WriteCountingEcKv { + /// An empty store named `name`. + #[must_use] + pub fn new(name: impl Into) -> Self { + Self { + inner: InMemoryEcKv::new(name), + add: AtomicUsize::new(0), + if_generation_match: AtomicUsize::new(0), + overwrite: AtomicUsize::new(0), + lookups: AtomicUsize::new(0), + } + } + + /// The counts so far, seeding included. + #[must_use] + pub fn writes(&self) -> WriteCounts { + WriteCounts { + add: self.add.load(Ordering::SeqCst), + if_generation_match: self.if_generation_match.load(Ordering::SeqCst), + overwrite: self.overwrite.load(Ordering::SeqCst), + lookups: self.lookups.load(Ordering::SeqCst), + } + } + + /// Zeroes the counts, to count only what follows the seeding. + pub fn reset_counts(&self) { + for count in [ + &self.add, + &self.if_generation_match, + &self.overwrite, + &self.lookups, + ] { + count.store(0, Ordering::SeqCst); + } + } +} + +impl EcKvStore for WriteCountingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.lookups.fetch_add(1, Ordering::SeqCst); + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + let count = match write.mode { + EcKvWriteMode::Add => &self.add, + EcKvWriteMode::IfGenerationMatch(_) => &self.if_generation_match, + EcKvWriteMode::Overwrite => &self.overwrite, + }; + count.fetch_add(1, Ordering::SeqCst); + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } +} + +/// Drives one provider the way the coordinator does, so a vendor's tests +/// stay short. +/// +/// A source with a [`with_stored`](Self::with_stored) record is a `Refresh`, +/// any other an `Initial`. Sources whose kind needs an input the harness +/// does not hold are left out, and with none left +/// [`request`](Self::request) returns `Ok(None)` without calling the +/// provider. +pub struct ProviderHarness<'provider> { + provider: &'provider dyn IdentityProvider, + client_ip: Option, + user_agent: Option, + page_url: Option, + region: Option, + hashed_email: Option, + stored: BTreeMap, + state_json: Option, + now: UnixTime, + random: SeededRandom, + ec_id: Option, + last_request: Option, +} + +impl<'provider> ProviderHarness<'provider> { + /// A harness for `provider` with no inputs, the clock at 1,760,000,000 + /// and random seed 1. + #[must_use] + pub fn new(provider: &'provider dyn IdentityProvider) -> Self { + Self { + provider, + client_ip: None, + user_agent: None, + page_url: None, + region: None, + hashed_email: None, + stored: BTreeMap::new(), + state_json: None, + now: UnixTime::from_secs(1_760_000_000), + random: SeededRandom::new(1), + ec_id: None, + last_request: None, + } + } + + /// Sets the client IP. + /// + /// # Panics + /// + /// Panics when `ip` is not an IP address. + #[must_use] + pub fn with_client_ip(mut self, ip: &str) -> Self { + self.client_ip = Some(ip.parse().expect("should parse the client IP")); + self + } + + /// Sets the `User-Agent`. + #[must_use] + pub fn with_user_agent(mut self, user_agent: &str) -> Self { + self.user_agent = Some(user_agent.to_owned()); + self + } + + /// Sets the page URL. + /// + /// # Panics + /// + /// Panics when `url` is not a URL. + #[must_use] + pub fn with_page_url(mut self, url: &str) -> Self { + self.page_url = Some(Url::parse(url).expect("should parse the page URL")); + self + } + + /// Sets the region. + /// + /// # Panics + /// + /// Panics when `country` is not a country code. + #[must_use] + pub fn with_region(mut self, country: &str, subdivision: Option<&str>) -> Self { + self.region = + Some(Region::from_geo(country, subdivision).expect("should accept the country")); + self + } + + /// Sets the hashed email. + #[must_use] + pub fn with_hashed_email(mut self, email: HashedEmail) -> Self { + self.hashed_email = Some(email); + self + } + + /// Gives the provider a stored record, valid or expired. Its source is + /// the one its first entry names, or the provider's primary source. + #[must_use] + pub fn with_stored(mut self, stored: StoredId) -> Self { + let source = stored + .eids() + .first() + .and_then(|eid| eid.source.parse().ok()) + .unwrap_or_else(|| self.provider.descriptor().primary_source().clone()); + self.stored.insert(source, stored); + self + } + + /// Gives the provider its private state, as stored. + /// + /// # Panics + /// + /// Panics when `state` does not serialize. + #[must_use] + pub fn with_state(mut self, state: &impl Serialize) -> Self { + self.state_json = Some(serde_json::to_string(state).expect("should serialize the state")); + self + } + + /// Sets the fixed clock. + #[must_use] + pub fn with_now(mut self, now: UnixTime) -> Self { + self.now = now; + self + } + + /// Reseeds the random source. + #[must_use] + pub fn with_random_seed(mut self, seed: u64) -> Self { + self.random = SeededRandom::new(seed); + self + } + + /// Gives the provider the EC ID. Only generic pull reads it. + #[must_use] + pub(crate) fn with_ec_id(mut self, ec_id: &str) -> Self { + self.ec_id = Some(ec_id.to_owned()); + self + } + + /// Calls `request` and keeps the built request and its call context. + /// + /// # Errors + /// + /// Returns what the provider returns. + pub fn request(&mut self) -> Result, Report> { + let kinds = self.kinds(); + let request = if kinds.is_empty() { + None + } else { + let signals = self.signals(); + let input = ResolveInput::new(self.provider.descriptor(), kinds, &signals) + .with_state_json(self.state_json.clone()); + self.provider.request(&input)? + }; + self.last_request.clone_from(&request); + Ok(request) + } + + /// Decodes the last built request's JSON body. + /// + /// # Panics + /// + /// Panics when no request was built or its body is not a `T`. + #[must_use] + pub fn request_json(&self) -> T { + let request = self + .last_request + .as_ref() + .expect("should have built a request"); + serde_json::from_slice(request.body()).expect("should decode the request body") + } + + /// Calls `response` with the kept call context. + /// + /// # Errors + /// + /// Returns what the provider returns. + /// + /// # Panics + /// + /// Panics when `body` does not serialize. + pub fn respond( + &mut self, + status: StatusCode, + body: &serde_json::Value, + ) -> Result> { + let bytes = serde_json::to_vec(body).expect("should serialize the response body"); + self.respond_raw(status, &[], &bytes) + } + + /// Calls `response` with `headers` and a raw `body`, such as an empty 204. + /// + /// # Errors + /// + /// Returns what the provider returns. + /// + /// # Panics + /// + /// Panics when a header is not valid. + pub fn respond_raw( + &mut self, + status: StatusCode, + headers: &[(&str, &str)], + body: &[u8], + ) -> Result> { + let mut header_map = HeaderMap::new(); + for (name, value) in headers { + header_map.insert( + HeaderName::from_bytes(name.as_bytes()).expect("should be a header name"), + HeaderValue::from_str(value).expect("should be a header value"), + ); + } + let call_context = self + .last_request + .as_ref() + .and_then(|request| request.call_context_json().map(str::to_owned)); + let response = ProviderResponse::new(status, header_map, body.to_vec(), call_context); + let signals = self.signals(); + let input = ResolveInput::new(self.provider.descriptor(), self.kinds(), &signals) + .with_state_json(self.state_json.clone()); + self.provider.response(&response, &input) + } + + /// Resolves an expiry or refresh against the fixed clock. + #[must_use] + pub fn resolve(&self, when: When) -> UnixTime { + when.resolve(self.now) + } + + fn available_inputs(&self) -> BTreeSet { + let mut available = BTreeSet::new(); + for (input, present) in [ + (Input::ClientIp, self.client_ip.is_some()), + (Input::UserAgent, self.user_agent.is_some()), + (Input::PageUrl, self.page_url.is_some()), + (Input::Region, self.region.is_some()), + (Input::HashedEmail, self.hashed_email.is_some()), + ] { + if present { + available.insert(input); + } + } + available + } + + fn kinds(&self) -> BTreeMap { + let descriptor = self.provider.descriptor(); + let Some(resolves) = descriptor.resolves() else { + return BTreeMap::new(); + }; + let available = self.available_inputs(); + descriptor + .sources() + .iter() + .filter_map(|source| { + let kind = if self.stored.contains_key(source) { + ResolveKind::Refresh + } else { + ResolveKind::Initial + }; + (resolves.kinds().contains(&kind) && resolves.needs(kind).is_subset(&available)) + .then(|| (source.clone(), kind)) + }) + .collect() + } + + fn signals(&self) -> ResolveSignals<'_> { + ResolveSignals { + client_ip: self.client_ip, + user_agent: self.user_agent.as_deref(), + page_url: self.page_url.clone(), + region: self.region.clone(), + hashed_email: self.hashed_email.clone(), + consent: ProviderConsent::default(), + publisher_domain: "www.example.com", + now: self.now, + random: &self.random, + ec_id: self.ec_id.clone(), + stored: self.stored.clone(), + } + } +} + struct DescriptorOnly(ProviderDescriptor); impl IdentityProvider for DescriptorOnly { @@ -313,8 +766,13 @@ impl IdentityProvider for DescriptorOnly { } /// A fake HTTP client, backend and provider for dispatch tests. -#[cfg(test)] -pub(crate) mod fake_http { +#[cfg(any(test, feature = "test-utils"))] +#[allow( + clippy::missing_panics_doc, + clippy::new_without_default, + reason = "test doubles lock their own mutexes and are built with new" +)] +pub mod fake_http { use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -326,12 +784,15 @@ pub(crate) mod fake_http { use crate::ec::identity::input::ResolveInput; use crate::ec::identity::provider::IdentityProvider; use crate::ec::identity::resolution::{IdentityProviderError, Resolution}; + #[cfg(test)] use crate::platform::test_support::{NoopConfigStore, NoopGeo, NoopSecretStore}; use crate::platform::{ - BackendNamingPolicy, ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, + BackendNamingPolicy, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, RuntimeServices, + PlatformSelectResult, }; + #[cfg(test)] + use crate::platform::{ClientInfo, RuntimeServices}; /// What the fake does for one host. #[derive(Clone)] @@ -351,7 +812,7 @@ pub(crate) mod fake_http { } /// An HTTP client that answers per host, and records what it was sent. - pub(crate) struct FakeHttpClient { + pub struct FakeHttpClient { routes: Mutex>, delays: Mutex>, sent: Mutex>, @@ -359,7 +820,9 @@ pub(crate) mod fake_http { } impl FakeHttpClient { - pub(crate) fn new() -> Self { + /// A client with no routes. + #[must_use] + pub fn new() -> Self { Self { routes: Mutex::new(BTreeMap::new()), delays: Mutex::new(BTreeMap::new()), @@ -369,12 +832,12 @@ pub(crate) mod fake_http { } /// Answers `host` with `status` and `body`. - pub(crate) fn respond(&self, host: &str, status: u16, body: &[u8]) { + pub fn respond(&self, host: &str, status: u16, body: &[u8]) { self.respond_with_headers(host, status, body, &[]); } /// Answers `host` with `status`, `headers` and `body`. - pub(crate) fn respond_with_headers( + pub fn respond_with_headers( &self, host: &str, status: u16, @@ -396,7 +859,7 @@ pub(crate) mod fake_http { } /// Makes the wait for `host` fail, as a first-byte timeout does. - pub(crate) fn time_out(&self, host: &str) { + pub fn time_out(&self, host: &str) { self.routes .lock() .expect("should lock the routes") @@ -404,7 +867,7 @@ pub(crate) mod fake_http { } /// Makes the wait for `host` take `delay` of wall-clock time. - pub(crate) fn delay(&self, host: &str, delay: Duration) { + pub fn delay(&self, host: &str, delay: Duration) { self.delays .lock() .expect("should lock the delays") @@ -412,12 +875,12 @@ pub(crate) mod fake_http { } /// The URIs requests were sent to, in order. - pub(crate) fn sent_uris(&self) -> Vec { + pub fn sent_uris(&self) -> Vec { self.uris.lock().expect("should lock the sent URIs").clone() } /// The hosts requests were sent to, in order. - pub(crate) fn sent_hosts(&self) -> Vec { + pub fn sent_hosts(&self) -> Vec { self.sent .lock() .expect("should lock the sent hosts") @@ -515,19 +978,21 @@ pub(crate) mod fake_http { } /// A backend named after its host, that records each spec it is given. - pub(crate) struct FakeBackend { + pub struct FakeBackend { specs: Mutex>, } impl FakeBackend { - pub(crate) fn new() -> Self { + /// A backend that has registered nothing. + #[must_use] + pub fn new() -> Self { Self { specs: Mutex::new(Vec::new()), } } /// The first-byte timeouts the backends were registered with, in order. - pub(crate) fn first_byte_timeouts(&self) -> Vec { + pub fn first_byte_timeouts(&self) -> Vec { self.specs .lock() .expect("should lock the specs") @@ -559,6 +1024,7 @@ pub(crate) mod fake_http { } /// Services that send through `client` and register backends in `backend`. + #[cfg(test)] pub(crate) fn fake_services( client: &Arc, backend: &Arc, diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 274efafee9..d5cb672d21 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -1612,6 +1612,31 @@ impl KvIdentityGraph { } } +#[cfg(any(test, feature = "test-utils"))] +impl KvIdentityGraph { + /// Test helper: a graph backed by an in-memory store that counts lookups + /// and writes by mode, with the store to read the counts from. + /// + /// Counts include any seeding, so call + /// [`WriteCountingEcKv::reset_counts`](crate::ec::identity::test_support::WriteCountingEcKv::reset_counts) + /// after seeding to count only what the code under test does. + #[must_use] + pub fn write_counting( + store_name: impl Into, + ) -> ( + Self, + Arc, + ) { + let store = Arc::new(crate::ec::identity::test_support::WriteCountingEcKv::new( + store_name, + )); + let graph = Self { + store: store.clone(), + }; + (graph, store) + } +} + #[cfg(test)] impl KvIdentityGraph { /// Test helper: a graph whose every store operation fails, mimicking a diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index bfe74ca6f3..79d581e140 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -171,11 +171,116 @@ pub trait EcKvStore { fn delete(&self, key: &str) -> Result<(), Report>; } +/// An in-memory store for tests, reachable from other crates through the +/// `test-utils` feature. +#[cfg(any(test, feature = "test-utils"))] +pub mod in_memory { + use std::collections::BTreeMap; + use std::sync::Mutex; + + use super::*; + + /// In-memory [`EcKvStore`] with generation tracking for CAS tests. + pub struct InMemoryEcKv { + name: String, + entries: Mutex>, + } + + struct StoredEntry { + body: Vec, + metadata: Option>, + generation: u64, + } + + impl InMemoryEcKv { + /// An empty store named `name`. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + entries: Mutex::new(BTreeMap::new()), + } + } + } + + impl EcKvStore for InMemoryEcKv { + fn store_name(&self) -> &str { + &self.name + } + + fn lookup(&self, key: &str) -> Result, Report> { + let entries = self.entries.lock().expect("should lock in-memory store"); + Ok(entries.get(key).map(|stored| EcKvLookup { + body: stored.body.clone(), + metadata: stored.metadata.clone(), + generation: stored.generation, + })) + } + + fn key_exists(&self, key: &str) -> Result> { + let entries = self.entries.lock().expect("should lock in-memory store"); + Ok(entries.contains_key(key)) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + let mut entries = self.entries.lock().expect("should lock in-memory store"); + let existing_generation = entries.get(key).map(|stored| stored.generation); + + match write.mode { + EcKvWriteMode::Add if existing_generation.is_some() => { + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + EcKvWriteMode::IfGenerationMatch(expected) + if existing_generation != Some(expected) => + { + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + EcKvWriteMode::Add + | EcKvWriteMode::Overwrite + | EcKvWriteMode::IfGenerationMatch(_) => {} + } + + entries.insert( + key.to_owned(), + StoredEntry { + body: write.body.as_bytes().to_vec(), + metadata: Some(write.metadata.as_bytes().to_vec()), + generation: existing_generation.unwrap_or(0) + 1, + }, + ); + Ok(EcKvWriteOutcome::Written) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + let entries = self.entries.lock().expect("should lock in-memory store"); + Ok(entries + .keys() + .filter(|key| key.starts_with(prefix)) + .take(limit as usize) + .cloned() + .collect()) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + let mut entries = self.entries.lock().expect("should lock in-memory store"); + entries.remove(key); + Ok(()) + } + } +} + #[cfg(test)] pub(crate) mod test_support { - use std::collections::BTreeMap; use std::sync::Mutex; + pub(crate) use super::in_memory::InMemoryEcKv; use super::*; /// [`EcKvStore`] wrapper that counts `lookup` calls through a shared counter @@ -312,100 +417,6 @@ pub(crate) mod test_support { } } - /// In-memory [`EcKvStore`] with generation tracking for CAS tests. - pub(crate) struct InMemoryEcKv { - name: String, - entries: Mutex>, - } - - struct StoredEntry { - body: Vec, - metadata: Option>, - generation: u64, - } - - impl InMemoryEcKv { - pub(crate) fn new(name: impl Into) -> Self { - Self { - name: name.into(), - entries: Mutex::new(BTreeMap::new()), - } - } - } - - impl EcKvStore for InMemoryEcKv { - fn store_name(&self) -> &str { - &self.name - } - - fn lookup(&self, key: &str) -> Result, Report> { - let entries = self.entries.lock().expect("should lock in-memory store"); - Ok(entries.get(key).map(|stored| EcKvLookup { - body: stored.body.clone(), - metadata: stored.metadata.clone(), - generation: stored.generation, - })) - } - - fn key_exists(&self, key: &str) -> Result> { - let entries = self.entries.lock().expect("should lock in-memory store"); - Ok(entries.contains_key(key)) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - let mut entries = self.entries.lock().expect("should lock in-memory store"); - let existing_generation = entries.get(key).map(|stored| stored.generation); - - match write.mode { - EcKvWriteMode::Add if existing_generation.is_some() => { - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - EcKvWriteMode::IfGenerationMatch(expected) - if existing_generation != Some(expected) => - { - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - EcKvWriteMode::Add - | EcKvWriteMode::Overwrite - | EcKvWriteMode::IfGenerationMatch(_) => {} - } - - entries.insert( - key.to_owned(), - StoredEntry { - body: write.body.as_bytes().to_vec(), - metadata: Some(write.metadata.as_bytes().to_vec()), - generation: existing_generation.unwrap_or(0) + 1, - }, - ); - Ok(EcKvWriteOutcome::Written) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - let entries = self.entries.lock().expect("should lock in-memory store"); - Ok(entries - .keys() - .filter(|key| key.starts_with(prefix)) - .take(limit as usize) - .cloned() - .collect()) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - let mut entries = self.entries.lock().expect("should lock in-memory store"); - entries.remove(key); - Ok(()) - } - } - /// [`EcKvStore`] that fails every operation, mimicking a missing or /// unreachable platform store. pub(crate) struct FailingEcKv { diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 54acc102fa..c4d61e612e 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -338,8 +338,8 @@ mod tests { pull_sync_enabled: false, pull_sync_url: None, pull_sync_allowed_domains: vec![], - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: None, } } diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 6f714a9452..3a0940a672 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -22,6 +22,7 @@ use crate::settings::Settings; use super::generation::{ec_hash, is_valid_ec_id}; use super::identity::coordinator::{Waited, buffered_body, drain_in_flight}; +use super::identity::generic_pull::generic_pull_id; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; use super::rate_limiter::RateLimiter; @@ -188,12 +189,19 @@ pub fn dispatch_pull_sync( continue; } - let Some(url) = validated_pull_sync_url(partner) else { + let Some(url) = validated_pull_sync_url( + partner.pull_sync_url.as_deref(), + &partner.pull_sync_allowed_domains, + ) else { continue; }; let rate_key = pull_rate_limit_key(&partner.source_domain, context.ec_id()); - match rate_limiter.exceeded(&rate_key, partner.pull_sync_rate_limit) { + let provider_settings = settings + .ec + .identity + .provider(&generic_pull_id(&partner.source_domain)); + match rate_limiter.exceeded_per_minute(&rate_key, provider_settings.max_calls_per_minute) { Ok(true) => { log::debug!( "Pull sync: rate-limited partner '{}' for ec_id '{}'", @@ -308,57 +316,53 @@ fn is_partner_pull_eligible(partner: &PartnerConfig, kv_entry: Option<&KvEntry>) .is_none() } -fn validated_pull_sync_url(partner: &PartnerConfig) -> Option { - let pull_sync_url = partner.pull_sync_url.as_deref()?; +/// Returns the pull URL when it is HTTPS and its host is allowlisted. +/// +/// The host comparison ignores case and a trailing dot on both sides. +/// Returns `None`, with a log line that names no partner, for a missing, +/// invalid or non-HTTPS URL and for a host outside `allowed_domains`. +pub(crate) fn validated_pull_sync_url( + pull_sync_url: Option<&str>, + allowed_domains: &[String], +) -> Option { + let pull_sync_url = pull_sync_url?; let parsed = match Url::parse(pull_sync_url) { Ok(url) => url, Err(err) => { - log::error!( - "Pull sync: partner '{}' has invalid pull_sync_url '{}': {err}", - partner.source_domain, - pull_sync_url - ); + log::error!("Pull sync: invalid pull_sync_url '{pull_sync_url}': {err}"); return None; } }; if parsed.scheme() != "https" { log::error!( - "Pull sync: partner '{}' pull_sync_url must use HTTPS, got scheme '{}'", - partner.source_domain, + "Pull sync: pull_sync_url must use HTTPS, got scheme '{}'", parsed.scheme() ); return None; } let Some(hostname) = parsed.host_str() else { - log::error!( - "Pull sync: partner '{}' pull_sync_url has no hostname: {}", - partner.source_domain, - pull_sync_url - ); + log::error!("Pull sync: pull_sync_url has no hostname: {pull_sync_url}"); return None; }; let hostname = hostname.trim_end_matches('.').to_ascii_lowercase(); - if !partner.pull_sync_allowed_domains.iter().any(|domain| { + if !allowed_domains.iter().any(|domain| { domain .trim() .trim_end_matches('.') .eq_ignore_ascii_case(&hostname) }) { - log::error!( - "Pull sync: partner '{}' URL host '{}' not in pull_sync_allowed_domains", - partner.source_domain, - hostname - ); + log::error!("Pull sync: URL host '{hostname}' not in pull_sync_allowed_domains"); return None; } Some(parsed) } -fn build_pull_request_url(mut base_url: Url, ec_id: &str) -> Url { +/// Appends the `ec_id` query parameter to a validated pull URL. +pub(crate) fn build_pull_request_url(mut base_url: Url, ec_id: &str) -> Url { base_url.query_pairs_mut().append_pair("ec_id", ec_id); base_url } @@ -483,7 +487,7 @@ mod tests { ) } - fn pull_partner(ttl_sec: u64) -> PartnerConfig { + fn pull_partner() -> PartnerConfig { PartnerConfig { name: "SSP X".to_owned(), api_key_hash: Some("deadbeef".to_owned()), @@ -494,8 +498,6 @@ mod tests { pull_sync_enabled: true, pull_sync_url: Some("https://sync.partner.test/pull".to_owned()), pull_sync_allowed_domains: vec!["sync.partner.test".to_owned()], - pull_sync_ttl_sec: ttl_sec, - pull_sync_rate_limit: 20, ts_pull_token: Some(Redacted::new("token".to_owned())), } } @@ -601,7 +603,7 @@ mod tests { #[test] fn partner_is_eligible_when_missing_from_entry() { - let partner = pull_partner(3600); + let partner = pull_partner(); let entry = KvEntry::minimal("other_partner", "uid-1", 100); assert!( @@ -612,7 +614,7 @@ mod tests { #[test] fn partner_is_not_eligible_when_already_present() { - let partner = pull_partner(3600); + let partner = pull_partner(); let entry = KvEntry::minimal("ssp.example.com", "uid-1", 1000); assert!( @@ -648,10 +650,13 @@ mod tests { #[test] fn validated_pull_sync_url_rejects_http_scheme() { - let mut partner = pull_partner(3600); + let mut partner = pull_partner(); partner.pull_sync_url = Some("http://sync.partner.test/pull".to_owned()); - let validated = validated_pull_sync_url(&partner); + let validated = validated_pull_sync_url( + partner.pull_sync_url.as_deref(), + &partner.pull_sync_allowed_domains, + ); assert!( validated.is_none(), "should reject pull_sync_url with HTTP scheme" @@ -660,10 +665,13 @@ mod tests { #[test] fn validated_pull_sync_url_rejects_non_allowlisted_host() { - let mut partner = pull_partner(3600); + let mut partner = pull_partner(); partner.pull_sync_url = Some("https://evil.test/pull".to_owned()); - let validated = validated_pull_sync_url(&partner); + let validated = validated_pull_sync_url( + partner.pull_sync_url.as_deref(), + &partner.pull_sync_allowed_domains, + ); assert!( validated.is_none(), "should reject runtime pull_sync_url host outside allowlist" @@ -672,11 +680,14 @@ mod tests { #[test] fn validated_pull_sync_url_accepts_normalized_allowlist_match() { - let mut partner = pull_partner(3600); + let mut partner = pull_partner(); partner.pull_sync_url = Some("https://SYNC.PARTNER.TEST./pull".to_owned()); partner.pull_sync_allowed_domains = vec!["sync.partner.test".to_owned()]; - let validated = validated_pull_sync_url(&partner); + let validated = validated_pull_sync_url( + partner.pull_sync_url.as_deref(), + &partner.pull_sync_allowed_domains, + ); assert!( validated.is_some(), "should accept allowlist match after hostname normalization" @@ -859,8 +870,8 @@ mod tests { pull_sync_enabled: true, pull_sync_url: Some(format!("https://{source_domain}/sync")), pull_sync_allowed_domains: vec![source_domain.to_owned()], - pull_sync_ttl_sec: 3600, - pull_sync_rate_limit: 100, + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: Some(Redacted::new("outbound-token".to_owned())), } } diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 8a3510b284..efa96233f3 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -348,8 +348,8 @@ mod tests { pull_sync_enabled: true, pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: Some(Redacted::new("pull-token".to_owned())), } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index c4637b4310..689666448a 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -38,13 +38,6 @@ pub struct PartnerConfig { pub pull_sync_url: Option, /// Allowlist of domains TS may call for this partner's pull sync. pub pull_sync_allowed_domains: Vec, - /// Legacy seconds between pull sync refreshes retained for config compatibility. - /// - /// Pull sync currently fills missing partner UIDs only, so this value is - /// not used to refresh existing UIDs. - pub pull_sync_ttl_sec: u64, - /// Max pull sync calls per EC hash per partner per hour. - pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, } @@ -103,14 +96,13 @@ impl PartnerRegistry { })); } - validate_rate_limits_values(partner.batch_rate_limit, partner.pull_sync_rate_limit) - .map_err(|error| { - Report::new(TrustedServerError::Configuration { - message: format!( - "ec.partners: invalid rate limits for '{normalized_source}': {error}" - ), - }) - })?; + validate_rate_limits_values(partner.batch_rate_limit).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: invalid rate limits for '{normalized_source}': {error}" + ), + }) + })?; if partner.pull_sync_enabled { validate_pull_sync_fields( @@ -302,34 +294,23 @@ fn build_partner_config( pull_sync_enabled: partner.pull_sync_enabled, pull_sync_url: partner.pull_sync_url.clone(), pull_sync_allowed_domains: partner.pull_sync_allowed_domains.clone(), - pull_sync_ttl_sec: partner.pull_sync_ttl_sec, - pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), } } fn validate_rate_limits(config: &PartnerConfig) -> Result<(), Report> { - validate_rate_limits_values(config.batch_rate_limit, config.pull_sync_rate_limit).map_err( - |message| { - Report::new(TrustedServerError::Configuration { - message: message.to_owned(), - }) - }, - ) + validate_rate_limits_values(config.batch_rate_limit).map_err(|message| { + Report::new(TrustedServerError::Configuration { + message: message.to_owned(), + }) + }) } -fn validate_rate_limits_values( - batch_rate_limit: u32, - pull_sync_rate_limit: u32, -) -> Result<(), &'static str> { +fn validate_rate_limits_values(batch_rate_limit: u32) -> Result<(), &'static str> { if batch_rate_limit == 0 { return Err("batch_rate_limit must be greater than 0"); } - if pull_sync_rate_limit == 0 { - return Err("pull_sync_rate_limit must be greater than 0"); - } - Ok(()) } @@ -429,8 +410,8 @@ mod tests { pull_sync_enabled: false, pull_sync_url: None, pull_sync_allowed_domains: vec![], - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: None, } } @@ -669,17 +650,4 @@ mod tests { let result = PartnerRegistry::from_config(&[partner]); assert!(result.is_err(), "should reject zero batch_rate_limit"); } - - #[test] - fn zero_pull_sync_rate_limit_is_rejected() { - let mut partner = make_partner("pull.example.com", &valid_api_token("token-p")); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); - partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; - partner.pull_sync_rate_limit = 0; - partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); - - let result = PartnerRegistry::from_config(&[partner]); - assert!(result.is_err(), "should reject zero pull_sync_rate_limit"); - } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 7ba3e1b19d..1379684024 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -7971,8 +7971,8 @@ mod tests { pull_sync_enabled: true, pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: Some(crate::redacted::Redacted::new("pull-token".to_owned())), } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index e583508508..d523eacce2 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -422,22 +422,21 @@ pub struct EcPartner { /// Allowlist of domains TS may call for this partner's pull sync. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub pull_sync_allowed_domains: Vec, - /// Legacy pull-sync refresh interval retained for config compatibility. + /// Catches the removed `pull_sync_ttl_sec` key. /// - /// EC identity entries no longer store per-partner sync timestamps, so - /// this value is not used by the current fill-missing-only pull sync - /// behavior. - #[serde( - default = "EcPartner::default_pull_sync_ttl_sec", - deserialize_with = "from_value_or_str" - )] - pub pull_sync_ttl_sec: u64, - /// Max pull sync calls per EC hash per partner per hour. - #[serde( - default = "EcPartner::default_pull_sync_rate_limit", - deserialize_with = "from_value_or_str" - )] - pub pull_sync_rate_limit: u32, + /// The key was replaced by `refresh_after_sec` in + /// `[ec.identity.providers.]`. It is parsed only so validation + /// can name its replacement instead of failing with a generic + /// unknown-field message, and it is never serialized. + #[serde(default, rename = "pull_sync_ttl_sec", skip_serializing)] + pub removed_pull_sync_ttl_sec: Option, + /// Catches the removed `pull_sync_rate_limit` key. + /// + /// The key was replaced by `max_calls_per_minute` in + /// `[ec.identity.providers.]`. It is parsed only so validation + /// can name its replacement, and it is never serialized. + #[serde(default, rename = "pull_sync_rate_limit", skip_serializing)] + pub removed_pull_sync_rate_limit: Option, /// Outbound bearer token for pull sync requests. #[serde(default)] pub ts_pull_token: Option>, @@ -522,16 +521,6 @@ impl EcPartner { pub const fn default_batch_rate_limit() -> u32 { 60 } - - #[must_use] - pub const fn default_pull_sync_ttl_sec() -> u64 { - 86400 - } - - #[must_use] - pub const fn default_pull_sync_rate_limit() -> u32 { - 10 - } } /// Edge Cookie (EC) configuration. @@ -4237,8 +4226,8 @@ mod tests { pull_sync_enabled: false, pull_sync_url: None, pull_sync_allowed_domains: Vec::new(), - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: Some(Redacted::new(CANARY_EC_PARTNER_TS_PULL_TOKEN.to_string())), }]; @@ -6049,8 +6038,8 @@ source_domain = "partner.example.com" pull_sync_enabled: true, pull_sync_url: Some("https://partner.example.com/sync".to_owned()), pull_sync_allowed_domains: vec!["partner.example.com".to_owned()], - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, ts_pull_token: Some(Redacted::new(ts_pull_token.to_owned())), } } From 273c1486bad615bc18f142c9ab4c4cb65824f707 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 21:57:53 +0530 Subject: [PATCH 32/89] Decide the writer request once routing allows dispatch Only an organic real-browser top-level document navigation that is not a prefetch is the writer. Integration-proxy EC generation now requires a real browser explicitly. The adapter builds one IdentityRuntime per settings load, and the auction, publisher, identify and admin readers check servability against its catalog and the row's opt-outs. Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-axum/src/app.rs | 1 + .../src/app.rs | 1 + .../trusted-server-adapter-fastly/src/app.rs | 44 ++- .../trusted-server-adapter-fastly/src/main.rs | 10 +- crates/trusted-server-adapter-spin/src/app.rs | 1 + .../src/auction/endpoints.rs | 89 +++++- crates/trusted-server-core/src/ec/admin.rs | 106 ++++++-- crates/trusted-server-core/src/ec/identify.rs | 19 +- .../src/ec/identity/catalog.rs | 2 - .../src/ec/identity/mod.rs | 3 + .../src/ec/identity/mutation.rs | 12 +- .../src/ec/identity/request.rs | 255 ++++++++++++++++++ .../src/ec/identity/revision.rs | 3 - .../src/ec/identity/runtime.rs | 198 ++++++++++++++ .../src/ec/identity/servable.rs | 16 +- crates/trusted-server-core/src/ec/mod.rs | 148 +++++++++- crates/trusted-server-core/src/http_util.rs | 57 ++++ .../src/integrations/gpt_diagnostics.rs | 2 +- .../src/integrations/registry.rs | 17 +- .../src/migration_guards.rs | 8 + crates/trusted-server-core/src/publisher.rs | 16 +- 21 files changed, 922 insertions(+), 86 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/request.rs create mode 100644 crates/trusted-server-core/src/ec/identity/runtime.rs diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index caba714d68..af653b7e0d 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -232,6 +232,7 @@ async fn dispatch_fallback( kv: None, ec_context: &mut ec_context, services, + is_real_browser: false, req, }) .await diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 87b9567e7f..90b3bf536e 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -544,6 +544,7 @@ fn build_router(state: &Arc) -> RouterService { kv: None, ec_context: &mut ec_context, services: &services, + is_real_browser: false, req, }) .await diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index bf09c2086e..0d071408c8 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -113,6 +113,7 @@ use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; +use trusted_server_core::ec::identity::IdentityRuntime; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EidSyncSource}; @@ -189,6 +190,8 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, + /// Identity providers, keys and policy, built once per settings load. + pub(crate) identity: Arc, } /// Build the application state, loading settings and constructing all per-application components. @@ -215,6 +218,17 @@ pub(crate) fn load_settings_from_config_store( ) } +/// Builds the identity runtime for one settings load. +/// +/// # Errors +/// +/// Returns an error when a provider cannot be described or the catalog rules fail. +pub(crate) fn build_identity_runtime( + settings: &Settings, +) -> Result, Report> { + IdentityRuntime::from_settings(settings).map(Arc::new) +} + pub(crate) fn build_state_from_settings( settings: Settings, ) -> Result, Report> { @@ -224,6 +238,7 @@ pub(crate) fn build_state_from_settings( plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; let registry = IntegrationRegistry::with_plan(&settings, plan)?; + let identity = build_identity_runtime(&settings)?; let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; @@ -234,6 +249,7 @@ pub(crate) fn build_state_from_settings( registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, + identity, })) } @@ -402,10 +418,11 @@ fn device_signals_for(req: &Request) -> DeviceSignals { /// Builds the per-request EC state, mirroring the pre-routing prelude of the /// legacy `route_request` step by step. fn build_ec_request_state( - settings: &Settings, + state: &AppState, services: &RuntimeServices, req: &Request, ) -> EcRequestState { + let settings = &*state.settings; let device_signals = device_signals_for(req); let is_real_browser = device_signals.looks_like_browser(); if !is_real_browser { @@ -432,6 +449,8 @@ fn build_ec_request_state( match EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) { Ok(mut context) => { context.set_device_signals(device_signals); + context.set_real_browser(is_real_browser); + context.set_identity_runtime(Arc::clone(&state.identity)); // Orphan-recovery eligibility is intentionally left false here. // Authorizing it during generic pre-routing would let named // routes and request-filter short circuits (e.g. a DataDome @@ -583,7 +602,12 @@ async fn execute_named( // copy is bot-gated, while operators use curl for this // authenticated diagnostic. let kv = crate::maybe_identity_graph(&state.settings); - handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) + handle_admin_ec_lookup( + kv.as_ref(), + ®istry, + Some(state.identity.as_ref()), + &req, + ) } NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), _ => unreachable!("admin diagnostics should use early dispatch"), @@ -599,7 +623,7 @@ async fn execute_named( return Ok(http_error(&report)); } - let mut ec = build_ec_request_state(&state.settings, &services, &req); + let mut ec = build_ec_request_state(&state, &services, &req); // EcContext creation errors short-circuit before filters, mirroring legacy: // the legacy path returns its error response before running filter_request. if let Some(report) = ec.setup_error.take() { @@ -788,7 +812,7 @@ async fn dispatch_fallback( return http_error(&report); } - let mut ec = build_ec_request_state(&state.settings, services, &req); + let mut ec = build_ec_request_state(state, services, &req); if let Some(report) = ec.setup_error.take() { let response = http_error(&report); return attach_dispatch_extensions(response, ec, RequestFilterEffects::default()); @@ -810,6 +834,7 @@ async fn dispatch_fallback( // Integration-proxy responses are not bounded by // publisher.max_buffered_body_bytes. Publisher fallback below uses the // publisher-specific streaming finalizer instead. + ec.ec_context.decide_writer(&req, true, ec.is_real_browser); state .registry .handle_proxy(ProxyDispatchInput { @@ -819,6 +844,7 @@ async fn dispatch_fallback( kv: ec.kv_graph.as_ref(), ec_context: &mut ec.ec_context, services, + is_real_browser: ec.is_real_browser, req, }) .await @@ -844,6 +870,7 @@ async fn dispatch_fallback( // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. let is_navigation = is_navigation_request(&req); + ec.ec_context.decide_writer(&req, true, ec.is_real_browser); if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); } @@ -1372,9 +1399,9 @@ mod tests { use super::{ AppState, AuctionDispatch, EcContext, EdgeCacheHeader, EidSyncSource, HandlerFuture, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, - TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, - build_state_from_settings, compile_auction_plan, handle_publisher_request, - publisher_response_into_streaming_response, startup_error_router, + TrustedServerApp, build_identity_runtime, build_orchestrator_with_plan, + build_per_request_services, build_state_from_settings, compile_auction_plan, + handle_publisher_request, publisher_response_into_streaming_response, startup_error_router, }; use base64::Engine as _; use bytes::Bytes; @@ -1566,6 +1593,8 @@ mod tests { filters: Vec>, ) -> RouterService { let settings = test_settings(); + let identity = + build_identity_runtime(&settings).expect("should build the identity runtime"); let plan = Arc::new( trusted_server_core::auction::compile_auction_plan(&settings) .expect("should compile auction plan"), @@ -1584,6 +1613,7 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + identity, }); TrustedServerApp::routes_for_state(&state) } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index b8f814a01c..c4ca8662c7 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -43,7 +43,7 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, + AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, build_identity_runtime, load_settings_from_config_store, }; use crate::ec_kv::FastlyEcKvStore; @@ -526,6 +526,14 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re } else { match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { + // No application state was built, so build the identity + // runtime from the reloaded settings. + match build_identity_runtime(&settings) { + Ok(identity) => ec_state.ec_context.set_identity_runtime(identity), + Err(e) => { + log::warn!("identity runtime unavailable after settings reload: {e:?}"); + } + } match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { Ok(partner_registry) => { send_edgezero_response( diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 804a23b620..9c309d9319 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -825,6 +825,7 @@ fn build_router(state: &Arc) -> RouterService { kv: None, ec_context: &mut ec_context, services: &services, + is_real_browser: false, req, }) .await diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 13c5846438..a2d63440af 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -20,7 +20,8 @@ use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::identity::UnixTime; use crate::ec::identity::catalog::ProviderCatalog; use crate::ec::identity::policy::PolicySignals; -use crate::ec::identity::servable::{ServeContext, opted_out_providers}; +use crate::ec::identity::request::opted_out; +use crate::ec::identity::servable::ServeContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::partner::normalize_partner_source_domain; use crate::ec::prebid_eids::parse_prebid_eids_cookie; @@ -480,15 +481,22 @@ pub(crate) fn resolve_auction_eids( let Some(now) = checked_current_timestamp() else { return Some(Vec::new()); }; - // Until the request's identity runtime is on `EcContext`, no provider is - // configured, so only unowned records without a writer are servable. - let catalog = ProviderCatalog::empty(); - let opted_out = opted_out_providers(entry); + // Without a runtime no provider is configured, so only unowned records + // without a writer are servable. + let empty_catalog; + let catalog = match ec_context.identity_runtime() { + Some(runtime) => runtime.catalog(), + None => { + empty_catalog = ProviderCatalog::empty(); + &empty_catalog + } + }; + let opted_out = opted_out(entry); let serve = ServeContext { now: UnixTime::from_secs(now), consent: ec_context.consent(), signals, - catalog: &catalog, + catalog, opted_out: &opted_out, }; let resolved = resolve_partner_ids(registry, entry, &serve); @@ -1789,6 +1797,75 @@ mod tests { ); } + #[test] + fn generic_pull_record_with_writer_is_served_after_flush() { + use crate::ec::identity::IdentityRuntime; + use crate::ec::identity::test_support::{ec_partner, record_for_test}; + use crate::test_support::tests::create_test_settings; + + let mut partner = ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull", + "example-token-value-0123456789abcdef", + ); + partner.api_token = Some(crate::redacted::Redacted::new( + "token-ssp-example-com-32-bytes-minimum".to_owned(), + )); + let registry = PartnerRegistry::from_config(std::slice::from_ref(&partner)) + .expect("should build registry"); + let mut settings = create_test_settings(); + settings.ec.partners = vec![partner]; + let with_provider = std::sync::Arc::new( + IdentityRuntime::from_settings(&settings).expect("should build the runtime"), + ); + settings.ec.partners = Vec::new(); + let without_provider = std::sync::Arc::new( + IdentityRuntime::from_settings(&settings).expect("should build the runtime"), + ); + + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = crate::ec::kv_types::KvEntry::minimal("other.example.com", "other", 1); + entry.ids.insert( + "ssp.example.com".to_owned(), + record_for_test("pulled-uid", Some("pull-ssp-example-com"), 1), + ); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + let resolve = |runtime: Option>| { + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + if let Some(runtime) = runtime { + ec_context.set_identity_runtime(runtime); + } + resolve_auction_eids( + &snapshot, + Some(®istry), + &ec_context, + PolicySignals::default(), + ) + .expect("should resolve EIDs") + }; + + let served = resolve(Some(with_provider)); + let dropped = resolve(Some(without_provider)); + let no_runtime = resolve(None); + + assert!( + served.iter().any(|eid| eid.source == "ssp.example.com"), + "should send the pulled UID when the runtime's catalog holds its writer" + ); + assert!( + dropped.iter().all(|eid| eid.source != "ssp.example.com"), + "control: should drop the record when the catalog lacks its writer" + ); + assert!( + no_runtime.iter().all(|eid| eid.source != "ssp.example.com"), + "control: should drop the record without a runtime" + ); + } + #[test] fn resolve_auction_eids_without_providers_matches_pre_identity_output() { let partners = vec![ diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e12e1e61b5..ed93077dd4 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -36,10 +36,12 @@ use crate::openrtb::Eid; use super::checked_current_timestamp; use super::eids::{resolve_partner_ids, to_eids}; use super::generation::is_valid_ec_id; +use super::identity::IdentityRuntime; use super::identity::UnixTime; use super::identity::catalog::ProviderCatalog; use super::identity::policy::PolicySignals; -use super::identity::servable::{ServeContext, opted_out_providers}; +use super::identity::request::opted_out; +use super::identity::servable::ServeContext; use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; use super::kv_types::{KvEntry, KvMetadata}; @@ -283,6 +285,7 @@ struct SkippedPartnerId { pub fn handle_admin_ec_lookup( kv: Option<&KvIdentityGraph>, registry: &PartnerRegistry, + runtime: Option<&IdentityRuntime>, req: &Request, ) -> Result, Report> { let Some(kv) = kv else { @@ -304,7 +307,7 @@ pub fn handle_admin_ec_lookup( }; log::info!("Admin EC lookup: returning entry for '{}'", log_id(&ec_id)); - let payload = build_lookup_response(registry, kv.store_name(), ec_id, &lookup); + let payload = build_lookup_response(registry, runtime, kv.store_name(), ec_id, &lookup); let body = serde_json::to_string(&payload).change_context(TrustedServerError::Configuration { message: "failed to serialize admin EC lookup response".to_owned(), @@ -394,6 +397,7 @@ fn redact_provider_state(entry: &mut JsonValue) { /// corrupt entries remain inspectable. fn build_lookup_response( registry: &PartnerRegistry, + runtime: Option<&IdentityRuntime>, store_name: &str, ec_id: String, lookup: &EcKvLookup, @@ -427,7 +431,7 @@ fn build_lookup_response( .and_then(|dropped| entry.validate().map(|()| dropped)); match upgraded { Ok(dropped) => { - payload.auction = Some(build_auction_view(registry, &entry)); + payload.auction = Some(build_auction_view(registry, &entry, runtime)); if !dropped.is_empty() { payload.entry_error = Some(format!("dropped on read: {}", dropped.join("; "))); @@ -513,21 +517,32 @@ fn iso_timestamp(unix_seconds: u64) -> Option { /// Derives the auction EID view for a valid entry, mirroring the filters in /// [`resolve_partner_ids`] and reporting why each stored ID was skipped. -fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { +fn build_auction_view( + registry: &PartnerRegistry, + entry: &KvEntry, + runtime: Option<&IdentityRuntime>, +) -> AuctionEidsView { // The admin view has no request, so it reads with permissive consent. A // failed clock resolves no KV IDs, as the auction does (spec 4.4). let consent = ConsentContext { jurisdiction: Jurisdiction::NonRegulated, ..ConsentContext::default() }; - let catalog = ProviderCatalog::empty(); - let opted_out = opted_out_providers(entry); + let empty_catalog; + let catalog = match runtime { + Some(runtime) => runtime.catalog(), + None => { + empty_catalog = ProviderCatalog::empty(); + &empty_catalog + } + }; + let opted_out = opted_out(entry); let now = checked_current_timestamp(); let serve = ServeContext { now: UnixTime::from_secs(now.unwrap_or(0)), consent: &consent, signals: PolicySignals::default(), - catalog: &catalog, + catalog, opted_out: &opted_out, }; let resolved = if now.is_some() { @@ -1031,7 +1046,7 @@ mod tests { let kv = kv_with_entry(&ec_id, entry); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); response_json(response) } @@ -1088,7 +1103,7 @@ mod tests { let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); let json = response_json( - handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"), ); @@ -1103,6 +1118,49 @@ mod tests { ); } + #[test] + fn admin_view_serves_generic_pull_records_with_the_runtime_catalog() { + use crate::ec::identity::test_support::{ec_partner, record_for_test}; + use crate::test_support::tests::create_test_settings; + + let mut partner = ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull", + "example-token-value-0123456789abcdef", + ); + partner.api_token = Some(crate::redacted::Redacted::new( + "token-ssp-example-com-32-bytes-minimum".to_owned(), + )); + let registry = PartnerRegistry::from_config(std::slice::from_ref(&partner)) + .expect("should build registry"); + let mut settings = create_test_settings(); + settings.ec.partners = vec![partner]; + let runtime = IdentityRuntime::from_settings(&settings).expect("should build the runtime"); + let mut entry = KvEntry::minimal("other.example.com", "other", 1); + entry.ids.insert( + "ssp.example.com".to_owned(), + record_for_test("pulled-uid", Some("pull-ssp-example-com"), 1), + ); + + let with_runtime = build_auction_view(®istry, &entry, Some(&runtime)); + let without_runtime = build_auction_view(®istry, &entry, None); + + assert!( + with_runtime + .eids + .iter() + .any(|eid| eid.source == "ssp.example.com"), + "should serve the pulled record when the runtime's catalog holds its writer" + ); + assert!( + without_runtime + .eids + .iter() + .all(|eid| eid.source != "ssp.example.com"), + "control: should not serve it without a runtime" + ); + } + #[test] fn auction_view_reports_an_expired_record_as_not_servable() { let mut entry = KvEntry::minimal("bidstream.example", "uid-expired", 1_741_824_000); @@ -1112,7 +1170,7 @@ mod tests { .expect("should hold the source") .exp = Some(50); - let view = build_auction_view(&test_registry(), &entry); + let view = build_auction_view(&test_registry(), &entry, None); assert!(view.eids.is_empty(), "should not resolve the expired ID"); assert_eq!(view.skipped.len(), 1, "should explain the dropped ID"); @@ -1128,7 +1186,7 @@ mod tests { let kv = kv_with_entry(&ec_id, &sample_entry()); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::OK); @@ -1224,7 +1282,7 @@ mod tests { let kv = kv_with_raw_body_and_metadata(&ec_id, &body, &metadata); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); let json = response_json(response); @@ -1251,7 +1309,7 @@ mod tests { let kv = kv_with_raw_body(&ec_id, &body); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); let json = response_json(response); @@ -1283,7 +1341,7 @@ mod tests { ); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::OK); @@ -1303,7 +1361,7 @@ mod tests { let kv = KvIdentityGraph::in_memory("test-store"); let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::NOT_FOUND); @@ -1314,7 +1372,7 @@ mod tests { let kv = KvIdentityGraph::in_memory("test-store"); let req = get_request("/_ts/admin/ec/not-a-valid-id"); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::BAD_REQUEST); @@ -1326,7 +1384,7 @@ mod tests { let kv = kv_with_raw_body(&ec_id, "not json at all"); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!( @@ -1367,7 +1425,7 @@ mod tests { let kv = kv_with_raw_body(&ec_id, &body); let req = get_request(&format!("/_ts/admin/ec/{ec_id}")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::OK); @@ -1392,7 +1450,7 @@ mod tests { let kv = kv_with_entry(&ec_id, &sample_entry()); let req = get_request_with_cookie("/_ts/admin/ec", &format!("other=1; ts-ec={ec_id}; x=2")); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::OK); @@ -1417,7 +1475,7 @@ mod tests { &format!("ts-ec={stale_ec_id}; ts-ec={live_ec_id}"), ); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::OK); @@ -1434,7 +1492,7 @@ mod tests { let kv = KvIdentityGraph::in_memory("test-store"); let req = get_request("/_ts/admin/ec"); - let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req) + let response = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req) .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::NOT_FOUND); @@ -1452,8 +1510,8 @@ mod tests { fn missing_identity_graph_returns_501() { let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); - let response = - handle_admin_ec_lookup(None, &test_registry(), &req).expect("should handle lookup"); + let response = handle_admin_ec_lookup(None, &test_registry(), None, &req) + .expect("should handle lookup"); assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); } @@ -1487,7 +1545,7 @@ mod tests { let kv = KvIdentityGraph::failing("broken-store"); let req = get_request(&format!("/_ts/admin/ec/{}", test_ec_id())); - let result = handle_admin_ec_lookup(Some(&kv), &test_registry(), &req); + let result = handle_admin_ec_lookup(Some(&kv), &test_registry(), None, &req); assert!(result.is_err(), "should propagate KV read failures"); } diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 7670116376..d050449ac5 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -21,7 +21,8 @@ use super::identity::SourceDomain; use super::identity::UnixTime; use super::identity::catalog::ProviderCatalog; use super::identity::policy::PolicySignals; -use super::identity::servable::{ServeContext, is_servable, opted_out_providers}; +use super::identity::request::opted_out; +use super::identity::servable::{ServeContext, is_servable}; use super::kv::KvIdentityGraph; use super::kv_types::KvPartnerId; use super::log_id; @@ -107,15 +108,21 @@ pub fn handle_identify( // the row's stored snapshot. A failed clock serves nothing, // because a zero time would make every expiry check pass. if let Some(now) = checked_current_timestamp() { - // Until the request's identity runtime is on `EcContext`, no - // provider is configured. - let catalog = ProviderCatalog::empty(); - let opted_out = opted_out_providers(&entry); + // Without a runtime no provider is configured. + let empty_catalog; + let catalog = match ec_context.identity_runtime() { + Some(runtime) => runtime.catalog(), + None => { + empty_catalog = ProviderCatalog::empty(); + &empty_catalog + } + }; + let opted_out = opted_out(&entry); let serve = ServeContext { now: UnixTime::from_secs(now), consent: ec_context.consent(), signals: PolicySignals::from_headers(req.headers()), - catalog: &catalog, + catalog, opted_out: &opted_out, }; if let Some(partner_uid) = entry diff --git a/crates/trusted-server-core/src/ec/identity/catalog.rs b/crates/trusted-server-core/src/ec/identity/catalog.rs index 1f0e075eb3..fed9a7071b 100644 --- a/crates/trusted-server-core/src/ec/identity/catalog.rs +++ b/crates/trusted-server-core/src/ec/identity/catalog.rs @@ -112,7 +112,6 @@ impl ProviderCatalog { /// Returns [`TrustedServerError::Configuration`] when a pull partner /// cannot be described or the catalog rules fail. // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn from_settings(settings: &Settings) -> Result> { let providers = settings .ec @@ -171,7 +170,6 @@ impl ProviderCatalog { /// does not declare and a narrowing to nothing. #[must_use] // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn with_narrowing( mut self, narrowing: BTreeMap>, diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 7ccd217584..856ba8a4dc 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -18,8 +18,10 @@ pub(crate) mod mutation; pub(crate) mod policy; pub mod provider; pub(crate) mod removal; +pub(crate) mod request; pub mod resolution; pub(crate) mod revision; +pub mod runtime; pub(crate) mod servable; #[cfg(any(test, feature = "test-utils"))] pub mod test_support; @@ -35,3 +37,4 @@ pub use provider::{IdentityProvider, IdentityProviderConfig}; pub use resolution::{ IdentityProviderError, Resolution, ResolvedEid, ResolvedUid, StateChange, When, }; +pub use runtime::IdentityRuntime; diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs index 71db219cf9..6e444bac31 100644 --- a/crates/trusted-server-core/src/ec/identity/mutation.rs +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -194,6 +194,13 @@ impl IdentityMutation { self.cluster_size = Some(size); } + /// How many changes the mutation holds: stand-alone ones, provider + /// results and provider state changes. Holds no UID, so it is safe to log. + #[must_use] + pub(crate) fn change_count(&self) -> usize { + self.partner_ids.len() + self.results.len() + self.provider_state.len() + } + /// Whether nothing would change. #[must_use] // Temporary: first used by a later identity task. @@ -576,9 +583,8 @@ fn apply_clear(entry: &mut KvEntry, provider: &IdentityProviderId, clear: u64) { } } -// Temporary: first used by a later identity task. -#[allow(dead_code)] -fn is_opted_out(entry: &KvEntry, provider: &str) -> bool { +/// Whether `provider` has a sticky opt-out in `entry` that no later clear lifts. +pub(crate) fn is_opted_out(entry: &KvEntry, provider: &str) -> bool { entry .ps .get(provider) diff --git a/crates/trusted-server-core/src/ec/identity/request.rs b/crates/trusted-server-core/src/ec/identity/request.rs new file mode 100644 index 0000000000..b31e71c2d3 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/request.rs @@ -0,0 +1,255 @@ +//! Identity state one request carries on [`crate::ec::EcContext`] (spec 5.1). + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use url::Url; + +use super::ids::{IdentityProviderId, SourceDomain}; +use super::mutation::{IdentityMutation, is_opted_out}; +use super::runtime::IdentityRuntime; +use crate::ec::kv_types::KvEntry; + +/// The providers with a sticky opt-out (`ps.o`) in `entry`, unless a later +/// `oc` clears it (spec 7.1). +/// +/// KV is the only source: no cookie carries an opt-out. A key that is not a +/// valid provider ID cannot name a configured provider and is skipped. +#[must_use] +pub(crate) fn opted_out(entry: &KvEntry) -> BTreeSet { + entry + .ps + .keys() + .filter(|provider| is_opted_out(entry, provider)) + .filter_map(|provider| provider.parse().ok()) + .collect() +} + +/// What one request decides and collects for identity. +/// +/// Its [`Debug`](core::fmt::Debug) prints counts and flags only: the mutation +/// holds UIDs that must not reach a log (spec 9.3). +#[derive(Default, Clone)] +pub(crate) struct RequestIdentity { + writer: bool, + is_real_browser: bool, + user_agent: Option, + dnt: bool, + page_url: Option, + runtime: Option>, + mutation: IdentityMutation, + pending_removals: Option, + removal_lost: u32, + capture_unpersisted: BTreeMap<(IdentityProviderId, SourceDomain), u32>, +} + +impl core::fmt::Debug for RequestIdentity { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("RequestIdentity") + .field("writer", &self.writer) + .field("is_real_browser", &self.is_real_browser) + .field("mutation_changes", &self.mutation.change_count()) + .field("has_pending_removals", &self.pending_removals.is_some()) + .field("removal_lost", &self.removal_lost) + .field("has_runtime", &self.runtime.is_some()) + .finish_non_exhaustive() + } +} + +// Temporary: the accessors and setters below are first used by later identity tasks. +#[allow(dead_code)] +impl RequestIdentity { + /// Whether this request is the one browser-driven writer. + #[must_use] + pub(crate) fn writer(&self) -> bool { + self.writer + } + + /// Whether the request passed the real-browser check. + #[must_use] + pub(crate) fn is_real_browser(&self) -> bool { + self.is_real_browser + } + + /// The request's `User-Agent`, captured in phase 1. + #[must_use] + pub(crate) fn user_agent(&self) -> Option<&str> { + self.user_agent.as_deref() + } + + /// Whether the request sent `DNT: 1`. + #[must_use] + pub(crate) fn dnt(&self) -> bool { + self.dnt + } + + /// The page URL without query or fragment. + #[must_use] + pub(crate) fn page_url(&self) -> Option<&Url> { + self.page_url.as_ref() + } + + /// The request's identity runtime, when the adapter set one. + #[must_use] + pub(crate) fn runtime(&self) -> Option<&Arc> { + self.runtime.as_ref() + } + + /// The writer's one flush. + #[must_use] + pub(crate) fn mutation(&self) -> &IdentityMutation { + &self.mutation + } + + /// The writer's one flush, to fill. + pub(crate) fn mutation_mut(&mut self) -> &mut IdentityMutation { + &mut self.mutation + } + + /// A removal write that failed before send, to try once more after send. + #[must_use] + pub(crate) fn pending_removals(&self) -> Option<&IdentityMutation> { + self.pending_removals.as_ref() + } + + /// Keeps a removal write for one retry after send. + pub(crate) fn set_pending_removals(&mut self, removals: Option) { + self.pending_removals = removals; + } + + /// Removals every attempt failed to write (spec 5.2 phase 4, 5.3). + #[must_use] + pub(crate) fn removal_lost(&self) -> u32 { + self.removal_lost + } + + /// Counts one removal that every attempt failed to write. + pub(crate) fn record_removal_lost(&mut self) { + self.removal_lost = self.removal_lost.saturating_add(1); + } + + /// Positive capture IDs counted but never stored (spec 4.7). + #[must_use] + pub(crate) fn capture_unpersisted(&self) -> &BTreeMap<(IdentityProviderId, SourceDomain), u32> { + &self.capture_unpersisted + } + + /// Counts one positive capture ID that was not stored. + pub(crate) fn record_unpersisted( + &mut self, + provider: IdentityProviderId, + source: SourceDomain, + ) { + let count = self + .capture_unpersisted + .entry((provider, source)) + .or_insert(0); + *count = count.saturating_add(1); + } + + /// Sets the writer decision and the signals captured with it. + pub(crate) fn set_writer_decision(&mut self, decision: WriterDecision) { + self.writer = decision.writer; + self.is_real_browser = decision.is_real_browser; + self.user_agent = decision.user_agent; + self.dnt = decision.dnt; + self.page_url = decision.page_url; + } + + /// Records only the real-browser flag, for a route that is never the writer. + pub(crate) fn set_real_browser(&mut self, is_real_browser: bool) { + self.is_real_browser = is_real_browser; + } + + /// Sets the runtime built once per settings load. + pub(crate) fn set_runtime(&mut self, runtime: Arc) { + self.runtime = Some(runtime); + } +} + +/// The writer decision and the signals captured with it. +pub(crate) struct WriterDecision { + pub(crate) writer: bool, + pub(crate) is_real_browser: bool, + pub(crate) user_agent: Option, + pub(crate) dnt: bool, + pub(crate) page_url: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::mutation::PartnerIdChange; + use crate::ec::identity::revision::Revision; + use crate::ec::identity::test_support::{pid, record_for_test, source}; + use crate::ec::kv_types::{KvEntry, KvOptOut, KvProviderState}; + + #[test] + fn request_identity_debug_prints_no_uid() { + let mut identity = RequestIdentity::default(); + identity + .mutation_mut() + .add_change(PartnerIdChange::FillMissing { + source: source("ssp.example.com"), + record: record_for_test("secret-uid-1", None, 10), + }); + identity + .mutation_mut() + .add_change(PartnerIdChange::OwnerRemove { + source: source("dsp.example.com"), + provider: pid("owner"), + expected: Revision::Unset, + }); + + let printed = format!("{identity:?}"); + + assert!( + !printed.contains("secret-uid-1"), + "should print no UID: {printed}" + ); + assert!( + printed.contains("mutation_changes: 2"), + "should print the change count: {printed}" + ); + } + + #[test] + fn opted_out_reads_ps_o_and_honours_a_later_clear() { + let mut entry = KvEntry::minimal("ssp.example.com", "uid", 1); + let opt_out = || KvOptOut { + at: 100, + fp: String::new(), + }; + entry.ps.insert( + "a".to_owned(), + KvProviderState { + o: Some(opt_out()), + ..KvProviderState::default() + }, + ); + entry.ps.insert( + "b".to_owned(), + KvProviderState { + o: Some(opt_out()), + oc: Some(101), + ..KvProviderState::default() + }, + ); + entry.ps.insert( + "c".to_owned(), + KvProviderState { + oc: Some(50), + ..KvProviderState::default() + }, + ); + + let result = opted_out(&entry); + + assert_eq!( + result, + BTreeSet::from([pid("a")]), + "should keep only the provider whose opt-out is not cleared later" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/revision.rs b/crates/trusted-server-core/src/ec/identity/revision.rs index cae88de048..6690ba9695 100644 --- a/crates/trusted-server-core/src/ec/identity/revision.rs +++ b/crates/trusted-server-core/src/ec/identity/revision.rs @@ -16,8 +16,6 @@ type HmacSha256 = Hmac; /// It is the only keyed value: nothing stored in the row is keyed /// (spec 9.3). #[derive(Clone)] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) struct IdentityKeys { visitor: [u8; 32], } @@ -32,7 +30,6 @@ impl IdentityKeys { /// Derives the visitor key from `ec.passphrase`. #[must_use] // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn from_settings(settings: &Settings) -> Self { let passphrase = settings.ec.passphrase.expose().as_bytes(); let mut mac = diff --git a/crates/trusted-server-core/src/ec/identity/runtime.rs b/crates/trusted-server-core/src/ec/identity/runtime.rs new file mode 100644 index 0000000000..19d0aa044d --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/runtime.rs @@ -0,0 +1,198 @@ +//! The identity state built once per settings load (spec 10.3). + +use std::collections::BTreeMap; + +use error_stack::Report; + +use super::catalog::ProviderCatalog; +use super::coordinator::IdentityDeps; +use super::ids::{IdentityProviderId, SourceDomain}; +use super::input::RandomSource; +use super::revision::IdentityKeys; +use crate::ec::rate_limiter::RateLimiter; +use crate::error::TrustedServerError; +use crate::platform::RuntimeServices; +use crate::settings::{IdentitySettings, Settings}; + +/// The provider catalog, the keys and the identity policy of one settings load. +/// +/// The adapter builds it once and shares it through an `Arc`. It is the one +/// identity type the adapter names, so the crate-private catalog and keys +/// never cross the crate boundary. +pub struct IdentityRuntime { + catalog: ProviderCatalog, + // Temporary: first read by a later identity task. + #[allow(dead_code)] + keys: IdentityKeys, + // Temporary: first read by a later identity task. + #[allow(dead_code)] + identity: IdentitySettings, +} + +impl core::fmt::Debug for IdentityRuntime { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("IdentityRuntime") + .field("providers", &self.catalog.providers().count()) + .finish_non_exhaustive() + } +} + +impl IdentityRuntime { + /// Builds the runtime from settings, without resolving any secret. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when a provider cannot be + /// described or the catalog rules fail. + pub fn from_settings(settings: &Settings) -> Result> { + let catalog = ProviderCatalog::from_settings(settings)? + .with_narrowing(narrowing(&settings.ec.identity)); + Ok(Self { + catalog, + keys: IdentityKeys::from_settings(settings), + identity: settings.ec.identity.clone(), + }) + } + + /// The enabled providers. + #[must_use] + pub(crate) fn catalog(&self) -> &ProviderCatalog { + &self.catalog + } + + /// The keys derived from `ec.passphrase`. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn keys(&self) -> &IdentityKeys { + &self.keys + } + + /// The `[ec.identity]` policy. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn identity(&self) -> &IdentitySettings { + &self.identity + } + + /// What the coordinator needs besides the request. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn deps<'a>( + &'a self, + settings: &'a Settings, + services: &'a RuntimeServices, + random: &'a dyn RandomSource, + rate_limiter: Option<&'a dyn RateLimiter>, + ) -> IdentityDeps<'a> { + IdentityDeps { + settings, + catalog: &self.catalog, + keys: &self.keys, + random, + services, + rate_limiter, + } + } +} + +/// Each provider table's `sources`, for the providers that list any. +/// +/// Config validation has rejected a malformed source, so one that does not +/// parse is skipped. +fn narrowing(identity: &IdentitySettings) -> BTreeMap> { + identity + .providers + .iter() + .filter_map(|(id, table)| { + let listed = table.sources.as_ref()?; + Some(( + id.parse().ok()?, + listed + .iter() + .filter_map(|source| source.parse().ok()) + .collect(), + )) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::identity::test_support::{ec_partner, pid, source}; + use crate::settings::ProviderSettings; + use crate::test_support::tests::create_test_settings; + + fn settings_with_listed_sources(listed: Option>) -> Settings { + let mut settings = create_test_settings(); + settings.ec.partners = vec![ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull", + "example-token-value-0123456789abcdef", + )]; + settings.ec.identity.providers.insert( + "pull-ssp-example-com".to_owned(), + ProviderSettings { + sources: listed, + ..ProviderSettings::default() + }, + ); + settings + } + + #[test] + fn runtime_debug_prints_no_key_material() { + let settings = settings_with_listed_sources(None); + let runtime = IdentityRuntime::from_settings(&settings).expect("should build"); + + let printed = format!("{runtime:?}"); + + assert!( + printed.contains("providers: 1"), + "should print the provider count: {printed}" + ); + assert!( + !printed.contains(settings.ec.passphrase.expose()), + "should not print the passphrase: {printed}" + ); + assert!( + !printed.contains("keys"), + "should not print the keys: {printed}" + ); + } + + #[test] + fn runtime_narrows_providers_by_their_sources_table() { + let id = pid("pull-ssp-example-com"); + + let unlisted = IdentityRuntime::from_settings(&settings_with_listed_sources(None)) + .expect("should build"); + let kept = IdentityRuntime::from_settings(&settings_with_listed_sources(Some(vec![ + "ssp.example.com".to_owned(), + ]))) + .expect("should build"); + let dropped = IdentityRuntime::from_settings(&settings_with_listed_sources(Some(vec![ + "other.example.com".to_owned(), + ]))) + .expect("should build"); + + assert_eq!( + unlisted.catalog().sources_of(&id), + [source("ssp.example.com")], + "control: should claim the descriptor's source with no table" + ); + assert_eq!( + kept.catalog().sources_of(&id), + [source("ssp.example.com")], + "should keep a listed source" + ); + assert!( + dropped.catalog().sources_of(&id).is_empty(), + "should drop a source the table does not list" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/servable.rs b/crates/trusted-server-core/src/ec/identity/servable.rs index 613c3c879e..19fb7ee237 100644 --- a/crates/trusted-server-core/src/ec/identity/servable.rs +++ b/crates/trusted-server-core/src/ec/identity/servable.rs @@ -7,7 +7,7 @@ use super::catalog::ProviderCatalog; use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; use super::policy::{PolicySignals, policy_allows}; use crate::consent::ConsentContext; -use crate::ec::kv_types::{KvEntry, KvPartnerId}; +use crate::ec::kv_types::KvPartnerId; /// What a servability decision needs from the request. pub(crate) struct ServeContext<'a> { @@ -19,20 +19,6 @@ pub(crate) struct ServeContext<'a> { pub(crate) opted_out: &'a BTreeSet, } -/// The providers with a sticky opt-out (`ps.o`) in the row the request read. -/// -/// Read from KV only, never from the request (spec 7.1). A key that is not a -/// valid provider ID cannot name a configured provider and is skipped. -#[must_use] -pub(crate) fn opted_out_providers(entry: &KvEntry) -> BTreeSet { - entry - .ps - .iter() - .filter(|(_, state)| state.o.is_some()) - .filter_map(|(provider, _)| provider.parse().ok()) - .collect() -} - /// Whether a stored record, the source's whole entry set, may be sent from /// the server. #[must_use] diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index ce599c0c36..7f038461a2 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -69,10 +69,13 @@ pub fn log_id(ec_id: &str) -> String { format!("{prefix}\u{2026}") } +use std::sync::Arc; + use cookie::CookieJar; use edgezero_core::body::Body as EdgeBody; use error_stack::Report; -use http::Request; +use http::{Request, header}; +use url::Url; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE}; @@ -80,11 +83,14 @@ use crate::cookies::handle_request_cookies; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; use crate::geo::GeoInfo; +use crate::http_util::{is_document_navigation, is_prefetch_request}; use crate::openrtb::Eid; use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; +use self::identity::request::{RequestIdentity, WriterDecision}; +use self::identity::runtime::IdentityRuntime; use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; use self::kv_types::KvEntry; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; @@ -284,6 +290,8 @@ pub struct EcContext { pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, + /// Identity state this request decides and collects. + identity: RequestIdentity, } impl EcContext { @@ -366,6 +374,7 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, + identity: RequestIdentity::default(), }) } @@ -515,6 +524,79 @@ impl EcContext { self.device_signals = Some(signals); } + /// The identity state this request decides and collects. + #[must_use] + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn identity(&self) -> &RequestIdentity { + &self.identity + } + + /// The identity state this request decides and collects, to fill. + // Temporary: first used by a later identity task. + #[allow(dead_code)] + pub(crate) fn identity_mut(&mut self) -> &mut RequestIdentity { + &mut self.identity + } + + /// Gives the request the identity runtime built for the settings load. + /// + /// The adapter calls this where it builds the request's EC state. + pub fn set_identity_runtime(&mut self, runtime: Arc) { + self.identity.set_runtime(runtime); + } + + /// The identity runtime, when the adapter set one. + #[must_use] + pub(crate) fn identity_runtime(&self) -> Option<&IdentityRuntime> { + self.identity.runtime().map(Arc::as_ref) + } + + /// Records whether the request passed the real-browser check, for a route + /// that is never the writer. + pub fn set_real_browser(&mut self, is_real_browser: bool) { + self.identity.set_real_browser(is_real_browser); + } + + /// Decides whether this request is the one browser-driven identity writer. + /// + /// Only an organic real-browser top-level document navigation that is not + /// a prefetch writes (spec 5.1). Also captures the user agent, `DNT` and + /// the page URL without query or fragment. + pub fn decide_writer( + &mut self, + req: &Request, + organic_route: bool, + is_real_browser: bool, + ) { + let writer = organic_route + && is_document_navigation(req) + && !is_prefetch_request(req) + && is_real_browser; + let user_agent = req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let dnt = req + .headers() + .get("dnt") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim() == "1"); + let page_url = Url::parse(&req.uri().to_string()).ok().map(|mut url| { + url.set_query(None); + url.set_fragment(None); + url + }); + self.identity.set_writer_decision(WriterDecision { + writer, + is_real_browser, + user_agent, + dnt, + page_url, + }); + } + /// Returns the device signals, if set. #[must_use] pub fn device_signals(&self) -> Option<&DeviceSignals> { @@ -667,6 +749,7 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + identity: RequestIdentity::default(), } } @@ -692,6 +775,7 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + identity: RequestIdentity::default(), } } @@ -720,6 +804,7 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + identity: RequestIdentity::default(), } } } @@ -775,6 +860,67 @@ mod tests { use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; + #[test] + fn decide_writer_needs_every_condition() { + let navigation = || { + Request::builder() + .uri("https://www.example.com/a?b=c#d") + .header("sec-fetch-dest", "document") + .header(header::USER_AGENT, "ExampleBrowser/1.0") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build the request") + }; + let mut context = EcContext::default(); + context.decide_writer(&navigation(), true, true); + assert!( + context.identity().writer(), + "should make an organic real-browser document navigation the writer" + ); + assert!( + context.identity().is_real_browser(), + "should record the real browser" + ); + assert_eq!( + context.identity().page_url().map(Url::as_str), + Some("https://www.example.com/a"), + "should drop the query and fragment" + ); + assert_eq!( + context.identity().user_agent(), + Some("ExampleBrowser/1.0"), + "should capture the user agent" + ); + assert!(context.identity().dnt(), "should capture DNT"); + + for (name, organic, browser, request) in [ + ("named route", false, true, navigation()), + ("bot", true, false, navigation()), + ( + "prefetch", + true, + true, + Request::builder() + .header("sec-fetch-dest", "document") + .header("sec-purpose", "prefetch") + .body(EdgeBody::empty()) + .expect("should build the request"), + ), + ] { + let mut context = EcContext::default(); + context.decide_writer(&request, organic, browser); + assert!( + !context.identity().writer(), + "should not be a writer: {name}" + ); + assert_eq!( + context.identity().is_real_browser(), + browser, + "should keep the real-browser flag: {name}" + ); + } + } + /// [`EcKvStore`] wrapper whose first `collisions` `Add` writes report a /// precondition failure, forcing generation to retry with a fresh suffix. struct AddCollidingEcKv { diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 457e503568..beea21e6e4 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -78,6 +78,31 @@ pub fn sanitize_forwarded_headers(req: &mut Request) { } } +/// Returns whether the request is a top-level document navigation. +/// +/// True only for `Sec-Fetch-Dest: document`. Unlike [`is_navigation_request`] +/// there is no `Accept` fallback, because `fetch()` can set `Accept: text/html`. +#[must_use] +pub fn is_document_navigation(req: &Request) -> bool { + req.headers() + .get("sec-fetch-dest") + .and_then(|v| v.to_str().ok()) + .is_some_and(|dest| dest.trim().eq_ignore_ascii_case("document")) +} + +/// Returns true when the request advertises itself as a prefetch via either +/// the standard `Sec-Purpose` or the legacy `Purpose` header. +#[must_use] +pub fn is_prefetch_request(req: &Request) -> bool { + let header = |name: &str| { + req.headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("prefetch")) + }; + header("sec-purpose") || header("purpose") +} + /// Returns `true` when the request looks like a top-level document navigation. /// /// Uses [`Sec-Fetch-Dest`] when available (preferred — it is a forbidden @@ -491,6 +516,38 @@ mod tests { use crate::settings::TrustedClientIpConfig; use http::{HeaderName, HeaderValue, Method}; + #[test] + fn document_navigation_requires_sec_fetch_dest_document() { + let with_dest = Request::builder() + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build the request"); + let accept_only = Request::builder() + .header(header::ACCEPT, "text/html") + .body(EdgeBody::empty()) + .expect("should build the request"); + let iframe = Request::builder() + .header("sec-fetch-dest", "iframe") + .body(EdgeBody::empty()) + .expect("should build the request"); + assert!( + is_document_navigation(&with_dest), + "should count a document destination" + ); + assert!( + !is_document_navigation(&accept_only), + "should not count the Accept fallback" + ); + assert!( + is_navigation_request(&accept_only), + "control: the broad check should still accept it" + ); + assert!( + !is_document_navigation(&iframe), + "should not count an iframe" + ); + } + fn build_request(method: Method, uri: &str) -> Request { Request::builder() .method(method) diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index 1447a83586..e31890f982 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -271,7 +271,7 @@ pub fn prepare_request( let cookie_state = console_cookie_state(request); let eligible_navigation = request.method() == Method::GET && is_navigation_request(request) - && !crate::publisher::is_prefetch_request(request) + && !crate::http_util::is_prefetch_request(request) && !crate::publisher::is_bot_user_agent(request); sanitize_console_cookie(request); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d3c09cd7d7..0237746a12 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -806,6 +806,8 @@ pub struct ProxyDispatchInput<'a> { pub kv: Option<&'a KvIdentityGraph>, pub ec_context: &'a mut EcContext, pub services: &'a RuntimeServices, + /// Whether the client looks like a real browser. EC generation needs it. + pub is_real_browser: bool, pub req: Request, } @@ -1039,19 +1041,21 @@ impl IntegrationRegistry { kv, ec_context, services, + is_real_browser, mut req, } = input; if let Some((proxy, _)) = self.find_route(method, path) { // Organic proxy handler: generate if needed (best effort). - // Only generate for document navigations — subresource requests - // may lack consent signals such as the Sec-GPC header. - if is_navigation_request(&req) { + // Only generate for document navigations by real browsers — + // subresource requests may lack consent signals such as the + // Sec-GPC header. + if is_real_browser && is_navigation_request(&req) { if let Err(err) = ec_context.generate_if_needed(settings, kv) { log::warn!("EC generation failed for integration proxy: {err:?}"); } } else { log::debug!( - "EC generation skipped for integration proxy: non-document request (path={path})", + "EC generation skipped for integration proxy: not a real-browser document request (path={path})", ); } @@ -1690,6 +1694,7 @@ mod tests { kv: None, ec_context: &mut ec_context, services: &noop_services(), + is_real_browser: true, req, })) .expect("should match route") @@ -2047,6 +2052,7 @@ mod tests { kv: None, ec_context: &mut ec_context, services: &services, + is_real_browser: true, req, })); @@ -2097,6 +2103,7 @@ mod tests { kv: None, ec_context: &mut ec_context, services: &services, + is_real_browser: true, req, })) .expect("should handle proxy request"); @@ -2146,6 +2153,7 @@ mod tests { kv: None, ec_context: &mut ec_context, services: &services, + is_real_browser: true, req, })) .expect("should handle proxy request"); @@ -2192,6 +2200,7 @@ mod tests { kv: None, ec_context: &mut ec_context, services: &services, + is_real_browser: true, req, })); diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index d212fee216..720e8b7ca0 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -122,10 +122,18 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { "ec/identity/removal.rs", include_str!("ec/identity/removal.rs"), ), + ( + "ec/identity/request.rs", + include_str!("ec/identity/request.rs"), + ), ( "ec/identity/resolution.rs", include_str!("ec/identity/resolution.rs"), ), + ( + "ec/identity/runtime.rs", + include_str!("ec/identity/runtime.rs"), + ), ( "ec/identity/servable.rs", include_str!("ec/identity/servable.rs"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1379684024..cc4ee6abc1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -72,7 +72,9 @@ use crate::ec::registry::PartnerRegistry; use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; use crate::html_processor::BodyCloseInjection; -use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; +use crate::http_util::{ + RequestInfo, is_navigation_request, is_prefetch_request, serve_static_with_etag, +}; use crate::integrations::IntegrationRegistry; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, @@ -3201,18 +3203,6 @@ pub(crate) fn is_bot_user_agent(req: &Request) -> bool { .any(|frag| ua.contains(frag)) } -/// Returns true when the request advertises itself as a prefetch via either -/// the standard `Sec-Purpose` or the legacy `Purpose` header. -pub(crate) fn is_prefetch_request(req: &Request) -> bool { - let header = |name: &str| { - req.headers() - .get(name) - .and_then(|v| v.to_str().ok()) - .is_some_and(|v| v.contains("prefetch")) - }; - header("sec-purpose") || header("purpose") -} - /// Returns whether request-scoped signals permit an ad-eligible navigation. /// /// This is the request half of the shared ad-stack gate: the configuration From cc068bd19b43c9432950941d859de2fc7e2aacd2 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 22:01:12 +0530 Subject: [PATCH 33/89] Test the real-browser gate on proxy EC generation Also log only the error context when the identity runtime cannot be rebuilt, and drop orphaned temporary comments. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 5 +- .../src/ec/identity/catalog.rs | 2 - .../src/ec/identity/revision.rs | 1 - .../src/integrations/registry.rs | 55 +++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index c4ca8662c7..8c7ecff2a0 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -531,7 +531,10 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re match build_identity_runtime(&settings) { Ok(identity) => ec_state.ec_context.set_identity_runtime(identity), Err(e) => { - log::warn!("identity runtime unavailable after settings reload: {e:?}"); + log::warn!( + "identity runtime unavailable after settings reload: {}", + e.current_context() + ); } } match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { diff --git a/crates/trusted-server-core/src/ec/identity/catalog.rs b/crates/trusted-server-core/src/ec/identity/catalog.rs index fed9a7071b..07dbc66fb0 100644 --- a/crates/trusted-server-core/src/ec/identity/catalog.rs +++ b/crates/trusted-server-core/src/ec/identity/catalog.rs @@ -111,7 +111,6 @@ impl ProviderCatalog { /// /// Returns [`TrustedServerError::Configuration`] when a pull partner /// cannot be described or the catalog rules fail. - // Temporary: first used by a later identity task. pub(crate) fn from_settings(settings: &Settings) -> Result> { let providers = settings .ec @@ -169,7 +168,6 @@ impl ProviderCatalog { /// (spec 10.1). Config validation has rejected a source a descriptor /// does not declare and a narrowing to nothing. #[must_use] - // Temporary: first used by a later identity task. pub(crate) fn with_narrowing( mut self, narrowing: BTreeMap>, diff --git a/crates/trusted-server-core/src/ec/identity/revision.rs b/crates/trusted-server-core/src/ec/identity/revision.rs index 6690ba9695..f5f37b618e 100644 --- a/crates/trusted-server-core/src/ec/identity/revision.rs +++ b/crates/trusted-server-core/src/ec/identity/revision.rs @@ -29,7 +29,6 @@ impl core::fmt::Debug for IdentityKeys { impl IdentityKeys { /// Derives the visitor key from `ec.passphrase`. #[must_use] - // Temporary: first used by a later identity task. pub(crate) fn from_settings(settings: &Settings) -> Self { let passphrase = settings.ec.passphrase.expose().as_bytes(); let mut mac = diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 0237746a12..001ba97c7b 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -2069,6 +2069,61 @@ mod tests { ); } + #[test] + fn handle_proxy_generates_an_ec_only_for_a_real_browser() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::from_routes(vec![( + Method::GET, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]); + let services = noop_services(); + let mut minted = Vec::new(); + + for is_real_browser in [false, true] { + let req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/ec") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..crate::consent::ConsentContext::default() + }, + Some("192.0.2.1".to_owned()), + ); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: None, + ec_context: &mut ec_context, + services: &services, + is_real_browser, + req, + })); + + assert!( + result.is_some_and(|response| response.is_ok()), + "should handle the request" + ); + minted.push(ec_context.ec_value().is_some()); + } + + assert_eq!( + minted, + [false, true], + "should mint an EC only when the client is a real browser" + ); + } + #[test] fn handle_proxy_rejects_invalid_ec_request_header() { let settings = create_test_settings(); From 33322d0398ff50a3e7e478b8d8ab80d5b6502778 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 22:05:13 +0530 Subject: [PATCH 34/89] Stop writing cluster size from /identify Pins the response shape for multi-entry records: the first UID with the partner's atype, as for a one-UID record. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/identify.rs | 111 +++++++++++++++--- crates/trusted-server-core/src/ec/kv.rs | 15 ++- 2 files changed, 109 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index d050449ac5..587fb91320 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -96,11 +96,10 @@ pub fn handle_identify( let mut cluster_size: Option = None; match kv.get(ec_id) { - Ok(Some((entry, generation))) => { + Ok(Some((entry, _generation))) => { if !entry.consent.ok { // Tombstone entries preserve the withdrawal signal for 24 hours. - // Do not extract IDs or evaluate cluster size because that would - // write back with the live-entry TTL. + // Do not extract IDs or evaluate cluster size. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { // Extract only this partner's UID, and only when the shared @@ -141,16 +140,25 @@ pub fn handle_identify( } } - // Evaluate cluster size lazily for identify responses. Existing - // stored cluster_size values are reused without a prefix-list call. - match kv.evaluate_cluster(ec_id, &entry, generation) { - Ok(size) => { - cluster_size = size; - } - Err(err) => { - log::warn!("Cluster evaluation failed for '{}': {err:?}", log_id(ec_id)); - } - } + // Reuse a stored cluster_size; otherwise count without writing. + // The writer stores the size, never this read path. + cluster_size = match entry + .network + .as_ref() + .and_then(|network| network.cluster_size) + { + Some(size) => Some(size), + None => match kv.compute_cluster_size(ec_id) { + Ok(size) => Some(size), + Err(err) => { + log::warn!( + "Cluster evaluation failed for '{}': {err:?}", + log_id(ec_id) + ); + None + } + }, + }; } } Ok(None) => {} @@ -376,7 +384,7 @@ mod tests { use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; use crate::ec::identity::test_support::{example_entry_set, full_record}; - use crate::ec::kv_types::KvEntry; + use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -683,8 +691,20 @@ mod tests { } fn identify_body(kv: &KvIdentityGraph, ec_id: &str) -> serde_json::Value { + identify_body_for( + kv, + ec_id, + make_test_partner("ssp.example.com", VALID_API_TOKEN), + ) + } + + fn identify_body_for( + kv: &KvIdentityGraph, + ec_id: &str, + partner: EcPartner, + ) -> serde_json::Value { let settings = create_test_settings(); - let partners = vec![make_test_partner("ssp.example.com", VALID_API_TOKEN)]; + let partners = vec![partner]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let req = Request::builder() .method("GET") @@ -737,6 +757,67 @@ mod tests { assert_eq!(body["uid"], "live-uid", "should return the unexpired ID"); } + #[test] + fn identify_computes_cluster_size_without_writing() { + let (kv, store) = KvIdentityGraph::write_counting("identify-store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = KvEntry::minimal("ssp.example.com", "example-uid", 1); + assert!( + entry.network.is_none(), + "should seed a row without a stored cluster size" + ); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + store.reset_counts(); + + let body = identify_body(&kv, &ec_id); + + assert!(body["cluster_size"].is_number(), "should report a size"); + assert_eq!(store.writes().total(), 0, "should not write"); + } + + #[test] + fn identify_returns_the_first_uid_of_a_multi_entry_record_with_the_partner_atype() { + let (kv, _store) = KvIdentityGraph::write_counting("identify-store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); + let set = [&["example-a", "example-b"][..], &["example-c"][..]] + .iter() + .enumerate() + .map(|(index, ids)| KvEid { + uids: ids + .iter() + .map(|id| KvUid { + id: (*id).to_owned(), + atype: Some(1), + ext: None, + }) + .collect(), + inserter: (index == 0).then(|| "example.com".to_owned()), + ..KvEid::default() + }) + .collect(); + entry + .ids + .insert("partner.example.com".to_owned(), full_record(None, set)); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + let mut partner = make_test_partner("partner.example.com", VALID_API_TOKEN); + partner.openrtb_atype = 3; + partner.bidstream_enabled = true; + + let body = identify_body_for(&kv, &ec_id, partner); + + assert_eq!( + body["uid"], + serde_json::json!("example-a"), + "should return the first UID of the first entry" + ); + assert_eq!( + body["eid"], + serde_json::json!({ "source": "partner.example.com", "uids": [{ "id": "example-a", "atype": 3 }] }), + "should return one UID with the partner's openrtb_atype, as for a one-UID record" + ); + } + #[test] fn handle_identify_returns_one_uid_for_a_full_record() { let ec_id = format!("{}.ABC123", "a".repeat(64)); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index d5cb672d21..f313b0fd88 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -1516,6 +1516,18 @@ impl KvIdentityGraph { .count_keys_with_prefix(hash_prefix, CLUSTER_LIST_LIMIT) } + /// Counts the entries sharing `ec_id`'s hash prefix, without writing. + /// + /// The prefix list and count only. A caller that wants the size kept + /// stages it on a write of its own. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store or list failure. + pub fn compute_cluster_size(&self, ec_id: &str) -> Result> { + self.count_hash_prefix_keys(ec_hash(ec_id)) + } + /// Evaluates the cluster size for an EC entry. /// /// Returns the stored `cluster_size` when it has already been evaluated @@ -1550,8 +1562,7 @@ impl KvIdentityGraph { } // Compute cluster size via prefix list. - let hash_prefix = ec_hash(ec_id); - let cluster_size = self.count_hash_prefix_keys(hash_prefix)?; + let cluster_size = self.compute_cluster_size(ec_id)?; log::debug!( "evaluate_cluster: computed cluster_size={cluster_size} for '{}'", From 405abe9c7187a67371062c113dfc26c3cde75e2c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 22:18:17 +0530 Subject: [PATCH 35/89] Queue browser ingest in the writer's mutation and stop stager writes Returning-visitor finalize no longer writes KV. The writer navigation adds ingest from ts-eids and sharedId to its single mutation; /auction, /_ts/page-bids and other stagers ingest nothing. A new visitor's create, from the publisher fallback or an integration proxy, and an orphan rotation carry browser ingest, so each stays one write. Browser ingest still writes one UID per registered source. The 'EID sync measurement' log line no longer runs for returning visitors (it stays for the new-EC create); the identity counters of the Part 1 metrics task replace it. Operator dashboards may read that line. Tests updated to assert no write plus a queued mutation on the writer: finalize_navigation_routes_never_write_returning_user_eid_updates_directly, finalize_auction_stale_miss_neither_ingests_nor_writes, finalize_ignores_auction_body_eids_as_ingest, finalize_generated_ec_writes_no_ingest_of_its_own, generate_if_needed_seeds_the_ts_eids_cookie, finalize_withholds_eid_kv_writes_when_purpose_four_is_denied, finalize_recovers_orphaned_ec_when_purpose_four_denial_empties_updates, and in auction/endpoints.rs the snapshot-reuse, body-EID, disabled-auction and navigation ingestion tests. Signed-off-by: dhruv8sh --- crates/trusted-server-adapter-axum/src/app.rs | 2 + .../src/app.rs | 2 + .../trusted-server-adapter-fastly/src/app.rs | 34 +- crates/trusted-server-adapter-spin/src/app.rs | 2 + .../src/auction/endpoints.rs | 105 +- crates/trusted-server-core/src/ec/finalize.rs | 909 ++++++++++++++---- .../src/ec/identity/mutation.rs | 10 +- crates/trusted-server-core/src/ec/kv.rs | 10 + crates/trusted-server-core/src/ec/mod.rs | 21 +- .../src/integrations/registry.rs | 98 +- 10 files changed, 957 insertions(+), 236 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index af653b7e0d..7c5051c3db 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -18,6 +18,7 @@ use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported, deny_admin_diagnostic_fallback, handle_admin_eids_lookup, }; +use trusted_server_core::ec::finalize::BrowserSeed; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -233,6 +234,7 @@ async fn dispatch_fallback( ec_context: &mut ec_context, services, is_real_browser: false, + seed: &BrowserSeed::empty(), req, }) .await diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 90b3bf536e..f4278f3e50 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -22,6 +22,7 @@ use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, deny_admin_diagnostic_fallback, handle_admin_eids_lookup, }; +use trusted_server_core::ec::finalize::BrowserSeed; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; @@ -545,6 +546,7 @@ fn build_router(state: &Arc) -> RouterService { ec_context: &mut ec_context, services: &services, is_real_browser: false, + seed: &BrowserSeed::empty(), req, }) .await diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 0d071408c8..2894f7afdd 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -112,6 +112,7 @@ use trusted_server_core::ec::admin::{ use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::finalize::{BrowserSeed, browser_seed}; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::identity::IdentityRuntime; use trusted_server_core::ec::kv::KvIdentityGraph; @@ -383,6 +384,30 @@ impl EcRequestState { } } +/// Collects the request's browser ingest for a new EC's one create. +/// +/// An unreadable partner registry yields an empty seed: the create still +/// happens, without the ingest. +fn browser_seed_for(state: &AppState, ec: &EcRequestState) -> BrowserSeed { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()); + match PartnerRegistry::from_config(&state.settings.ec.partners) { + Ok(registry) => browser_seed( + &state.settings, + ®istry, + ec.ec_context.consent(), + ec.eids_cookie.as_deref(), + ec.sharedid_cookie.as_deref(), + now, + ), + Err(err) => { + log::warn!("Browser ingest skipped: partner registry is unavailable: {err:?}"); + BrowserSeed::empty() + } + } +} + /// Derives device signals from the request's `User-Agent` header. /// /// Used as the fallback when no entry-point-captured [`DeviceSignals`] are @@ -835,6 +860,7 @@ async fn dispatch_fallback( // publisher.max_buffered_body_bytes. Publisher fallback below uses the // publisher-specific streaming finalizer instead. ec.ec_context.decide_writer(&req, true, ec.is_real_browser); + let seed = browser_seed_for(state, &ec); state .registry .handle_proxy(ProxyDispatchInput { @@ -845,6 +871,7 @@ async fn dispatch_fallback( ec_context: &mut ec.ec_context, services, is_real_browser: ec.is_real_browser, + seed: &seed, req, }) .await @@ -875,10 +902,11 @@ async fn dispatch_fallback( ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); } let is_publisher_navigation = ec.is_real_browser && is_navigation; + let seed = browser_seed_for(state, &ec); if is_publisher_navigation - && let Err(err) = ec - .ec_context - .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) + && let Err(err) = + ec.ec_context + .generate_if_needed(&state.settings, ec.kv_graph.as_ref(), &seed) { log::warn!("EC generation failed for publisher proxy: {err:?}"); } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9c309d9319..27e40fdb71 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -24,6 +24,7 @@ use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, deny_admin_diagnostic_fallback, handle_admin_eids_lookup, }; +use trusted_server_core::ec::finalize::BrowserSeed; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; @@ -826,6 +827,7 @@ fn build_router(state: &Arc) -> RouterService { ec_context: &mut ec_context, services: &services, is_real_browser: false, + seed: &BrowserSeed::empty(), req, }) .await diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index a2d63440af..152dc06044 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -743,14 +743,13 @@ mod tests { 1, "finalization must reuse the endpoint snapshot instead of reading again" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("row should exist"); - assert_eq!( - stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), - Some("shared-cookie-id"), - "the sharedId update must still be ingested from the shared snapshot" + assert!( + stored_ids(&graph, &ec_id).is_empty(), + "/auction is a stager and must not write the sharedId" + ); + assert!( + ec_context.identity().mutation().is_empty(), + "/auction is a stager and must not queue the sharedId" ); } @@ -799,7 +798,7 @@ mod tests { } #[tokio::test] - async fn auction_ingests_its_body_eids_and_not_a_cookie_only_source() { + async fn auction_ingests_nothing_from_its_body_or_the_cookie() { let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -851,15 +850,18 @@ mod tests { &mut response, ); - assert_eq!( - stored_ids(&graph, &ec_id), - vec![("body.example.com".to_owned(), "body-uid".to_owned())], - "KV should learn the body's source and not the cookie-only source" + assert!( + stored_ids(&graph, &ec_id).is_empty(), + "KV should learn nothing from an /auction" + ); + assert!( + ec_context.identity().mutation().is_empty(), + "/auction should queue nothing" ); } #[tokio::test] - async fn disabled_auction_ingests_nothing_from_the_cookie_but_keeps_shared_id() { + async fn disabled_auction_ingests_nothing() { let settings = create_test_settings(); let orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: false, @@ -899,23 +901,35 @@ mod tests { &mut response, ); - assert_eq!( - stored_ids(&graph, &ec_id), - vec![("sharedid.org".to_owned(), "shared-cookie-id".to_owned())], - "an /auction that resolved no client set should ingest only the sharedId cookie" + assert!( + stored_ids(&graph, &ec_id).is_empty(), + "a disabled /auction should ingest nothing" + ); + assert!( + ec_context.identity().mutation().is_empty(), + "a disabled /auction should queue nothing" ); } #[test] - fn navigation_ingests_the_cookie_only_source() { + fn writer_navigation_queues_the_cookie_only_source() { // Control for the two tests above: the registry does match the - // cookie-only source when the route reads the cookie. + // cookie-only source when the writer navigation reads the cookie. let settings = create_test_settings(); let registry = ingestion_registry("body.example.com"); let ec_id = format!("{}.ingnav", "a".repeat(64)); let graph = seeded_graph(&ec_id); let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); ec_context.set_eid_sync_source(crate::ec::EidSyncSource::Navigation); + ec_context.identity_mut().set_writer_decision( + crate::ec::identity::request::WriterDecision { + writer: true, + is_real_browser: true, + user_agent: None, + dnt: false, + page_url: None, + }, + ); let cookie = cookie_only_ts_eids(); let mut response = http::Response::new(EdgeBody::empty()); @@ -929,21 +943,26 @@ mod tests { &mut response, ); - assert_eq!( - stored_ids(&graph, &ec_id), - vec![(COOKIE_ONLY_SOURCE.to_owned(), "cookie-uid".to_owned())], - "a navigation should ingest the cookie's registered source" + assert!( + stored_ids(&graph, &ec_id).is_empty(), + "a navigation should leave the write to phase 4" + ); + let queued = ec_context.identity().mutation().positive_changes(); + assert!( + matches!( + queued, + [crate::ec::identity::mutation::PartnerIdChange::FillMissing { source, record }] + if source.as_str() == COOKIE_ONLY_SOURCE + && record.first_uid() == Some("cookie-uid") + ), + "a navigation should queue the cookie's registered source" ); } #[tokio::test] - async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { - // Regression test for #1184: `/auction` sends every EID it has in the - // request body, but response finalization used to ingest identity - // graph updates only from the `ts-eids` cookie — which the browser - // caps in size and may not have sent at all. `handle_auction` must - // hand its parsed body EIDs to `ec_context` so finalization ingests - // them regardless of what the cookie carried. + async fn auction_body_eids_never_reach_kv() { + // The `/auction` body reaches bidders but is not an ingest input + // (spec G10): finalization neither writes nor queues it. let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -998,15 +1017,6 @@ mod tests { ) .await; - assert_eq!( - ec_context.client_eids().map(|eids| eids - .iter() - .map(|eid| eid.source.as_str()) - .collect::>()), - Some(vec!["id5-sync.com"]), - "the endpoint must hand its parsed body EIDs to the request context" - ); - let mut response = http::Response::new(EdgeBody::empty()); crate::ec::finalize::ec_finalize_response( &settings, @@ -1018,14 +1028,13 @@ mod tests { &mut response, ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("row should exist"); - assert_eq!( - stored.ids.get("id5-sync.com").and_then(|id| id.first_uid()), - Some("ID5_from_body"), - "the body's EID must be ingested into KV without a ts-eids cookie" + assert!( + stored_ids(&graph, &ec_id).is_empty(), + "the body's EID must not be ingested into KV" + ); + assert!( + ec_context.identity().mutation().is_empty(), + "the body's EID must not be queued" ); } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index c20adbf59b..edf1862bcf 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -10,17 +10,18 @@ use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; use crate::consent::{ConsentContext, allows_eid_persistence}; -use crate::openrtb::Eid; use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; use super::generation::{generate_ec_id, is_valid_ec_id}; +use super::identity::ids::SourceDomain; +use super::identity::mutation::{IdentityMutation, PartnerIdChange}; use super::kv::{ CreateIfAbsentOutcome, EidCookieSyncOutcome, KvIdentityGraph, PartnerIdUpdate, apply_partner_id_updates, }; -use super::kv_types::KvEntry; +use super::kv_types::{KvEntry, KvPartnerId}; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; @@ -36,16 +37,20 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// Finalizes EC response behavior for all routes. /// -/// Applies withdrawal handling, last-seen updates, cookie reconciliation, -/// Prebid EID ingestion, and cookie writes for new EC generation. +/// Applies withdrawal handling, cookie reconciliation, orphan recovery and the +/// `ts-ec` cookie for a new EC. +/// +/// A returning visitor's response writes no KV row. On the writer request the +/// browser ingest from `ts-eids` and sharedId joins the request's identity +/// mutation, which phase 4 flushes; every other request ingests nothing. A new +/// EC's ingest rode in its create ([`BrowserSeed`]). /// /// On consent withdrawal, the browser response clears the EC cookie /// immediately and the EC identity-graph KV tombstone is the authoritative /// revocation marker. There is no separate consent KV store to clean up. /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted -/// from the request *before* routing consumes it. `/auction` responses ignore -/// it in favor of the request's own client set. +/// from the request *before* routing consumes it. pub fn ec_finalize_response( settings: &Settings, ec_context: &mut EcContext, @@ -80,27 +85,23 @@ pub fn ec_finalize_response( // Returning user: consent is granted and EC came from request. if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { - let source = ec_context.eid_sync_source(); - let collected = source - .map(|_| { - let (cookie_input, client_input) = browser_eid_input(ec_context, eids_cookie); - collect_eid_updates( - cookie_input, - sharedid_cookie, - client_input, - registry, - checked_current_timestamp(), - ) - }) - .unwrap_or_default(); + // Only the writer request ingests browser data (spec 5.3). Every + // other request leaves `ts-eids` and sharedId for the writer. + let writer = ec_context.identity().writer(); + let collected = if writer { + collect_eid_updates( + eids_cookie, + sharedid_cookie, + None, + registry, + checked_current_timestamp(), + ) + } else { + Vec::new() + }; let had_eid_updates = !collected.is_empty(); let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); - // `sync_eid_cookie_updates` early-returns without refreshing the - // snapshot when `updates` is empty, which is correct when there - // was never anything to write (an unconfigured registry, no sync - // source, or no EID cookies/body this request — the case - // `finalize_not_read_snapshot_does_not_rotate` covers). But when - // consent gating (e.g. TCF Purpose 4 denial) is what emptied + // When consent gating (e.g. TCF Purpose 4 denial) is what emptied // `updates`, the request *did* have EID data, and orphan recovery // below still needs an actual `Missing` read to detect an // orphaned cookie. An unread `NotRead` snapshot proves nothing, so @@ -113,8 +114,6 @@ pub fn ec_finalize_response( { let snapshot = graph.load_snapshot(&ec_id); ec_context.set_kv_snapshot(snapshot); - } else if let Some(source) = source { - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, source); } if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) && ec_context.recovery_eligible() @@ -122,6 +121,8 @@ pub fn ec_finalize_response( confirm_then_recover_orphaned_ec( settings, ec_context, graph, &ec_id, &updates, response, ); + } else { + queue_browser_ingest(ec_context, &ec_id, &updates); } } @@ -136,25 +137,28 @@ pub fn ec_finalize_response( // there is no KV graph: that would mint a browser cookie with no backing // identity-graph row, producing a phantom ID on later requests. if ec_context.ec_generated() { - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { + let (Some(_), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { log::info!("Skipping generated EC response write because KV graph is unavailable"); reconcile_pull_sync_marker(settings, registry, ec_context, response); return; }; - let (cookie_input, client_input) = browser_eid_input(ec_context, eids_cookie); - let updates = gate_eid_updates_by_consent( - collect_eid_updates( - cookie_input, - sharedid_cookie, - client_input, - registry, - checked_current_timestamp(), - ), - ec_context.consent(), - ); - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); - if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { + // The create in `generate_if_needed` (or the integration proxy's) + // already carried the browser ingest, so finalize writes nothing. + if let Some(entry) = ec_context.kv_snapshot().entry_for(&ec_id) { + let seeded = gate_eid_updates_by_consent( + collect_eid_updates( + eids_cookie, + sharedid_cookie, + None, + registry, + checked_current_timestamp(), + ), + ec_context.consent(), + ); + if !seeded.is_empty() && partner_id_updates_stored(entry, &seeded) { + record_eid_sync_terminal(EidSyncSource::NewEc, EidCookieSyncOutcome::Written); + } set_ec_cookie_on_response(settings, ec_context, response); } else { log::warn!("Skipping generated EC cookie because backing row is not authoritative"); @@ -164,41 +168,92 @@ pub fn ec_finalize_response( reconcile_pull_sync_marker(settings, registry, ec_context, response); } -/// Chooses the browser EID input KV ingestion reads for this request. +/// The browser ingest of one request, ready to seed a new identity row. /// -/// `/auction` ingests only the client set the auction itself used (its body, -/// else its `ts-eids` cookie, as `handle_auction` resolved it). A cookie from -/// an earlier page is then never ingested next to a scoped body, and an -/// `/auction` that resolved no client set ingests nothing from the cookie. -/// Every other route, and requests with no sync source, read the cookie. -fn browser_eid_input<'a>( - ec_context: &'a EcContext, - eids_cookie: Option<&'a str>, -) -> (Option<&'a str>, Option<&'a [Eid]>) { - match ec_context.eid_sync_source() { - Some(EidSyncSource::Auction) => (None, ec_context.client_eids()), - _ => (eids_cookie, None), +/// Opaque on purpose: it holds UIDs, so it prints only a count. Build it with +/// [`browser_seed`], or use [`BrowserSeed::empty`] when there is nothing to seed. +#[derive(Clone, Default)] +pub struct BrowserSeed(Vec); + +impl BrowserSeed { + /// A seed that carries nothing. + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + /// The updates to apply to the new row. + pub(crate) fn updates(&self) -> &[PartnerIdUpdate] { + &self.0 } } -fn sync_eid_cookie_updates( - graph: &KvIdentityGraph, - ec_context: &mut EcContext, - ec_id: &str, - updates: &[PartnerIdUpdate], - source: EidSyncSource, -) { - if updates.is_empty() { - return; +impl core::fmt::Debug for BrowserSeed { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("BrowserSeed") + .field("updates", &self.0.len()) + .finish() } +} - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - ec_id, - updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); - record_eid_sync_terminal(source, outcome); +/// Collects the consent-gated browser ingest from `ts-eids` and sharedId. +/// +/// One UID per registered source: the `/auction` body is not an ingest input. +/// `now` is the acquisition time stored on each record. +#[must_use] +pub fn browser_seed( + _settings: &Settings, + registry: &PartnerRegistry, + consent: &ConsentContext, + eids_cookie: Option<&str>, + sharedid_cookie: Option<&str>, + now: u64, +) -> BrowserSeed { + BrowserSeed(gate_eid_updates_by_consent( + collect_eid_updates(eids_cookie, sharedid_cookie, None, registry, Some(now)), + consent, + )) +} + +/// Whether `entry` already holds every update's UID. +fn partner_id_updates_stored(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { + updates.iter().all(|update| { + entry + .ids + .get(&update.partner_id) + .is_some_and(|stored| stored.first_uid() == Some(update.uid.as_str())) + }) +} + +/// Adds the browser ingest to the writer's mutation, for phase 4 to flush. +/// +/// Nothing is queued against a withdrawn, missing or failed row. A `NotRead` +/// snapshot still queues: phase 4 re-reads before it writes. +fn queue_browser_ingest(ec_context: &mut EcContext, ec_id: &str, updates: &[PartnerIdUpdate]) { + if updates.is_empty() || !ec_context.identity().writer() { + return; + } + let usable = match ec_context.kv_snapshot() { + EcKvSnapshot::NotRead => true, + snapshot => snapshot + .entry_for(ec_id) + .is_some_and(|entry| entry.consent.ok), + }; + if !usable { + return; + } + let mutation = ec_context.identity_mut().mutation_mut(); + for update in updates { + let Ok(source) = update.partner_id.parse::() else { + log::warn!("Browser ingest skipped a source that is not a valid domain"); + continue; + }; + mutation.add_change(PartnerIdChange::FillMissing { + source, + record: KvPartnerId::browser(update.uid.as_str(), update.acquired_at), + }); + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -342,6 +397,9 @@ fn recover_orphaned_ec( generation: None, }; ec_context.replace_with_generated(ec_id, snapshot); + // The create carried this request's ingest: it is the page + // load's one write, so phase 4 has nothing left to flush. + *ec_context.identity_mut().mutation_mut() = IdentityMutation::default(); set_ec_cookie_on_response(settings, ec_context, response); return; } @@ -392,9 +450,9 @@ fn confirm_then_recover_orphaned_ec( match confirmed { EcKvSnapshot::Present { .. } => { // The row became visible after the origin round trip: adopt it and - // merge any pending updates rather than rotating a valid identity. + // queue the ingest rather than rotating a valid identity. ec_context.set_kv_snapshot(confirmed); - sync_eid_cookie_updates(graph, ec_context, ec_id, updates, EidSyncSource::Navigation); + queue_browser_ingest(ec_context, ec_id, updates); } EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), @@ -554,6 +612,7 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::identity::request::WriterDecision; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -1137,7 +1196,7 @@ mod tests { Some("192.0.2.11".to_owned()), ); ec_context.set_recovery_eligible(true); - ec_context.set_eid_sync_source(EidSyncSource::Navigation); + set_writer(&mut ec_context, true); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1269,7 +1328,7 @@ mod tests { } #[test] - fn finalize_navigation_routes_persist_returning_user_eid_updates() { + fn finalize_navigation_routes_never_write_returning_user_eid_updates_directly() { for (source, suffix, cookie_id) in [ (EidSyncSource::Navigation, "naveid", "navigation-cookie-id"), (EidSyncSource::PageBids, "spaeid", "page-bids-cookie-id"), @@ -1288,6 +1347,7 @@ mod tests { .expect("should seed the live row"); let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); ec_context.set_eid_sync_source(source); + set_writer(&mut ec_context, true); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1306,31 +1366,27 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should remain"); + assert!( + !stored.ids.contains_key("sharedid.org"), + "{source} should leave the flush to phase 4" + ); assert_eq!( - stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), - Some(cookie_id), - "{source} should persist the returning-user EID cookie" + ec_context.identity().mutation().change_count(), + 1, + "{source} should queue the returning-user EID cookie" ); } } #[test] - fn finalize_generated_ec_without_sync_source_ingests_the_ts_eids_cookie() { + fn generate_if_needed_seeds_the_ts_eids_cookie() { let settings = create_test_settings(); - let ec_id = sample_ec_id("genck1"); let graph = KvIdentityGraph::in_memory("test_store"); - let live = KvEntry::new( - &granting_consent(), + let mut ec_context = EcContext::new_for_test_with_ip( None, - current_timestamp(), - &settings.publisher.domain, + granting_consent(), + Some("192.0.2.10".to_owned()), ); - graph - .create(&ec_id, &live) - .expect("should seed generated row"); - let mut ec_context = - make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); - ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); let registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) .expect("should build registry"); let eids_cookie = base64::engine::general_purpose::STANDARD.encode( @@ -1339,22 +1395,24 @@ mod tests { ])) .expect("should serialize test cookie payload"), ); - let mut response = empty_response(); - - ec_finalize_response( + let seed = browser_seed( &settings, - &mut ec_context, - Some(&graph), ®istry, + ec_context.consent(), Some(&eids_cookie), None, - &mut response, + current_timestamp(), ); + ec_context + .generate_if_needed(&settings, Some(&graph), &seed) + .expect("should generate an EC"); + + let ec_id = ec_context.ec_value().expect("should hold the new EC"); let (stored, _) = graph - .get(&ec_id) + .get(ec_id) .expect("should read store") - .expect("row should remain"); + .expect("row should exist"); assert_eq!( stored .ids @@ -1362,12 +1420,12 @@ mod tests { .map(|(source, id)| (source.as_str(), id.first_uid().unwrap_or_default())) .collect::>(), vec![("ids.example.com", "cookie-uid")], - "a new EC with no sync source should ingest the cookie" + "a new EC should carry the ts-eids cookie in its create" ); } #[test] - fn finalize_generated_ec_persists_eid_updates() { + fn finalize_generated_ec_writes_no_ingest_of_its_own() { let settings = create_test_settings(); let ec_id = sample_ec_id("geneid"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1401,9 +1459,13 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should remain"); - assert_eq!( - stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), - Some("generated-cookie-id") + assert!( + !stored.ids.contains_key("sharedid.org"), + "finalize should leave a new EC's ingest to the create" + ); + assert!( + get_header(&response, "set-cookie").is_some(), + "should still set the ts-ec cookie" ); } @@ -1504,10 +1566,9 @@ mod tests { } #[test] - fn finalize_auction_transient_miss_still_persists_eid_updates() { - // `/auction` saves its first lookup into the context and is never - // recovery eligible, so a stale miss there has no later chance to - // retry. Finalization must revalidate before dropping collected IDs. + fn finalize_auction_stale_miss_neither_ingests_nor_writes() { + // `/auction` is a stager: even with a stale endpoint miss in its + // snapshot it does not re-read, ingest or write. let settings = create_test_settings(); let ec_id = sample_ec_id("named1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1547,20 +1608,21 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should remain"); - assert_eq!( - stored.ids.get("sharedid.org").and_then(|id| id.first_uid()), - Some("shared-cookie-id"), - "a stale endpoint miss must not suppress EID persistence" + assert!( + !stored.ids.contains_key("sharedid.org"), + "a stager must not write ingest" + ); + assert!( + ec_context.identity().mutation().is_empty(), + "a stager must not queue ingest" ); } #[test] - fn finalize_persists_every_configured_partner_from_client_eids_over_a_trimmed_cookie() { - // Regression test for #1184: the `ts-eids` cookie only carries what - // the browser could fit under its size cap, but `/auction` also - // hands finalization the full EID set from the request body via - // `EcContext::set_client_eids`. That full set must land in KV even - // when the cookie alone would have dropped a configured partner. + fn finalize_ignores_auction_body_eids_as_ingest() { + // The `/auction` body is no longer an ingest input (spec G10): the + // writer navigation ingests `ts-eids` and sharedId only, so a body + // that the cookie's size cap trimmed reaches bidders but never KV. let settings = create_test_settings(); let ec_id = sample_ec_id("cleids1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1573,40 +1635,16 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row this request updates"); - let mut ec_context = returning_user_context( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - false, - ); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), false); ec_context.set_eid_sync_source(EidSyncSource::Auction); - // Only `id5-sync.com` "fit" in the (simulated) trimmed cookie; - // `liveramp.com` was dropped by the browser's size cap. - let eids_cookie = base64::engine::general_purpose::STANDARD.encode( - serde_json::to_vec(&serde_json::json!([ - {"source": "id5-sync.com", "uids": [{"id": "ID5_from_cookie", "atype": 1}]} - ])) - .expect("should serialize test cookie payload"), - ); - ec_context.set_client_eids(vec![ - Eid::new( - "id5-sync.com".to_owned(), - vec![Uid { - id: "ID5_from_body".to_owned(), - atype: Some(1), - ext: None, - }], - ), - Eid::new( - "liveramp.com".to_owned(), - vec![Uid { - id: "LR_from_body".to_owned(), - atype: Some(3), - ext: None, - }], - ), - ]); + ec_context.set_client_eids(vec![Eid::new( + "liveramp.com".to_owned(), + vec![Uid { + id: "LR_from_body".to_owned(), + atype: Some(3), + ext: None, + }], + )]); let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1616,7 +1654,7 @@ mod tests { &mut ec_context, Some(&graph), ®istry, - Some(&eids_cookie), + None, None, &mut response, ); @@ -1624,16 +1662,14 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should exist after ingestion"); - assert_eq!( - stored.ids.get("id5-sync.com").and_then(|id| id.first_uid()), - Some("ID5_from_body"), - "the request body's EID should win over the cookie's stale value" + .expect("row should remain"); + assert!( + stored.ids.is_empty(), + "a body EID must not be written to KV" ); - assert_eq!( - stored.ids.get("liveramp.com").and_then(|id| id.first_uid()), - Some("LR_from_body"), - "a partner the cookie trimmed must still be ingested from the body" + assert!( + ec_context.identity().mutation().is_empty(), + "a body EID must not be queued" ); } @@ -1664,18 +1700,14 @@ mod tests { false, purpose1_only_consent, ); - ec_context.set_kv_snapshot(EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }); - ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid::new( - "id5-sync.com".to_owned(), - vec![Uid { - id: "ID5_should_not_persist".to_owned(), - atype: Some(1), - ext: None, - }], - )]); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + set_writer(&mut ec_context, true); + let eids_cookie = base64::engine::general_purpose::STANDARD.encode( + serde_json::to_vec(&serde_json::json!([ + { "source": "id5-sync.com", "uids": [{ "id": "ID5_should_not_persist", "atype": 1 }] } + ])) + .expect("should serialize test cookie payload"), + ); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1685,7 +1717,7 @@ mod tests { &mut ec_context, Some(&graph), ®istry, - None, + Some(&eids_cookie), None, &mut response, ); @@ -1699,6 +1731,10 @@ mod tests { "denying TCF Purpose 4 must keep the EID out of KV even though Purpose 1 \ (EC) consent is granted" ); + assert!( + ec_context.identity().mutation().is_empty(), + "denying TCF Purpose 4 must keep the EID out of the mutation" + ); } #[test] @@ -2055,6 +2091,18 @@ mod tests { ec } + fn set_writer(ec_context: &mut EcContext, writer: bool) { + ec_context + .identity_mut() + .set_writer_decision(WriterDecision { + writer, + is_real_browser: writer, + user_agent: None, + dnt: false, + page_url: None, + }); + } + fn assert_did_not_rotate(ec_context: &EcContext, orphan: &str, response: &Response) { assert_eq!( ec_context.ec_value(), @@ -2492,6 +2540,523 @@ mod tests { ); } + mod identity_flow { + use serde_json::json; + + use super::*; + use crate::ec::identity::catalog::ProviderCatalog; + use crate::ec::identity::ids::SourceDomain; + use crate::ec::identity::input::SeededRandom; + use crate::ec::identity::mutation::{ + ApplyContext, IdentityMutation, PartnerIdChange, apply, + }; + use crate::ec::kv_types::KvPartnerId; + + const SHARED_ID_SOURCE: &str = "sharedid.org"; + + fn settings_with_partner(_source: &str) -> Settings { + create_test_settings() + } + + fn registry_with_partner(source: &str) -> PartnerRegistry { + PartnerRegistry::from_config(&[make_partner(source)]).expect("should build registry") + } + + fn ts_eids_cookie(value: &serde_json::Value) -> String { + base64::engine::general_purpose::STANDARD + .encode(serde_json::to_vec(value).expect("should serialize test cookie payload")) + } + + fn queued_records(mutation: &IdentityMutation) -> Vec<(SourceDomain, KvPartnerId)> { + mutation + .positive_changes() + .iter() + .filter_map(|change| match change { + PartnerIdChange::FillMissing { source, record } + | PartnerIdChange::OwnerWrite { source, record, .. } => { + Some((source.clone(), record.clone())) + } + PartnerIdChange::OwnerRemove { .. } | PartnerIdChange::WriterRemove { .. } => { + None + } + }) + .collect() + } + + fn set_cookie_names(response: &Response) -> Vec { + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .filter_map(|cookie| cookie.split('=').next()) + .map(str::to_owned) + .collect() + } + + fn html_response() -> Response { + Response::builder() + .status(200) + .header(http::header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(EdgeBody::empty()) + .expect("should build test response") + } + + fn seed_live_row(graph: &KvIdentityGraph, entry: &KvEntry) -> String { + let ec_id = sample_ec_id("idflow"); + graph + .create(&ec_id, entry) + .expect("should seed the live row"); + ec_id + } + + fn live_row() -> KvEntry { + KvEntry::minimal("other.example.com", "x", 1) + } + + fn returning_context(ec_id: &str, writer: bool) -> EcContext { + let mut context = returning_user_context(ec_id, EcKvSnapshot::NotRead, false); + set_writer(&mut context, writer); + context + } + + fn finalize_with_shared_id( + context: &mut EcContext, + graph: &KvIdentityGraph, + shared_id: Option<&str>, + ) -> Response { + let mut response = html_response(); + ec_finalize_response( + &settings_with_partner(SHARED_ID_SOURCE), + context, + Some(graph), + ®istry_with_partner(SHARED_ID_SOURCE), + None, + shared_id, + &mut response, + ); + response + } + + #[test] + fn returning_writer_navigation_adds_ingest_to_the_mutation_and_writes_nothing() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + store.reset_counts(); + let mut context = returning_context(&ec_id, true); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + + let response = finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert_eq!( + store.writes().total(), + 0, + "should not write in finalize for a returning writer" + ); + let records = queued_records(context.identity().mutation()); + assert_eq!(records.len(), 1, "should queue the sharedId fill"); + assert_eq!( + records[0].1.first_uid(), + Some("example-shared-id"), + "should queue the cookie's UID" + ); + assert!( + set_cookie_names(&response).is_empty(), + "should set no cookie on a returning HTML response" + ); + } + + #[test] + fn writer_ingest_applied_to_the_live_row_matches_the_old_direct_write() { + let (graph, _store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + let mut context = returning_context(&ec_id, true); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let eids_cookie = ts_eids_cookie(&json!([ + { "source": SHARED_ID_SOURCE, "uids": [{ "id": "example-prebid" }] } + ])); + let mut response = html_response(); + let now = checked_current_timestamp().expect("should read the clock"); + + ec_finalize_response( + &settings_with_partner(SHARED_ID_SOURCE), + &mut context, + Some(&graph), + ®istry_with_partner(SHARED_ID_SOURCE), + Some(&eids_cookie), + Some("example-shared-id"), + &mut response, + ); + + let live = live_row(); + let mut direct = live.clone(); + apply_partner_id_updates( + &mut direct, + &[PartnerIdUpdate::new(SHARED_ID_SOURCE, "example-shared-id") + .with_acquired_at(now)], + ); + let catalog = ProviderCatalog::empty(); + let random = SeededRandom::new(7); + let applied = apply( + context.identity().mutation(), + &live, + &ApplyContext { + now, + random: &random, + catalog: &catalog, + }, + ) + .expect("should apply to a live row"); + assert!(applied.changed, "should change the live row"); + assert_eq!( + applied.entry.ids.keys().collect::>(), + direct.ids.keys().collect::>(), + "should store the same sources as the old direct write" + ); + let staged = applied + .entry + .ids + .get(SHARED_ID_SOURCE) + .expect("should store the source"); + let written = direct + .ids + .get(SHARED_ID_SOURCE) + .expect("should store the source"); + assert_eq!( + staged.first_uid(), + written.first_uid(), + "should store the same UID" + ); + assert_eq!( + staged.at, written.at, + "should store the same acquisition time" + ); + } + + #[test] + fn golden_browser_ingest_stays_one_uid_whatever_the_browser_sent() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + store.reset_counts(); + let mut context = returning_context(&ec_id, true); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let eids_cookie = ts_eids_cookie(&json!([ + { + "source": "partner.example.com", + "uids": [ + { "id": "example-a", "atype": 1, "ext": { "stype": "ppuid" } }, + { "id": "example-b", "atype": 3 } + ], + "inserter": "example.com", + "mm": 3 + }, + { "source": "unregistered.example.com", "uids": [{ "id": "example-c", "atype": 1 }] } + ])); + let mut response = html_response(); + + ec_finalize_response( + &settings_with_partner("partner.example.com"), + &mut context, + Some(&graph), + ®istry_with_partner("partner.example.com"), + Some(&eids_cookie), + None, + &mut response, + ); + + assert_eq!(store.writes().total(), 0, "should not write in finalize"); + let records = queued_records(context.identity().mutation()); + assert_eq!( + records.len(), + 1, + "should queue only the registered source, as today" + ); + let (source, record) = &records[0]; + assert_eq!( + source.as_str(), + "partner.example.com", + "should queue the registered source" + ); + assert_eq!( + serde_json::to_value(record).expect("should serialize the record"), + json!({ "at": record.at, "eids": [{ "uids": [{ "id": "example-a" }] }] }), + "should store the UID collect_eid_updates picks today, with no atype, ext or EID-level field" + ); + } + + #[test] + fn stagers_ingest_nothing_and_write_nothing() { + for source in [ + Some(EidSyncSource::Auction), + Some(EidSyncSource::PageBids), + None, + ] { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + store.reset_counts(); + let mut context = returning_context(&ec_id, false); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + if let Some(source) = source { + context.set_eid_sync_source(source); + } + context.set_client_eids(vec![Eid::new( + SHARED_ID_SOURCE.to_owned(), + vec![Uid { + id: "example-body-id".to_owned(), + atype: None, + ext: None, + }], + )]); + + let response = + finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert_eq!( + store.writes().total(), + 0, + "{source:?} should not write in finalize" + ); + assert!( + context.identity().mutation().is_empty(), + "{source:?} should leave the mutation empty" + ); + assert!( + set_cookie_names(&response).is_empty(), + "{source:?} should set no cookie" + ); + } + } + + #[test] + fn unusable_consent_writes_nothing_and_queues_nothing() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + store.reset_counts(); + let mut context = make_context( + Some(&ec_id), + Some(&ec_id), + true, + false, + Jurisdiction::Unknown, + ); + set_writer(&mut context, true); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + + finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert_eq!(store.writes().total(), 0, "should not write"); + assert!( + context.identity().mutation().is_empty(), + "should queue nothing" + ); + } + + #[test] + fn withdrawal_still_writes_its_tombstone_on_any_request() { + for writer in [true, false] { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + store.reset_counts(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut context = + make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + set_writer(&mut context, writer); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + + finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert!( + store.writes().total() > 0, + "writer={writer} should write the tombstone" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("row should remain"); + assert!( + !stored.consent.ok, + "writer={writer} should tombstone the row" + ); + assert!( + context.identity().mutation().is_empty(), + "writer={writer} should queue nothing" + ); + } + } + + #[test] + fn tombstone_read_leaves_the_mutation_empty() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &KvEntry::tombstone(1000)); + store.reset_counts(); + let mut context = returning_context(&ec_id, true); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + + finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert_eq!(store.writes().total(), 0, "should not write"); + assert!( + context.identity().mutation().is_empty(), + "should leave the mutation empty" + ); + } + + #[test] + fn orphan_rotation_carries_browser_ingest_in_one_create() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let orphan = sample_ec_id("orphn2"); + let mut context = returning_context(&orphan, true); + context.set_recovery_eligible(true); + context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }); + + let response = finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + let replacement = context.ec_value().expect("should keep an EC ID").to_owned(); + assert_ne!(replacement, orphan, "should rotate the orphan"); + let writes = store.writes(); + assert_eq!(writes.add, 1, "should write one create"); + assert_eq!(writes.total(), 1, "should write nothing else"); + let (stored, _) = graph + .get(&replacement) + .expect("should read store") + .expect("should hold the replacement row"); + assert_eq!( + stored + .ids + .get(SHARED_ID_SOURCE) + .and_then(KvPartnerId::first_uid), + Some("example-shared-id"), + "should carry the sharedId as a one-UID record" + ); + assert!( + context.identity().mutation().is_empty(), + "should leave the mutation empty" + ); + assert_eq!( + set_cookie_names(&response), + vec!["ts-ec".to_owned()], + "should set only the replacement cookie" + ); + } + + #[test] + fn confirm_read_that_finds_the_row_queues_ingest_instead_of_writing() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = seed_live_row(&graph, &live_row()); + store.reset_counts(); + let mut context = returning_context(&ec_id, true); + context.set_recovery_eligible(true); + context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }); + + let response = finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert_eq!(store.writes().total(), 0, "should not write"); + assert_eq!( + context.ec_value(), + Some(ec_id.as_str()), + "should not rotate" + ); + assert_eq!( + queued_records(context.identity().mutation()).len(), + 1, + "should queue the ingest" + ); + assert!( + context.kv_snapshot().entry_for(&ec_id).is_some(), + "should adopt the confirmed row as the snapshot" + ); + assert!( + set_cookie_names(&response).is_empty(), + "should set no cookie" + ); + } + + #[test] + fn integration_proxy_writer_never_rotates_on_missing_row() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let orphan = sample_ec_id("orphn3"); + let mut context = returning_context(&orphan, true); + context.set_recovery_eligible(false); + context.set_kv_snapshot(EcKvSnapshot::Missing { + ec_id: orphan.clone(), + }); + + let response = finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); + + assert_eq!(store.writes().total(), 0, "should not create"); + assert_did_not_rotate(&context, &orphan, &response); + assert!( + context.identity().mutation().is_empty(), + "should queue nothing against a missing row" + ); + } + + #[test] + fn new_visitor_create_carries_browser_ingest_in_one_write() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let settings = create_test_settings(); + let registry = registry_with_partner(SHARED_ID_SOURCE); + let mut context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + let seed = browser_seed( + &settings, + ®istry, + context.consent(), + None, + Some("example-shared-id"), + 1_760_000_000, + ); + + context + .generate_if_needed(&settings, Some(&graph), &seed) + .expect("should generate an EC"); + let mut response = html_response(); + ec_finalize_response( + &settings, + &mut context, + Some(&graph), + ®istry, + None, + Some("example-shared-id"), + &mut response, + ); + + let writes = store.writes(); + assert_eq!(writes.add, 1, "should write one create"); + assert_eq!(writes.total(), 1, "should write nothing else"); + let ec_id = context.ec_value().expect("should hold the new EC"); + let (stored, _) = graph + .get(ec_id) + .expect("should read store") + .expect("should hold the new row"); + assert_eq!( + stored + .ids + .get(SHARED_ID_SOURCE) + .and_then(KvPartnerId::first_uid), + Some("example-shared-id"), + "should carry the sharedId as a one-UID record" + ); + assert_eq!( + set_cookie_names(&response), + vec!["ts-ec".to_owned()], + "should set the ts-ec cookie as today" + ); + } + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs index 6e444bac31..760b465e06 100644 --- a/crates/trusted-server-core/src/ec/identity/mutation.rs +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -131,8 +131,6 @@ pub(crate) struct ProviderResult { /// Everything one request changes in the identity row. #[derive(Debug, Clone, Default, PartialEq)] -// Temporary: first used by a later identity task. -#[allow(dead_code)] pub(crate) struct IdentityMutation { partner_ids: Vec, provider_state: BTreeMap, @@ -142,8 +140,6 @@ pub(crate) struct IdentityMutation { impl IdentityMutation { /// Adds a change that stands alone. - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn add_change(&mut self, change: PartnerIdChange) { self.partner_ids.push(change); } @@ -201,6 +197,12 @@ impl IdentityMutation { self.partner_ids.len() + self.results.len() + self.provider_state.len() } + /// The stand-alone changes, for tests that check what a caller staged. + #[cfg(test)] + pub(crate) fn positive_changes(&self) -> &[PartnerIdChange] { + &self.partner_ids + } + /// Whether nothing would change. #[must_use] // Temporary: first used by a later identity task. diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index f313b0fd88..654815fcaa 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -100,6 +100,8 @@ impl PartnerIdUpdate { } /// Terminal result of one browser EID-cookie persistence attempt. +// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. +#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] pub(crate) enum EidCookieSyncOutcome { /// The stored values already matched without a write. @@ -134,6 +136,8 @@ pub(crate) enum EidCookieSyncOutcome { Failed, } +// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. +#[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum CookieUpdateApplication { AlreadyMatched, @@ -141,6 +145,8 @@ enum CookieUpdateApplication { DeferredFreshness, } +// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. +#[allow(dead_code)] fn apply_cookie_partner_id_updates( entry: &mut KvEntry, updates: &[PartnerIdUpdate], @@ -182,6 +188,8 @@ fn apply_cookie_partner_id_updates( } } +// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. +#[allow(dead_code)] fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -586,6 +594,8 @@ impl KvIdentityGraph { /// conflict this method never writes again. A live follow-up becomes the /// authoritative snapshot; a missing or failed follow-up retains the live /// pre-write snapshot as proof and defers the update. + // Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. + #[allow(dead_code)] pub(crate) fn sync_eid_cookie_updates_from_snapshot( &self, ec_id: &str, diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 7f038461a2..e253dff984 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -89,9 +89,10 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; +use self::finalize::BrowserSeed; use self::identity::request::{RequestIdentity, WriterDecision}; use self::identity::runtime::IdentityRuntime; -use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; +use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; use self::kv_types::KvEntry; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; @@ -387,6 +388,10 @@ impl EcContext { /// If an EC ID already exists (from the request), this is a no-op. /// If consent does not permit EC creation, this is a no-op. /// + /// The new row carries `seed`, the request's browser ingest, so a new + /// visitor's create is the one write (spec 5.2 phase 1). Pass + /// [`BrowserSeed::empty`] when there is nothing to seed. + /// /// # Errors /// /// Returns an error if the client IP is unavailable and generation is @@ -395,6 +400,7 @@ impl EcContext { &mut self, settings: &Settings, kv: Option<&KvIdentityGraph>, + seed: &BrowserSeed, ) -> Result<(), Report> { if self.ec_value.is_some() { return Ok(()); @@ -429,6 +435,7 @@ impl EcContext { .device_signals .as_ref() .map(DeviceSignals::to_kv_device); + apply_partner_id_updates(&mut entry, seed.updates()); if let Some(graph) = kv { match graph.create_if_absent(&ec_id, &entry) { @@ -526,15 +533,11 @@ impl EcContext { /// The identity state this request decides and collects. #[must_use] - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn identity(&self) -> &RequestIdentity { &self.identity } /// The identity state this request decides and collects, to fill. - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn identity_mut(&mut self) -> &mut RequestIdentity { &mut self.identity } @@ -656,6 +659,8 @@ impl EcContext { /// Returns the current request's own EIDs, if the route captured any. #[must_use] + // Temporary: finalize no longer ingests the `/auction` body; Task 26 removes this plumbing. + #[allow(dead_code)] pub(crate) fn client_eids(&self) -> Option<&[Eid]> { self.client_eids.as_deref() } @@ -992,7 +997,7 @@ mod tests { EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.5".to_owned())); let graph = KvIdentityGraph::new(AddCollidingEcKv::new(2)); - ec.generate_if_needed(&settings, Some(&graph)) + ec.generate_if_needed(&settings, Some(&graph), &BrowserSeed::empty()) .expect("should generate after bounded collisions"); assert!(ec.ec_value().is_some(), "should allocate a fresh EC ID"); @@ -1011,7 +1016,7 @@ mod tests { // Collide on every attempt so the bounded retry is exhausted. let graph = KvIdentityGraph::new(AddCollidingEcKv::new(u32::MAX)); - let result = ec.generate_if_needed(&settings, Some(&graph)); + let result = ec.generate_if_needed(&settings, Some(&graph), &BrowserSeed::empty()); assert!(result.is_err(), "should fail after exhausting attempts"); assert!( @@ -1232,7 +1237,7 @@ mod tests { let mut ec = EcContext::read_from_request(&settings, &req, &noop_services()) .expect("should read EC context"); - ec.generate_if_needed(&settings, None) + ec.generate_if_needed(&settings, None, &BrowserSeed::empty()) .expect("should not error when EC already exists"); assert_eq!( diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 001ba97c7b..c5ad86d426 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -11,6 +11,7 @@ use matchit::Router; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; +use crate::ec::finalize::BrowserSeed; use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; @@ -808,6 +809,8 @@ pub struct ProxyDispatchInput<'a> { pub services: &'a RuntimeServices, /// Whether the client looks like a real browser. EC generation needs it. pub is_real_browser: bool, + /// The request's browser ingest, carried by an EC minted here. + pub seed: &'a BrowserSeed, pub req: Request, } @@ -1042,6 +1045,7 @@ impl IntegrationRegistry { ec_context, services, is_real_browser, + seed, mut req, } = input; if let Some((proxy, _)) = self.find_route(method, path) { @@ -1050,7 +1054,7 @@ impl IntegrationRegistry { // subresource requests may lack consent signals such as the // Sec-GPC header. if is_real_browser && is_navigation_request(&req) { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { + if let Err(err) = ec_context.generate_if_needed(settings, kv, seed) { log::warn!("EC generation failed for integration proxy: {err:?}"); } } else { @@ -1695,6 +1699,7 @@ mod tests { ec_context: &mut ec_context, services: &noop_services(), is_real_browser: true, + seed: &BrowserSeed::empty(), req, })) .expect("should match route") @@ -2053,6 +2058,7 @@ mod tests { ec_context: &mut ec_context, services: &services, is_real_browser: true, + seed: &BrowserSeed::empty(), req, })); @@ -2107,6 +2113,7 @@ mod tests { ec_context: &mut ec_context, services: &services, is_real_browser, + seed: &BrowserSeed::empty(), req, })); @@ -2124,6 +2131,92 @@ mod tests { ); } + #[test] + fn integration_proxy_new_ec_carries_browser_ingest() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::from_routes(vec![( + Method::GET, + "/integrations/test/ec", + ( + Arc::new(EcTestProxy) as Arc, + "ec_test", + ), + )]); + let services = noop_services(); + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..crate::consent::ConsentContext::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("192.0.2.1".to_owned())); + let partner_registry = + crate::ec::registry::PartnerRegistry::from_config(&[crate::settings::EcPartner { + name: "Shared ID".to_owned(), + source_domain: "sharedid.org".to_owned(), + openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(crate::redacted::Redacted::new( + "token-sharedid.org-32-bytes-minimum-value".to_owned(), + )), + batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), + pull_sync_enabled: false, + pull_sync_url: None, + pull_sync_allowed_domains: vec![], + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, + ts_pull_token: None, + }]) + .expect("should build partner registry"); + let seed = crate::ec::finalize::browser_seed( + &settings, + &partner_registry, + ec_context.consent(), + None, + Some("example-shared-id"), + 1_760_000_000, + ); + let req = Request::builder() + .method(Method::GET) + .uri("https://test-publisher.com/integrations/test/ec") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let result = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { + method: &Method::GET, + path: "/integrations/test/ec", + settings: &settings, + kv: Some(&graph), + ec_context: &mut ec_context, + services: &services, + is_real_browser: true, + seed: &seed, + req, + })); + + assert!( + result.is_some_and(|response| response.is_ok()), + "should handle the request" + ); + let writes = store.writes(); + assert_eq!(writes.add, 1, "should write one create"); + assert_eq!(writes.total(), 1, "should write nothing else"); + let ec_id = ec_context.ec_value().expect("should mint an EC"); + let (stored, _) = graph + .get(ec_id) + .expect("should read store") + .expect("should hold the new row"); + assert_eq!( + stored + .ids + .get("sharedid.org") + .and_then(crate::ec::kv_types::KvPartnerId::first_uid), + Some("example-shared-id"), + "should carry the seeded UID in the create" + ); + } + #[test] fn handle_proxy_rejects_invalid_ec_request_header() { let settings = create_test_settings(); @@ -2159,6 +2252,7 @@ mod tests { ec_context: &mut ec_context, services: &services, is_real_browser: true, + seed: &BrowserSeed::empty(), req, })) .expect("should handle proxy request"); @@ -2209,6 +2303,7 @@ mod tests { ec_context: &mut ec_context, services: &services, is_real_browser: true, + seed: &BrowserSeed::empty(), req, })) .expect("should handle proxy request"); @@ -2256,6 +2351,7 @@ mod tests { ec_context: &mut ec_context, services: &services, is_real_browser: true, + seed: &BrowserSeed::empty(), req, })); From ab358b88b0de669425358a3eb4ca3b96573af65f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 22:38:22 +0530 Subject: [PATCH 36/89] Return the ts-eids-s mirror from /auction and stop stager ingest /auction and /_ts/page-bids no longer write browser EIDs to KV. Both return a ts_eids_s payload built from the servable server set, which the shim writes as the client-only __Host-ts-eids-s cookie, or a clear on denied consent, a tombstone or a missing row. The shim's ts-eids trim keeps registered sources before unregistered ones, so the writer navigation's ingest still sees every registered source /auction used to ingest. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 580 ++++++++++++++++- .../src/auction/formats.rs | 52 +- crates/trusted-server-core/src/ec/eids.rs | 9 + crates/trusted-server-core/src/ec/finalize.rs | 64 -- .../src/ec/identity/mirror.rs | 4 - crates/trusted-server-core/src/ec/kv.rs | 614 ------------------ crates/trusted-server-core/src/ec/mod.rs | 25 - .../src/integrations/prebid.rs | 80 ++- crates/trusted-server-core/src/openrtb.rs | 5 + crates/trusted-server-core/src/publisher.rs | 222 ++++++- .../lib/src/core/eids_mirror.ts | 61 ++ .../lib/src/integrations/gpt/index.ts | 9 + .../lib/src/integrations/prebid/eids.ts | 27 +- .../lib/src/integrations/prebid/index.ts | 10 +- .../lib/test/core/eids_mirror.test.ts | 81 +++ .../test/integrations/gpt/spa_hook.test.ts | 49 ++ .../lib/test/integrations/prebid/eids.test.ts | 75 +++ .../test/integrations/prebid/index.test.ts | 57 ++ 18 files changed, 1276 insertions(+), 748 deletions(-) create mode 100644 crates/trusted-server-js/lib/src/core/eids_mirror.ts create mode 100644 crates/trusted-server-js/lib/test/core/eids_mirror.test.ts diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 152dc06044..0e80c9c041 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -16,9 +16,12 @@ use crate::ec::EcContext; use crate::ec::EcKvSnapshot; use crate::ec::checked_current_timestamp; use crate::ec::client_eids::parse_client_eids; -use crate::ec::eids::{resolve_partner_ids, to_eids}; +use crate::ec::current_timestamp; +use crate::ec::eids::{ResolvedPartnerId, resolve_partner_ids, to_eids}; +use crate::ec::identity::IdentityRuntime; use crate::ec::identity::UnixTime; use crate::ec::identity::catalog::ProviderCatalog; +use crate::ec::identity::mirror::{MirrorPayload, MirrorRead, MirrorSource, mirror_payload}; use crate::ec::identity::policy::PolicySignals; use crate::ec::identity::request::opted_out; use crate::ec::identity::servable::ServeContext; @@ -33,7 +36,7 @@ use crate::settings::Settings; use super::AuctionOrchestrator; use super::formats::{ - convert_to_openrtb_response, convert_to_openrtb_response_with_report, + convert_to_openrtb_response_with_mirror, convert_to_openrtb_response_with_report, convert_tsjs_to_auction_request, }; use super::telemetry::{ @@ -219,11 +222,17 @@ pub async fn handle_auction( total_time_ms: 0, metadata: HashMap::new(), }; - return convert_to_openrtb_response( + return convert_to_openrtb_response_with_mirror( &empty_result, settings, &auction_request, ec_context.ec_allowed(), + mirror_member( + &EcKvSnapshot::NotRead, + None, + ec_context, + PolicySignals::from_headers(http_req.headers()), + ), ); } @@ -272,11 +281,14 @@ pub async fn handle_auction( total_time_ms: 0, metadata: HashMap::new(), }; - return convert_to_openrtb_response( + // A denial clears the mirror so a withdrawal reaches the cookie + // (spec 5.6). + return convert_to_openrtb_response_with_mirror( &empty_result, settings, &auction_request, ec_context.ec_allowed(), + Some(MirrorPayload::Clear.to_json()), ); } @@ -301,12 +313,8 @@ pub async fn handle_auction( } else { None }; - // Carry the full request-local EID set to response finalization so KV - // ingestion uses what this request actually sent, not just whatever fits - // in the size-capped `ts-eids` cookie (see `ec::finalize::ec_finalize_response`). - if let Some(eids) = &client_eids { - ec_context.set_client_eids(eids.clone()); - } + // The browser's EIDs feed the merge below and nothing else: `/auction` is + // a stager, so they never reach KV (spec 5.3, G10). // Resolve partner EIDs from the KV identity graph when the user has a valid // EC and both KV and partner stores are available. Gate the read on a @@ -316,19 +324,15 @@ pub async fn handle_auction( (Some(graph), Some(ec_id), Some(_)) => graph.load_snapshot(ec_id), _ => EcKvSnapshot::NotRead, }; - // Hand the loaded row to the request context so response finalization — - // which runs on an EC context the adapter owns, after this handler returns - // — ingests `ts-eids`/`sharedId` updates from this read instead of paying - // for a second lookup. + // Hand the loaded row to the request context so response finalization, + // which runs on an EC context the adapter owns after this handler + // returns, reuses this read instead of paying for a second lookup. if !matches!(auction_kv_snapshot, EcKvSnapshot::NotRead) { ec_context.set_kv_snapshot(auction_kv_snapshot.clone()); } - let eids = resolve_auction_eids( - &auction_kv_snapshot, - registry, - ec_context, - PolicySignals::from_headers(http_req.headers()), - ); + let signals = PolicySignals::from_headers(http_req.headers()); + let eids = resolve_auction_eids(&auction_kv_snapshot, registry, ec_context, signals); + let mirror = mirror_member(&auction_kv_snapshot, registry, ec_context, signals); // Look up geo for device info. let geo = services @@ -404,6 +408,7 @@ pub async fn handle_auction( settings, &auction_request, ec_context.ec_allowed(), + mirror, ) { Ok(conversion) => conversion, Err(error) => { @@ -460,6 +465,17 @@ pub(crate) fn resolve_auction_eids( ec_context: &EcContext, signals: PolicySignals, ) -> Option> { + resolve_servable_ids(snapshot, registry, ec_context, signals).map(|resolved| to_eids(&resolved)) +} + +/// The servable stored records behind [`resolve_auction_eids`], before they +/// become EIDs. +fn resolve_servable_ids( + snapshot: &EcKvSnapshot, + registry: Option<&PartnerRegistry>, + ec_context: &EcContext, + signals: PolicySignals, +) -> Option> { let registry = registry?; if !ec_context.ec_allowed() { @@ -499,8 +515,82 @@ pub(crate) fn resolve_auction_eids( catalog, opted_out: &opted_out, }; - let resolved = resolve_partner_ids(registry, entry, &serve); - Some(to_eids(&resolved)) + Some(resolve_partner_ids(registry, entry, &serve)) +} + +/// What the request's KV read gives the `ts-eids-s` mirror (spec 6.2). +/// +/// `sources` receives the servable server set, one [`MirrorSource`] per +/// source, after `gate_eids_by_consent`: the mirror never carries an ID +/// consent filtered out. The returned [`MirrorRead`] borrows it. +/// +/// Consent that denies EC use clears the mirror before any KV read is +/// needed, so a withdrawal reaches the cookie. Otherwise the result follows +/// the row this request read: [`MirrorRead::NotRead`] when it read none, +/// [`MirrorRead::Cleared`] for a missing row, a tombstone or `consent.ok = +/// false`, and the servable set for a live row. +pub(crate) fn mirror_read<'a>( + snapshot: &EcKvSnapshot, + registry: Option<&PartnerRegistry>, + ec_context: &EcContext, + signals: PolicySignals, + sources: &'a mut Vec, +) -> MirrorRead<'a> { + if !ec_context.ec_allowed() { + return MirrorRead::Cleared; + } + let (Some(ec_id), Some(_)) = (ec_context.ec_value(), registry) else { + return MirrorRead::NotRead; + }; + match snapshot { + EcKvSnapshot::Missing { ec_id: read } if read == ec_id => return MirrorRead::Cleared, + EcKvSnapshot::Present { + ec_id: read, entry, .. + } if read == ec_id && !entry.consent.ok => { + return MirrorRead::Cleared; + } + EcKvSnapshot::Present { ec_id: read, .. } if read == ec_id => {} + _ => return MirrorRead::NotRead, + } + let resolved = + resolve_servable_ids(snapshot, registry, ec_context, signals).unwrap_or_default(); + let identity = ec_context.identity_runtime().map(IdentityRuntime::identity); + let candidates: Vec = resolved + .iter() + .map(|record| { + let priority = match (&record.writer, identity) { + (Some(writer), Some(identity)) => identity.provider(writer).cookie_priority, + _ => 0, + }; + MirrorSource { + source: record.source_domain.clone(), + provider: record.writer.clone(), + priority, + expires_at: record.expires_at, + eids: to_eids(core::slice::from_ref(record)), + } + }) + .collect(); + match gate_eids_by_consent(Some(candidates), Some(ec_context.consent())) { + Some(gated) => { + *sources = gated; + MirrorRead::Servable(sources) + } + None => MirrorRead::Cleared, + } +} + +/// The mirror member of an `/auction` or `/_ts/page-bids` response, from the +/// row this request read (spec 6.2). +pub(crate) fn mirror_member( + snapshot: &EcKvSnapshot, + registry: Option<&PartnerRegistry>, + ec_context: &EcContext, + signals: PolicySignals, +) -> Option { + let mut sources = Vec::new(); + let read = mirror_read(snapshot, registry, ec_context, signals, &mut sources); + mirror_payload(read, current_timestamp()).map(|payload| payload.to_json()) } /// Resolves the request's browser EIDs: the `/auction` body when it has a @@ -2337,4 +2427,450 @@ mod tests { ); }); } + + mod mirror { + use super::*; + use crate::ec::kv_types::{KvEntry, KvPartnerId}; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + + const MIRROR_SOURCE_A: &str = "a.example.com"; + const MIRROR_SOURCE_B: &str = "b.example.com"; + + fn mirror_registry() -> PartnerRegistry { + let mut disabled = counting_test_partner("off.example.com"); + disabled.bidstream_enabled = false; + PartnerRegistry::from_config(&[ + counting_test_partner(MIRROR_SOURCE_A), + counting_test_partner(MIRROR_SOURCE_B), + disabled, + ]) + .expect("should build partner registry") + } + + fn row_with(records: &[(&str, &str, Option)]) -> KvEntry { + let mut entry = KvEntry::minimal("unregistered.example.com", "x", 1); + entry.ids.clear(); + for (source, uid, exp) in records { + entry.ids.insert( + (*source).to_owned(), + KvPartnerId { + exp: *exp, + ..KvPartnerId::browser(*uid, Some(1)) + }, + ); + } + entry + } + + fn ec_id_for(tag: &str) -> String { + format!("{}.{tag}", "a".repeat(64)) + } + + /// Records the EIDs its request carried, then answers no-bid so the + /// auction completes and the response can be inspected. + struct NoBidRecordingProvider { + eids: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for NoBidRecordingProvider { + fn provider_name(&self) -> &str { + "eid_recording_provider" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + *self.eids.lock().expect("should lock recorded eids") = Some( + serde_json::to_value(&request.user.eids) + .expect("should serialize recorded eids"), + ); + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.provider_name(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run for an immediate answer"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name( + &self, + _services: &RuntimeServices, + _timeout_ms: u32, + ) -> Option { + Some("record-backend".to_string()) + } + } + + fn recording_orchestrator(recorded: &Arc>>) -> AuctionOrchestrator { + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["eid_recording_provider"]), + timeout_ms: 2000, + mediator: None, + ..Default::default() + }); + orchestrator.register_provider(Arc::new(NoBidRecordingProvider { + eids: Arc::clone(recorded), + })); + orchestrator + } + + async fn run_auction( + ec_context: &mut EcContext, + graph: Option<&KvIdentityGraph>, + registry: Option<&PartnerRegistry>, + body: Vec, + ) -> JsonValue { + let recorded = Arc::new(Mutex::new(None)); + let orchestrator = recording_orchestrator(&recorded); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from(body)) + .expect("should build auction request"); + let response = handle_auction( + &create_test_settings(), + &orchestrator, + graph, + registry, + ec_context, + &noop_services(), + req, + ) + .await + .expect("should answer the auction"); + let bytes = response + .into_body() + .into_bytes() + .expect("should buffer the body"); + serde_json::from_slice(&bytes).expect("should parse the response") + } + + fn mirror_of(response: &JsonValue) -> Option<&JsonValue> { + response.get("ext").and_then(|ext| ext.get("ts_eids_s")) + } + + fn decoded(mirror: &JsonValue) -> JsonValue { + let value = mirror["value"].as_str().expect("should hold a value"); + let bytes = URL_SAFE_NO_PAD + .decode(value) + .expect("should be unpadded base64url"); + serde_json::from_slice(&bytes).expect("should be JSON") + } + + fn live_context(ec_id: &str, graph: &KvIdentityGraph) -> EcContext { + let _ = graph; + make_ec_context(Jurisdiction::NonRegulated, Some(ec_id)) + } + + #[tokio::test] + async fn auction_body_eids_are_not_written_to_kv() { + let (graph, store) = KvIdentityGraph::write_counting("test_store"); + let ec_id = ec_id_for("nowrit"); + graph + .create(&ec_id, &row_with(&[])) + .expect("should seed the live row"); + store.reset_counts(); + let registry = mirror_registry(); + let recorded = Arc::new(Mutex::new(None)); + let orchestrator = recording_orchestrator(&recorded); + let mut ec_context = live_context(&ec_id, &graph); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&json!({ + "adUnits": [{ "code": "div-gpt-1", "mediaTypes": { "banner": { "sizes": [[300, 250]] } } }], + "eids": [{ "source": MIRROR_SOURCE_A, "uids": [{ "id": "body-uid", "atype": 1 }] }] + })) + .expect("should serialize body"), + )) + .expect("should build auction request"); + + let _ = handle_auction( + &create_test_settings(), + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + let mut response = http::Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &create_test_settings(), + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + + let writes = store.writes(); + assert_eq!( + (writes.add, writes.if_generation_match, writes.overwrite), + (0, 0, 0), + "should write nothing to KV" + ); + assert_eq!( + recorded.lock().expect("should lock recorded eids").clone(), + Some( + json!([{ "source": MIRROR_SOURCE_A, "uids": [{ "id": "body-uid", "atype": 1 }] }]) + ), + "should still send the body's UID to the bidder" + ); + } + + #[tokio::test] + async fn auction_returns_the_mirror_of_its_servable_set() { + let ec_id = ec_id_for("mirror"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create(&ec_id, &row_with(&[(MIRROR_SOURCE_A, "uid-a", None)])) + .expect("should seed the live row"); + let registry = mirror_registry(); + let mut ec_context = live_context(&ec_id, &graph); + + let response = run_auction( + &mut ec_context, + Some(&graph), + Some(®istry), + ad_units_only_body(), + ) + .await; + + let mirror = mirror_of(&response).expect("should carry the mirror"); + let atype = crate::settings::EcPartner::default_openrtb_atype(); + assert_eq!( + decoded(mirror), + json!({ "v": 1, "eids": [{ "source": MIRROR_SOURCE_A, "uids": [{ "id": "uid-a", "atype": atype }] }] }), + "should hold exactly the servable record's EID" + ); + assert_eq!( + mirror["max_age"], 86_400, + "should live a day without an exp" + ); + } + + #[tokio::test] + async fn auction_mirror_max_age_follows_the_earliest_exp() { + let now = crate::ec::current_timestamp(); + let ec_id = ec_id_for("maxage"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &ec_id, + &row_with(&[ + (MIRROR_SOURCE_A, "uid-a", Some(now + 600)), + (MIRROR_SOURCE_B, "uid-b", Some(now + 999_999)), + ]), + ) + .expect("should seed the live row"); + let registry = mirror_registry(); + let mut ec_context = live_context(&ec_id, &graph); + + let response = run_auction( + &mut ec_context, + Some(&graph), + Some(®istry), + ad_units_only_body(), + ) + .await; + + let max_age = mirror_of(&response).expect("should carry the mirror")["max_age"] + .as_u64() + .expect("should hold a number"); + assert!( + (595..=600).contains(&max_age), + "should take the earliest exp, got {max_age}" + ); + } + + #[tokio::test] + async fn auction_mirror_leaves_out_bidstream_disabled_sources() { + let ec_id = ec_id_for("bidoff"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &ec_id, + &row_with(&[ + (MIRROR_SOURCE_A, "uid-a", None), + ("off.example.com", "uid-off", None), + ]), + ) + .expect("should seed the live row"); + let registry = mirror_registry(); + let mut ec_context = live_context(&ec_id, &graph); + + let response = run_auction( + &mut ec_context, + Some(&graph), + Some(®istry), + ad_units_only_body(), + ) + .await; + + let sources: Vec = + decoded(mirror_of(&response).expect("should carry the mirror"))["eids"] + .as_array() + .expect("should hold EIDs") + .iter() + .map(|eid| eid["source"].as_str().unwrap_or_default().to_owned()) + .collect(); + assert_eq!( + sources, + vec![MIRROR_SOURCE_A.to_owned()], + "should leave out the source bidstream disabled" + ); + } + + #[tokio::test] + async fn auction_mirror_leaves_out_ids_consent_filters_out() { + // TCF Purpose 1 granted, Purpose 4 refused: the auction runs, but + // `gate_eids_by_consent` strips every EID the bidder would get. + let ec_id = ec_id_for("gated"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create(&ec_id, &row_with(&[(MIRROR_SOURCE_A, "uid-a", None)])) + .expect("should seed the live row"); + let registry = mirror_registry(); + let mut ec_context = + EcContext::new_for_test(Some(ec_id.clone()), gdpr_consent_with_purposes(&[1], 4)); + + let response = run_auction( + &mut ec_context, + Some(&graph), + Some(®istry), + ad_units_only_body(), + ) + .await; + + assert_eq!( + mirror_of(&response), + Some(&json!({ "clear": true })), + "should not mirror an ID consent filtered out" + ); + } + + fn gdpr_consent_with_purposes(granted: &[usize], _refused: usize) -> ConsentContext { + let mut consent = + crate::ec::identity::test_support::test_consent_gdpr_without_vendor(9_999); + let tcf = consent.tcf.as_mut().expect("should hold a TCF string"); + tcf.purpose_consents = (1..=24).map(|purpose| granted.contains(&purpose)).collect(); + consent + } + + #[tokio::test] + async fn auction_returns_clear_on_consent_denial_tombstone_and_missing_row() { + let registry = mirror_registry(); + let live = row_with(&[(MIRROR_SOURCE_A, "uid-a", None)]); + let mut withdrawn = live.clone(); + withdrawn.consent.ok = false; + let cases: [(&str, Option, bool, JsonValue); 5] = [ + ("live row", Some(live.clone()), false, json!("value")), + ( + "tombstone", + Some(KvEntry::tombstone(1_000)), + false, + json!({ "clear": true }), + ), + ( + "consent.ok false", + Some(withdrawn), + false, + json!({ "clear": true }), + ), + ("missing row", None, false, json!({ "clear": true })), + ("consent denial", Some(live), true, json!({ "clear": true })), + ]; + + for (label, row, deny_consent, expected) in cases { + let ec_id = ec_id_for("clears"); + let graph = KvIdentityGraph::in_memory("test_store"); + if let Some(row) = &row { + graph.create(&ec_id, row).expect("should seed the row"); + } + let mut ec_context = if deny_consent { + // No TCF Purpose 1: the auction consent gate answers early. + EcContext::new_for_test(Some(ec_id.clone()), gdpr_consent_with_purposes(&[], 4)) + } else { + live_context(&ec_id, &graph) + }; + + let response = run_auction( + &mut ec_context, + Some(&graph), + Some(®istry), + ad_units_only_body(), + ) + .await; + + let mirror = mirror_of(&response) + .unwrap_or_else(|| panic!("{label}: should carry a mirror")); + if expected == json!("value") { + assert!( + mirror.get("value").is_some(), + "{label}: should carry a value" + ); + } else { + assert_eq!(mirror, &expected, "{label}: should clear"); + } + } + } + + #[tokio::test] + async fn auction_returns_clear_when_the_servable_set_is_empty() { + let ec_id = ec_id_for("empty"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create(&ec_id, &row_with(&[("off.example.com", "uid-off", None)])) + .expect("should seed the live row"); + let registry = mirror_registry(); + let mut ec_context = live_context(&ec_id, &graph); + + let response = run_auction( + &mut ec_context, + Some(&graph), + Some(®istry), + ad_units_only_body(), + ) + .await; + + assert_eq!( + mirror_of(&response), + Some(&json!({ "clear": true })), + "should clear an empty servable set" + ); + } + + #[tokio::test] + async fn auction_without_a_kv_read_omits_the_mirror() { + let ec_id = ec_id_for("noread"); + let registry = mirror_registry(); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + + let response = + run_auction(&mut ec_context, None, Some(®istry), ad_units_only_body()).await; + + assert!( + response["ext"].get("ts_eids_s").is_none(), + "should leave the mirror cookie as it is" + ); + } + } } diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index cec4823005..de037e9d37 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -341,10 +341,30 @@ pub fn convert_to_openrtb_response( auction_request: &AuctionRequest, ec_allowed: bool, ) -> Result, Report> { - Ok( - convert_to_openrtb_response_with_report(result, settings, auction_request, ec_allowed)? - .response, - ) + convert_to_openrtb_response_with_mirror(result, settings, auction_request, ec_allowed, None) +} + +/// Like [`convert_to_openrtb_response`], carrying the `ts-eids-s` mirror +/// payload as `ext.ts_eids_s` (spec 6.2). +/// +/// # Errors +/// +/// Returns an error if response serialization fails. +pub(crate) fn convert_to_openrtb_response_with_mirror( + result: &OrchestrationResult, + settings: &Settings, + auction_request: &AuctionRequest, + ec_allowed: bool, + ts_eids_s: Option, +) -> Result, Report> { + Ok(convert_to_openrtb_response_with_report( + result, + settings, + auction_request, + ec_allowed, + ts_eids_s, + )? + .response) } pub(crate) fn convert_to_openrtb_response_with_report( @@ -352,6 +372,7 @@ pub(crate) fn convert_to_openrtb_response_with_report( settings: &Settings, auction_request: &AuctionRequest, ec_allowed: bool, + ts_eids_s: Option, ) -> Result> { let mut seatbids = Vec::with_capacity(result.winning_bids.len()); let rewrite_creatives = settings.auction.rewrite_creatives; @@ -512,6 +533,7 @@ pub(crate) fn convert_to_openrtb_response_with_report( dropped_winner_count: delivery.dropped_winner_count, dropped_winner_reasons: delivery.dropped_winner_reasons.clone(), }, + ts_eids_s, } .to_ext(), ..Default::default() @@ -1601,9 +1623,14 @@ mod tests { metadata: HashMap::new(), }; - let conversion = - convert_to_openrtb_response_with_report(&result, &settings, &auction_request, false) - .expect("should omit invalid winners and preserve valid slots"); + let conversion = convert_to_openrtb_response_with_report( + &result, + &settings, + &auction_request, + false, + None, + ) + .expect("should omit invalid winners and preserve valid slots"); assert_eq!( conversion.delivery.delivered_winner_slots, HashSet::from(["ordinary".to_string(), "renderer".to_string()]), @@ -1926,9 +1953,14 @@ mod tests { let auction_request = make_auction_request(); let result = make_result(make_bid("div-gpt-top", "appnexus", None)); - let conversion = - convert_to_openrtb_response_with_report(&result, &settings, &auction_request, false) - .expect("should omit a winner without a decoded price"); + let conversion = convert_to_openrtb_response_with_report( + &result, + &settings, + &auction_request, + false, + None, + ) + .expect("should omit a winner without a decoded price"); assert!(conversion.delivery.delivered_winner_slots.is_empty()); assert_eq!(conversion.delivery.dropped_winner_count, 1); assert_eq!( diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 26f05dc167..a9affaa36f 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -37,6 +37,9 @@ pub struct ResolvedPartnerId { /// The provider that wrote the record. `None` for browser ingest and /// version 1 rows. pub writer: Option, + /// Unix seconds at which the record stops being servable, when it has an + /// `exp`. + pub expires_at: Option, } /// Resolves source-domain keyed IDs from a KV entry against the partner registry. @@ -79,6 +82,7 @@ pub(crate) fn resolve_partner_ids( .w .as_deref() .and_then(|writer| writer.parse().ok()), + expires_at: partner_uid.exp, }); } @@ -406,6 +410,7 @@ mod tests { openrtb_atype: 3, entries: KvPartnerId::new("LR_xyz".to_owned()).eids, writer: None, + expires_at: None, }, ResolvedPartnerId { uid: "ID5_abc".to_owned(), @@ -413,6 +418,7 @@ mod tests { openrtb_atype: 1, entries: KvPartnerId::new("ID5_abc".to_owned()).eids, writer: None, + expires_at: None, }, ResolvedPartnerId { uid: "pair-id".to_owned(), @@ -420,6 +426,7 @@ mod tests { openrtb_atype: 571187, entries: KvPartnerId::new("pair-id".to_owned()).eids, writer: None, + expires_at: None, }, ]; @@ -451,6 +458,7 @@ mod tests { openrtb_atype: 3, entries: KvPartnerId::new(format!("uid_{}", "x".repeat(100))).eids, writer: None, + expires_at: None, }); } @@ -472,6 +480,7 @@ mod tests { openrtb_atype: 3, entries: KvPartnerId::new("u1".to_owned()).eids, writer: None, + expires_at: None, }]; let (encoded, truncated) = diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index edf1862bcf..b761e238c3 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -613,7 +613,6 @@ mod tests { use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; use crate::ec::identity::request::WriterDecision; - use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; use crate::test_support::tests::{create_test_settings, purpose_one_only_consent}; @@ -1618,61 +1617,6 @@ mod tests { ); } - #[test] - fn finalize_ignores_auction_body_eids_as_ingest() { - // The `/auction` body is no longer an ingest input (spec G10): the - // writer navigation ingests `ts-eids` and sharedId only, so a body - // that the cookie's size cap trimmed reaches bidders but never KV. - let settings = create_test_settings(); - let ec_id = sample_ec_id("cleids1"); - let graph = KvIdentityGraph::in_memory("test_store"); - let live = KvEntry::new( - &granting_consent(), - None, - current_timestamp(), - &settings.publisher.domain, - ); - graph - .create(&ec_id, &live) - .expect("should seed the live row this request updates"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), false); - ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid::new( - "liveramp.com".to_owned(), - vec![Uid { - id: "LR_from_body".to_owned(), - atype: Some(3), - ext: None, - }], - )]); - let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - ®istry, - None, - None, - &mut response, - ); - - let (stored, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("row should remain"); - assert!( - stored.ids.is_empty(), - "a body EID must not be written to KV" - ); - assert!( - ec_context.identity().mutation().is_empty(), - "a body EID must not be queued" - ); - } - #[test] fn finalize_withholds_eid_kv_writes_when_purpose_four_is_denied() { // A GDPR user can grant TCF Purpose 1 (storage/EC) while denying @@ -2799,14 +2743,6 @@ mod tests { if let Some(source) = source { context.set_eid_sync_source(source); } - context.set_client_eids(vec![Eid::new( - SHARED_ID_SOURCE.to_owned(), - vec![Uid { - id: "example-body-id".to_owned(), - atype: None, - ext: None, - }], - )]); let response = finalize_with_shared_id(&mut context, &graph, Some("example-shared-id")); diff --git a/crates/trusted-server-core/src/ec/identity/mirror.rs b/crates/trusted-server-core/src/ec/identity/mirror.rs index 95f911c259..35937cfffb 100644 --- a/crates/trusted-server-core/src/ec/identity/mirror.rs +++ b/crates/trusted-server-core/src/ec/identity/mirror.rs @@ -1,10 +1,6 @@ //! The `ts-eids-s` mirror payload: the servable server set for page scripts, //! capped to one cookie (spec 6.2). The server never reads the cookie back. -// Temporary: the coordinator, the first non-test caller, arrives in a later -// task. Remove this when it lands. -#![allow(dead_code)] - use base64::Engine as _; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use serde_json::json; diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 654815fcaa..4eb82d1c5d 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -136,74 +136,6 @@ pub(crate) enum EidCookieSyncOutcome { Failed, } -// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. -#[allow(dead_code)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CookieUpdateApplication { - AlreadyMatched, - Changed { deferred: bool }, - DeferredFreshness, -} - -// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. -#[allow(dead_code)] -fn apply_cookie_partner_id_updates( - entry: &mut KvEntry, - updates: &[PartnerIdUpdate], -) -> CookieUpdateApplication { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update); - } - - let mut changed = false; - let mut deferred = false; - for (partner_id, update) in latest_updates { - let uid = update.uid.as_str(); - match entry.ids.get(partner_id) { - Some(existing) if existing.first_uid() == Some(uid) => continue, - // Browser EID cookies carry no value-owned sequence or timestamp. - // A different value therefore has unknown freshness and must not - // replace the stored value. - Some(_) => { - deferred = true; - continue; - } - None => {} - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId::browser(uid, update.acquired_at), - ); - changed = true; - } - - if changed { - CookieUpdateApplication::Changed { deferred } - } else if deferred { - CookieUpdateApplication::DeferredFreshness - } else { - CookieUpdateApplication::AlreadyMatched - } -} - -// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. -#[allow(dead_code)] -fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update); - } - - latest_updates.into_iter().all(|(partner_id, update)| { - entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.first_uid() == Some(update.uid.as_str())) - }) -} - pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -587,156 +519,6 @@ impl KvIdentityGraph { ))) } - /// Persists browser EID cookies with one conditional write and one conflict read. - /// - /// Browser cookies do not carry a value-owned version, so a different - /// existing value has unknown freshness and is always deferred. After a CAS - /// conflict this method never writes again. A live follow-up becomes the - /// authoritative snapshot; a missing or failed follow-up retains the live - /// pre-write snapshot as proof and defers the update. - // Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. - #[allow(dead_code)] - pub(crate) fn sync_eid_cookie_updates_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> (EcKvSnapshot, EidCookieSyncOutcome) { - if updates.is_empty() { - return (snapshot, EidCookieSyncOutcome::AlreadyMatched); - } - - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - let current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return (snapshot, EidCookieSyncOutcome::Failed), - _ => self.load_snapshot(ec_id), - }; - - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - EcKvSnapshot::Missing { .. } => { - let outcome = if proven.is_some() { - EidCookieSyncOutcome::DeferredStaleRead - } else { - EidCookieSyncOutcome::Missing - }; - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, outcome); - } - EcKvSnapshot::Failed { .. } => { - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, EidCookieSyncOutcome::Failed); - } - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - } - }; - - if !entry.consent.ok { - return (current, EidCookieSyncOutcome::ConsentWithdrawn); - } - let deferred_freshness = match apply_cookie_partner_id_updates(&mut entry, updates) { - CookieUpdateApplication::AlreadyMatched => { - return (current, EidCookieSyncOutcome::AlreadyMatched); - } - CookieUpdateApplication::DeferredFreshness => { - return (current, EidCookieSyncOutcome::DeferredFreshness); - } - CookieUpdateApplication::Changed { deferred } => deferred, - }; - - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => ( - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }, - if deferred_freshness { - EidCookieSyncOutcome::WrittenWithDeferredFreshness - } else { - EidCookieSyncOutcome::Written - }, - ), - Ok(EcKvWriteOutcome::PreconditionFailed) => { - let refreshed = self.load_snapshot(ec_id); - let Some(refreshed_entry) = refreshed.entry_for(ec_id) else { - // The failed CAS proved `current`'s generation stale. Keep - // only its existence proof so later writes must reread. - let proof = match current { - EcKvSnapshot::Present { - ec_id: proven_id, - entry, - .. - } => EcKvSnapshot::Present { - ec_id: proven_id, - entry, - generation: None, - }, - other => other, - }; - let kept = Self::keep_proven(ec_id, refreshed, Some(&proof)); - return (kept, EidCookieSyncOutcome::DeferredConflict); - }; - if !refreshed_entry.consent.ok { - return (refreshed, EidCookieSyncOutcome::ConsentWithdrawn); - } - let outcome = if partner_id_updates_match(refreshed_entry, updates) { - EidCookieSyncOutcome::ConflictMatched - } else { - EidCookieSyncOutcome::DeferredConflict - }; - (refreshed, outcome) - } - Err(err) => { - log::warn!("EID cookie sync write failed: {err:?}"); - ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ) - } - } - } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. /// /// A caller-supplied `Present` snapshot is *proof* that the row exists — @@ -2087,131 +1869,6 @@ mod tests { } } - /// Store that replaces the row during the first EID CAS write and records - /// the request's reads and conditional writes. - struct EidConflictEcKv { - inner: InMemoryEcKv, - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - follow_up_miss: bool, - miss_next_lookup: std::sync::atomic::AtomicBool, - } - - impl EidConflictEcKv { - fn new( - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - ) -> Self { - Self { - inner: InMemoryEcKv::new("eid-conflict-store"), - concurrent_entry, - lookups, - conditional_writes, - follow_up_miss: false, - miss_next_lookup: std::sync::atomic::AtomicBool::new(false), - } - } - - fn with_follow_up_miss(mut self) -> Self { - self.follow_up_miss = true; - self - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize initial entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed initial entry"); - } - } - - impl EcKvStore for EidConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.lookups - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if self - .miss_next_lookup - .swap(false, std::sync::atomic::Ordering::Relaxed) - { - return Ok(None); - } - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - self.conditional_writes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let (body, meta) = KvIdentityGraph::serialize_entry( - &self.concurrent_entry, - self.inner.store_name(), - ) - .expect("should serialize concurrent entry"); - self.inner - .insert( - key, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Overwrite, - }, - ) - .expect("should write concurrent entry"); - if self.follow_up_miss { - self.miss_next_lookup - .store(true, std::sync::atomic::Ordering::Relaxed); - } - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn count_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result> { - self.inner.count_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn create_or_revive_retries_cas_conflict_and_succeeds() { let store = ConflictInjectingEcKv::new(2, false); @@ -2368,277 +2025,6 @@ mod tests { assert_eq!(entry.ids["ssp_x"].first_uid(), Some("original")); } - #[test] - fn eid_cookie_sync_conflict_matching_value_stops_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::new("desired-uid"), - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::ConflictMatched); - assert_eq!( - snapshot - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .and_then(|id| id.first_uid()), - Some("desired-uid"), - "the authoritative follow-up snapshot should replace stale request state" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - } - - #[test] - fn eid_cookie_sync_keeps_live_proof_when_conflict_follow_up_misses() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - live_entry(), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ) - .with_follow_up_miss(); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the live pre-write row should prevent conflict deferral from entering orphan recovery" - ); - assert_eq!( - snapshot.generation_for(&ec_id), - None, - "a failed CAS must not return its rejected generation" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "the initial refresh and conflict follow-up should be the only reads" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "the conflict must remain the request's only conditional write" - ); - } - - #[test] - fn eid_cookie_sync_reports_proven_refresh_miss_as_deferred() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("empty-store"); - let proven = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - proven, - ); - - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the add-confirmed row should survive an eventually consistent miss" - ); - assert_eq!( - outcome, - EidCookieSyncOutcome::DeferredStaleRead, - "a row already proven to exist should report a deferred stale read" - ); - } - - #[test] - fn partner_id_conflict_match_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::new("latest-uid"), - ); - let updates = [ - PartnerIdUpdate::new("ssp_x", "older-uid"), - PartnerIdUpdate::new("ssp_x", "latest-uid"), - ]; - - assert!( - partner_id_updates_match(&entry, &updates), - "conflict matching should use the same last-value-wins rule as writes" - ); - } - - #[test] - fn eid_cookie_sync_conflicting_value_defers_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::new("concurrent-uid"), - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "stale-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read concurrent row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].first_uid(), Some("concurrent-uid")); - } - - #[test] - fn eid_cookie_sync_preserves_concurrent_withdrawal_after_conflict() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - KvEntry::tombstone(2_000), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); - - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); - - assert_eq!(outcome, EidCookieSyncOutcome::ConsentWithdrawn); - assert!( - snapshot - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the refreshed withdrawal tombstone must remain authoritative" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - } - - #[test] - fn eid_cookie_sync_defers_different_values_without_value_owned_freshness() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::new("protected-uid"), - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "incoming-uid")], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::DeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read protected row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].first_uid(), Some("protected-uid")); - } - - #[test] - fn eid_cookie_sync_adds_missing_ids_while_deferring_different_values() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("mixed-freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::new("protected-uid"), - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[ - PartnerIdUpdate::new("ssp_x", "different-uid"), - PartnerIdUpdate::new("ssp_y", "new-uid"), - ], - graph.load_snapshot(&ec_id), - ); - - assert_eq!(outcome, EidCookieSyncOutcome::WrittenWithDeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read updated row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].first_uid(), Some("protected-uid")); - assert_eq!(stored.ids["ssp_y"].first_uid(), Some("new-uid")); - } - #[test] fn evaluate_cluster_returns_stored_value_without_store_io() { let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index e253dff984..1a9391edd2 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -84,7 +84,6 @@ use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::http_util::{is_document_navigation, is_prefetch_request}; -use crate::openrtb::Eid; use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; @@ -281,12 +280,6 @@ pub struct EcContext { kv_snapshot: EcKvSnapshot, /// Whether this request may rotate an orphaned EC identity. recovery_eligible: bool, - /// EIDs the route resolved from the current request: the `/auction` JSON - /// body, or the `ts-eids` cookie when the body carried none. Carried to - /// response finalization so KV ingestion can use the full set the client - /// sent instead of being limited to whatever fits in the size-capped - /// `ts-eids` cookie. - client_eids: Option>, /// Browser-carried proof of recent pull-partner completeness. pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. @@ -372,7 +365,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - client_eids: None, pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, identity: RequestIdentity::default(), @@ -651,20 +643,6 @@ impl EcContext { self.recovery_eligible } - /// Records the current request's own EIDs (e.g. an `/auction` body) for - /// use by response finalization's KV ingestion. - pub(crate) fn set_client_eids(&mut self, eids: Vec) { - self.client_eids = Some(eids); - } - - /// Returns the current request's own EIDs, if the route captured any. - #[must_use] - // Temporary: finalize no longer ingests the `/auction` body; Task 26 removes this plumbing. - #[allow(dead_code)] - pub(crate) fn client_eids(&self) -> Option<&[Eid]> { - self.client_eids.as_deref() - } - /// Validates a browser completeness marker against the active EC and partner set. pub(crate) fn validate_pull_sync_marker( &mut self, @@ -751,7 +729,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), @@ -777,7 +754,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), @@ -806,7 +782,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index f28ddfb297..3eaeb57827 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -973,6 +973,9 @@ fn validate_external_bundle_config( pub struct PrebidIntegration { config: PrebidIntegrationConfig, + /// Every `[[ec.partners]]` source domain, sorted, for the shim's + /// `ts-eids` trim (spec 5.3). + registered_sources: Vec, planned_head_inserts: Option>, #[cfg(test)] legacy_config: Option, @@ -986,6 +989,7 @@ impl PrebidIntegration { let engine = Arc::new(BidParamOverrideEngine::try_from_config(&config)?); Ok(Arc::new(Self { config: PrebidIntegrationConfig::from(&config), + registered_sources: Vec::new(), planned_head_inserts: None, legacy_config: Some(config), engine, @@ -997,9 +1001,14 @@ impl PrebidIntegration { Self::try_new(config).expect("should compile prebid bid param overrides") } - fn for_browser_plan(config: &PrebidIntegrationConfig, plan: &AuctionPlan) -> Arc { + fn for_browser_plan( + config: &PrebidIntegrationConfig, + plan: &AuctionPlan, + registered_sources: Vec, + ) -> Arc { let mut integration = Self { config: config.clone(), + registered_sources, planned_head_inserts: None, #[cfg(test)] legacy_config: None, @@ -1123,6 +1132,7 @@ impl PrebidIntegration { } else { Vec::new() }), + registered_sources: &self.registered_sources, ..InjectedPrebidClientConfig::from(browser_config) }; let config_json = serialize_injected_prebid_config(&payload); @@ -1368,7 +1378,15 @@ pub fn register_for_plan( config.excluded_gam_ad_unit_path_suffixes = canonical; validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; validate_browser_bidder_ownership(&config, plan)?; - let integration = PrebidIntegration::for_browser_plan(&config, plan); + let registered_sources = settings + .ec + .partners + .iter() + .map(|partner| partner.source_domain.clone()) + .collect::>() + .into_iter() + .collect(); + let integration = PrebidIntegration::for_browser_plan(&config, plan, registered_sources); Ok(Some( IntegrationRegistration::builder(PREBID_INTEGRATION_ID) .with_proxy(integration.clone()) @@ -1582,6 +1600,9 @@ struct InjectedPrebidClientConfig<'a> { client_side_bidders: &'a [String], #[serde(skip_serializing_if = "<[String]>::is_empty")] excluded_gam_ad_unit_path_suffixes: &'a [String], + /// The source domains of `[[ec.partners]]`, nothing else from the table. + #[serde(skip_serializing_if = "<[String]>::is_empty")] + registered_sources: &'a [String], } impl<'a> From<&'a PrebidIntegrationConfig> for InjectedPrebidClientConfig<'a> { @@ -1599,6 +1620,7 @@ impl<'a> From<&'a PrebidIntegrationConfig> for InjectedPrebidClientConfig<'a> { server_side_bidders: None, client_side_bidders: &config.client_side_bidders, excluded_gam_ad_unit_path_suffixes: &config.excluded_gam_ad_unit_path_suffixes, + registered_sources: &[], } } } @@ -4149,6 +4171,60 @@ server_url = "https://prebid.example/openrtb2/auction" "should inject the canonical suffix list: {config_insert}" ); } + #[test] + fn injected_config_lists_registered_sources() { + let mut settings = make_settings(); + settings + .integrations + .insert_config( + PREBID_INTEGRATION_ID, + &json!({ + "enabled": true, + "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js" + }), + ) + .expect("should replace Prebid test configuration"); + settings.ec.partners = ["zeta.example.com", "alpha.example.com", "mid.example.com"] + .into_iter() + .map(|source| { + crate::ec::identity::test_support::ec_partner( + source, + "https://sync.example.com/pull", + "example-token-value-0123456789abcdef", + ) + }) + .collect(); + let plan = Arc::new( + crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"), + ); + let registry = IntegrationRegistry::with_plan(&settings, plan) + .expect("should build integration registry"); + let document_state = IntegrationDocumentState::default(); + let context = IntegrationHtmlContext { + request_host: "pub.example", + request_scheme: "https", + origin_host: "origin.example", + document_state: &document_state, + }; + + let inserts = registry.head_inserts(&context); + let config_insert = inserts + .iter() + .find(|insert| insert.contains("window.__tsjs_prebid")) + .expect("should inject planned Prebid config"); + + assert!( + config_insert.contains( + r#""registeredSources":["alpha.example.com","mid.example.com","zeta.example.com"]"# + ), + "should list every partner source domain, sorted: {config_insert}" + ); + assert!( + !config_insert.contains("example-token") && !config_insert.contains("sync.example.com"), + "should carry nothing else from the partner table: {config_insert}" + ); + } + #[test] fn managed_user_ids_parse_with_opaque_params() { let config = parse_prebid_toml( diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index e586a0900c..849bdf942c 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -232,6 +232,10 @@ pub struct BidTrustedServerExt<'a> { #[derive(Debug, Serialize)] pub struct ResponseExt { pub orchestrator: OrchestratorExt, + /// The `ts-eids-s` mirror payload, absent when the request read no KV + /// (spec 6.2). + #[serde(skip_serializing_if = "Option::is_none")] + pub ts_eids_s: Option, } impl ToExt for ResponseExt {} @@ -352,6 +356,7 @@ mod tests { dropped_winner_count: 0, dropped_winner_reasons: Default::default(), }, + ts_eids_s: None, } .to_ext(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index cc4ee6abc1..28b587df95 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -38,7 +38,7 @@ use sha2::Digest as _; use web_time::Instant; use crate::auction::endpoints::{ - merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, + merge_auction_eids, mirror_member, resolve_auction_eids, resolve_client_auction_eids, }; use crate::auction::formats::sanitize_publisher_page_url; use crate::auction::orchestrator::{ @@ -7058,6 +7058,9 @@ pub async fn handle_page_bids( // unchanged) but skip the live auction, matching the existing behavior. let ad_stack_enabled = ad_templates_enabled && auction_enabled && consent_allows_auction; + // The row the auction read, for the `ts-eids-s` mirror. It stays + // `NotRead` when no auction runs, so the mirror is then absent. + let mut mirror_snapshot = crate::ec::EcKvSnapshot::NotRead; let (winning_bids, prebuilt_bid_map) = if matched_slots.is_empty() { (std::collections::HashMap::new(), None) } else { @@ -7090,9 +7093,9 @@ pub async fn handle_page_bids( _ => crate::ec::EcKvSnapshot::NotRead, }; // Hand the loaded row to the request context so response - // finalization — which runs on an EC context the adapter owns, - // after this handler returns — ingests `ts-eids`/`sharedId` updates - // from this read instead of paying for a second lookup. + // finalization, which runs on an EC context the adapter owns after + // this handler returns, reuses this read instead of paying for a + // second lookup. if !matches!(page_bids_kv_snapshot, crate::ec::EcKvSnapshot::NotRead) { ec_context.set_kv_snapshot(page_bids_kv_snapshot.clone()); } @@ -7118,6 +7121,7 @@ pub async fn handle_page_bids( path_label: "Page-bids", }, ); + mirror_snapshot = page_bids_kv_snapshot; let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); @@ -7229,10 +7233,18 @@ pub async fn handle_page_bids( Vec::new() }; - let body = serde_json::json!({ + let mut body = serde_json::json!({ "slots": slots_json, "bids": bid_map, }); + if let Some(mirror) = mirror_member( + &mirror_snapshot, + auction.registry, + ec_context, + PolicySignals::from_headers(req.headers()), + ) { + body["ts_eids_s"] = mirror; + } let body = serde_json::to_string(&body).change_context(TrustedServerError::Proxy { message: "Failed to serialize page-bids response".to_string(), })?; @@ -23932,6 +23944,7 @@ mod tests { use crate::platform::test_support::{StubHttpClient, noop_services}; use crate::platform::{PlatformHttpRequest, PlatformResponse}; use crate::test_support::tests::crate_test_settings_str; + use base64::Engine as _; use error_stack::{Report, ResultExt}; use http::Method; use std::sync::{Arc, Mutex}; @@ -24123,6 +24136,205 @@ mod tests { make_page_bids_request_on(PAGE_BIDS_PATH, path) } + const MIRROR_SOURCE: &str = "mirror.example.com"; + + fn mirror_registry() -> PartnerRegistry { + let mut partner = crate::ec::identity::test_support::ec_partner( + MIRROR_SOURCE, + "https://sync.mirror.example.com/pull", + "example-token-value-0123456789abcdef", + ); + partner.pull_sync_enabled = false; + partner.pull_sync_url = None; + partner.pull_sync_allowed_domains = Vec::new(); + partner.ts_pull_token = None; + partner.api_token = Some(crate::redacted::Redacted::new( + "token-mirror-example-com-32-bytes-minimum".to_owned(), + )); + PartnerRegistry::from_config(&[partner]).expect("should build partner registry") + } + + fn mirror_ec_id() -> String { + format!("{}.mirror", "a".repeat(64)) + } + + fn seeded_graph(records: &[(&str, &str)]) -> KvIdentityGraph { + let graph = KvIdentityGraph::in_memory("test_store"); + let mut entry = + crate::ec::kv_types::KvEntry::minimal("unregistered.example.com", "x", 1); + entry.ids.clear(); + for (source, uid) in records { + entry.ids.insert( + (*source).to_owned(), + crate::ec::kv_types::KvPartnerId::browser(*uid, Some(1)), + ); + } + graph + .create(&mirror_ec_id(), &entry) + .expect("should seed the live row"); + graph + } + + fn consenting_context_with_ec() -> EcContext { + EcContext::new_for_test( + Some(mirror_ec_id()), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }, + ) + } + + async fn page_bids_body( + settings: &Settings, + graph: &KvIdentityGraph, + ec_context: &mut EcContext, + req: Request, + ) -> serde_json::Value { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"no bid".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let orchestrator = + auction_id_test_orchestrator(settings, Arc::new(Mutex::new(None)), false); + let registry = mirror_registry(); + let slots = article_slot(); + let response = handle_page_bids( + settings, + &services, + Some(graph), + AuctionDispatch { + orchestrator: &orchestrator, + slots: &slots, + registry: Some(®istry), + }, + ec_context, + req, + ) + .await + .expect("should return ok response"); + serde_json::from_slice( + &response + .into_body() + .into_bytes() + .expect("should read the page-bids body"), + ) + .expect("should parse the page-bids body") + } + + fn auction_settings() -> Settings { + let mut settings = settings_with_co(); + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); + settings + } + + #[tokio::test] + async fn page_bids_never_writes_kv() { + let settings = auction_settings(); + let (graph, store) = KvIdentityGraph::write_counting("test_store"); + let mut entry = + crate::ec::kv_types::KvEntry::minimal("unregistered.example.com", "x", 1); + entry.ids.clear(); + graph + .create(&mirror_ec_id(), &entry) + .expect("should seed the live row"); + store.reset_counts(); + let cookie = base64::engine::general_purpose::STANDARD.encode( + serde_json::to_vec(&serde_json::json!([ + { "source": MIRROR_SOURCE, "uids": [{ "id": "cookie-uid" }] } + ])) + .expect("should serialize the cookie"), + ); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(&format!("ts-eids={cookie}")) + .expect("should build the cookie header"), + ); + let mut ec_context = consenting_context_with_ec(); + + let _ = page_bids_body(&settings, &graph, &mut ec_context, req).await; + let mut response = Response::new(EdgeBody::empty()); + crate::ec::finalize::ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &mirror_registry(), + Some(&cookie), + None, + &mut response, + ); + + let writes = store.writes(); + assert_eq!( + (writes.add, writes.if_generation_match, writes.overwrite), + (0, 0, 0), + "should write nothing to KV" + ); + assert!( + ec_context.identity().mutation().is_empty(), + "should queue nothing for a writer" + ); + } + + #[tokio::test] + async fn page_bids_returns_the_mirror_payload() { + let settings = auction_settings(); + let graph = seeded_graph(&[(MIRROR_SOURCE, "uid-mirror")]); + let mut ec_context = consenting_context_with_ec(); + + let body = page_bids_body( + &settings, + &graph, + &mut ec_context, + make_page_bids_request("/2024/01/my-article/"), + ) + .await; + + let mirror = &body["ts_eids_s"]; + let decoded: serde_json::Value = serde_json::from_slice( + &base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(mirror["value"].as_str().expect("should hold a value")) + .expect("should be unpadded base64url"), + ) + .expect("should be JSON"); + let atype = crate::settings::EcPartner::default_openrtb_atype(); + assert_eq!( + decoded, + serde_json::json!({ "v": 1, "eids": [{ + "source": MIRROR_SOURCE, + "uids": [{ "id": "uid-mirror", "atype": atype }] + }] }), + "should hold the servable record's EID" + ); + assert_eq!( + mirror["max_age"], 86_400, + "should live a day without an exp" + ); + } + + #[tokio::test] + async fn page_bids_without_an_auction_omits_the_mirror() { + let settings = settings_with_co_auction_disabled(); + let graph = seeded_graph(&[(MIRROR_SOURCE, "uid-mirror")]); + let mut ec_context = consenting_context_with_ec(); + + let body = page_bids_body( + &settings, + &graph, + &mut ec_context, + make_page_bids_request("/2024/01/my-article/"), + ) + .await; + + assert!( + body.get("ts_eids_s").is_none(), + "should leave the mirror cookie as it is" + ); + } + #[tokio::test] async fn page_bids_format_absent_or_json_returns_json() { let settings = settings_with_co(); diff --git a/crates/trusted-server-js/lib/src/core/eids_mirror.ts b/crates/trusted-server-js/lib/src/core/eids_mirror.ts new file mode 100644 index 0000000000..0f5f6c4e13 --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/eids_mirror.ts @@ -0,0 +1,61 @@ +// Writes the client-only `__Host-ts-eids-s` cookie from the mirror payload +// `/auction` and `/_ts/page-bids` return (spec 6.2). +// +// The cookie is read by page scripts, never by TS: the only read here is the +// comparison that skips an unchanged write. Nothing is copied into `ts-eids` +// or the `/auction` body, so a mirror ID never becomes a browser candidate. + +const MIRROR_COOKIE_NAME = '__Host-ts-eids-s'; + +// base64url without padding, the only alphabet the server emits. +const MIRROR_VALUE_PATTERN = /^[A-Za-z0-9_-]+$/; + +const ATTRIBUTES = 'Path=/; Secure; SameSite=Lax'; + +function currentMirrorValue(): string | undefined { + const prefix = `${MIRROR_COOKIE_NAME}=`; + for (const part of document.cookie.split(';')) { + const candidate = part.trim(); + if (candidate.startsWith(prefix)) { + return candidate.slice(prefix.length); + } + } + return undefined; +} + +/** + * Applies a mirror payload to `__Host-ts-eids-s`. + * + * `{ value, max_age }` writes the cookie unless it already holds `value`. + * `{ clear: true }` expires it. Anything else, an absent payload included, + * leaves the cookie as it is. + * + * @param payload the `ts_eids_s` member of an `/auction` or page-bids response + */ +export function writeEidsMirror(payload: unknown): void { + if (payload === null || typeof payload !== 'object') { + return; + } + const { value, max_age: maxAge, clear } = payload as Record; + try { + if (clear === true) { + document.cookie = `${MIRROR_COOKIE_NAME}=; ${ATTRIBUTES}; Max-Age=0`; + return; + } + if ( + typeof value !== 'string' || + !MIRROR_VALUE_PATTERN.test(value) || + typeof maxAge !== 'number' || + !Number.isInteger(maxAge) || + maxAge < 0 + ) { + return; + } + if (currentMirrorValue() === value) { + return; + } + document.cookie = `${MIRROR_COOKIE_NAME}=${value}; ${ATTRIBUTES}; Max-Age=${maxAge}`; + } catch { + // Cookie access can throw in sandboxed frames; the mirror is optional. + } +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index fc18beb5e9..3c56c7b1c6 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -6,6 +6,7 @@ import { releaseTrustedServerFirstImpressionClaim, reservePublisherFirstImpressionFallback, } from '../../core/first_impression'; +import { writeEidsMirror } from '../../core/eids_mirror'; import { log } from '../../core/log'; import { resolveSlotElementByDivId } from '../../core/slot_element'; import type { @@ -1490,6 +1491,8 @@ export function installTsAdInit(): void { interface PageBidsResponse { slots: AuctionSlot[]; bids: Record; + /** The `__Host-ts-eids-s` mirror payload; absent when the request read no KV. */ + ts_eids_s?: unknown; } /** Canonical SPA re-auction endpoint. Mirrors `PAGE_BIDS_PATH` in Rust. */ @@ -1717,6 +1720,12 @@ export function installSpaAuctionHook(): void { if (inflight === controller) currentPath = lastAppliedPath; return; } + // The mirror comes from the row this response read, so it is written + // even when a newer navigation has superseded the route. Absent means + // no KV read and the cookie stays as it is. + if (data.ts_eids_s !== undefined) { + writeEidsMirror(data.ts_eids_s); + } if (inflight !== controller) return; // Defer applying bids until the new route's ad containers exist, so a // fast edge response cannot beat the DOM and drop server-side bids. diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts b/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts index fd25b2e689..cb0dffeceb 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/eids.ts @@ -153,6 +153,13 @@ function cookieLabel(entry: unknown): string { return isRecord(entry) && typeof entry.source === 'string' ? entry.source : '(malformed entry)'; } +function isRegistered(entry: unknown, registeredSources: ReadonlySet): boolean { + if (!isRecord(entry) || typeof entry.source !== 'string') { + return false; + } + return registeredSources.has(entry.source.trim().toLowerCase().replace(/\.$/, '')); +} + function encodeUtf8Base64(text: string): string { let binary = ''; for (const byte of utf8Encoder.encode(text)) { @@ -171,11 +178,21 @@ function encodeUtf8Base64(text: string): string { * entry itself, and repeat. A last entry that is not an object with a `uids` * array is removed whole. The input is not modified. * + * Only the writer navigation ingests `ts-eids`, so when the set does not fit + * the entries of registered sources move ahead of the others and trimming + * reaches them last. Within each group the order is unchanged. A set that + * fits keeps its order. + * * @param eids the entries forwarded in the `/auction` body + * @param registeredSources the `[[ec.partners]]` source domains the server + * injected; empty keeps the plain tail trim * @returns the cookie value, or `undefined` when nothing fits, and the * sources trimming touched */ -export function fitEidCookie(eids: readonly unknown[]): FittedEidCookie { +export function fitEidCookie( + eids: readonly unknown[], + registeredSources: ReadonlySet = new Set() +): FittedEidCookie { const seen = new Set(); const payload: unknown[] = []; for (const entry of eids) { @@ -194,6 +211,7 @@ export function fitEidCookie(eids: readonly unknown[]): FittedEidCookie { const dropped = new Set(); const trimmed = new Set(); + let registeredFirst = false; while (payload.length > 0) { const value = encodeUtf8Base64(JSON.stringify(payload)); if (value.length <= MAX_EID_COOKIE_BYTES) { @@ -204,6 +222,13 @@ export function fitEidCookie(eids: readonly unknown[]): FittedEidCookie { }; } + if (!registeredFirst) { + registeredFirst = true; + const registered = payload.filter((entry) => isRegistered(entry, registeredSources)); + const others = payload.filter((entry) => !isRegistered(entry, registeredSources)); + payload.splice(0, payload.length, ...registered, ...others); + } + const last = payload[payload.length - 1]; if (isRecord(last) && Array.isArray(last.uids) && last.uids.length > 1) { last.uids = last.uids.slice(0, -1); diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index aec08160bb..9dd82a872c 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -24,6 +24,7 @@ import { } from '../../core/first_impression'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; +import { writeEidsMirror } from '../../core/eids_mirror'; import { registerApsPrebidRenderer, validateApsRenderer } from '../aps/render'; import type { AuctionBid } from '../../core/auction'; import type { AuctionSlot, TsjsApi } from '../../core/types'; @@ -189,6 +190,8 @@ interface InjectedPrebidConfig { excludedGamAdUnitPathSuffixes?: string[]; /** Operator-owned Prebid User ID module entries, forwarded verbatim. */ managedUserIds?: InjectedManagedUserId[]; + /** `[[ec.partners]]` source domains the `ts-eids` trim keeps first. */ + registeredSources?: string[]; } /** @@ -2320,6 +2323,11 @@ export function installPrebidNpm(config?: Partial): typeof pbjs ) { const body = serverResponse?.body; log.debug('[tsjs-prebid] interpretResponse', { hasSeatbid: !!body?.seatbid }); + // The mirror is client-only state for page scripts; absent means the + // request read no KV and the cookie stays as it is. + if (body?.ext?.ts_eids_s !== undefined) { + writeEidsMirror(body.ext.ts_eids_s); + } const auctionBids = parseAuctionResponse(body); const bidRequests = request?.tsjsBidRequests ?? request?.bidRequests ?? []; return auctionBidsToPrebidBids(auctionBids, bidRequests, apsRendererSupported); @@ -2874,7 +2882,7 @@ function logTrimmedEidSources(droppedSources: string[], trimmedSources: string[] */ function writePrebidEidsCookie(eids: unknown[]): EidCookieDecision { try { - const fitted = fitEidCookie(eids); + const fitted = fitEidCookie(eids, new Set(getInjectedConfig()?.registeredSources ?? [])); logTrimmedEidSources(fitted.droppedSources, fitted.trimmedSources); if (fitted.value === undefined) { clearPrebidEidsCookie(); diff --git a/crates/trusted-server-js/lib/test/core/eids_mirror.test.ts b/crates/trusted-server-js/lib/test/core/eids_mirror.test.ts new file mode 100644 index 0000000000..3f64bbf822 --- /dev/null +++ b/crates/trusted-server-js/lib/test/core/eids_mirror.test.ts @@ -0,0 +1,81 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { writeEidsMirror } from '../../src/core/eids_mirror'; + +const NAME = '__Host-ts-eids-s'; + +describe('core/eids_mirror', () => { + let writes: string[]; + let stored: string; + + beforeEach(() => { + writes = []; + stored = ''; + Object.defineProperty(document, 'cookie', { + configurable: true, + get: () => stored, + set: (assignment: string) => { + writes.push(assignment); + }, + }); + }); + + afterEach(() => { + delete (document as unknown as { cookie?: string }).cookie; + vi.restoreAllMocks(); + }); + + it('writes the value with the given Max-Age and the __Host- attributes', () => { + writeEidsMirror({ value: 'eyJ2IjoxfQ', max_age: 600 }); + + expect(writes).toEqual([`${NAME}=eyJ2IjoxfQ; Path=/; Secure; SameSite=Lax; Max-Age=600`]); + }); + + it('skips the write when the cookie already holds that value', () => { + stored = `other=1; ${NAME}=eyJ2IjoxfQ; ts-eids=abc`; + + writeEidsMirror({ value: 'eyJ2IjoxfQ', max_age: 600 }); + + expect(writes).toEqual([]); + }); + + it('rewrites when the stored value differs', () => { + stored = `${NAME}=old`; + + writeEidsMirror({ value: 'new', max_age: 60 }); + + expect(writes).toHaveLength(1); + }); + + it('expires the cookie on clear', () => { + writeEidsMirror({ clear: true }); + + expect(writes).toEqual([`${NAME}=; Path=/; Secure; SameSite=Lax; Max-Age=0`]); + }); + + it.each([ + ['absent', undefined], + ['null', null], + ['a string', 'eyJ2IjoxfQ'], + ['an empty object', {}], + ['a clear that is not true', { clear: 'yes' }], + ['a value without max_age', { value: 'abc' }], + ['a non-numeric max_age', { value: 'abc', max_age: '60' }], + ['a negative max_age', { value: 'abc', max_age: -1 }], + ['a value with cookie separators', { value: 'a; Domain=example.com', max_age: 60 }], + ['an empty value', { value: '', max_age: 60 }], + ])('does nothing for %s', (_label, payload) => { + writeEidsMirror(payload); + + expect(writes).toEqual([]); + }); + + it('never touches ts-eids', () => { + stored = 'ts-eids=abc'; + + writeEidsMirror({ value: 'abc', max_age: 60 }); + writeEidsMirror({ clear: true }); + + expect(writes.every((write) => write.startsWith(`${NAME}=`))).toBe(true); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 647de8967b..5af2f84b3c 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -5,6 +5,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import type { TsjsApi } from '../../../src/core/types'; import { GPT_BOOTSTRAP_PATH } from '../../fixtures/paths'; +const { mockWriteEidsMirror } = vi.hoisted(() => ({ mockWriteEidsMirror: vi.fn() })); + +vi.mock('../../../src/core/eids_mirror', () => ({ writeEidsMirror: mockWriteEidsMirror })); + type TestWindow = Window & { googletag?: unknown; tsjs?: TsjsApi; @@ -38,6 +42,7 @@ describe('installSpaAuctionHook', () => { const realAddEventListener = window.addEventListener.bind(window); beforeEach(() => { + mockWriteEidsMirror.mockClear(); vi.resetModules(); delete (window as TestWindow).tsjs; // Restore unwrapped history methods so each module import wraps exactly @@ -212,6 +217,50 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('writes the ts-eids-s mirror from a page-bids response that carries one', async () => { + const mirror = { value: 'eyJ2IjoxfQ', max_age: 600 }; + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {}, ts_eids_s: mirror }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + + history.pushState({}, '', '/mirrored-route'); + await flushAsync(); + + expect(mockWriteEidsMirror).toHaveBeenCalledTimes(1); + expect(mockWriteEidsMirror).toHaveBeenCalledWith(mirror); + }); + + it('passes a clear payload through to the mirror writer', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {}, ts_eids_s: { clear: true } }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + + history.pushState({}, '', '/cleared-route'); + await flushAsync(); + + expect(mockWriteEidsMirror).toHaveBeenCalledWith({ clear: true }); + }); + + it('leaves the mirror alone when the page-bids response has none', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + + history.pushState({}, '', '/unmirrored-route'); + await flushAsync(); + + expect(mockWriteEidsMirror).not.toHaveBeenCalled(); + }); + it('skips adInit on an empty page-bids response with no prior TS state', async () => { // A gated page-bids response (template switch, auction gate, or consent // denial) returns no slots. With no prior TS state to sweep, the hook must diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts index 296d85e3f1..05ba347366 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/eids.test.ts @@ -265,6 +265,81 @@ describe('collectBidderEids', () => { }); describe('fitEidCookie', () => { + describe('registered sources', () => { + const long = 'x'.repeat(400); + const entry = (source: string) => ({ source, uids: [{ id: long, atype: 1 }] }); + + it('keeps_registered_sources_before_unregistered_ones', () => { + // Eight entries are over the cookie cap; the registered source comes last. + const unregistered = [0, 1, 2, 3, 4, 5, 6].map((index) => entry(`u${index}.example.com`)); + const registered = entry('registered.example.com'); + + const fitted = fitEidCookie( + [...unregistered, registered], + new Set(['registered.example.com']) + ); + + const sources = (decodeCookie(fitted.value) as Array<{ source: string }>).map( + (item) => item.source + ); + expect(sources[0]).toBe('registered.example.com'); + expect(sources).not.toContain('u6.example.com'); + expect(fitted.droppedSources).toContain('u6.example.com'); + expect(fitted.droppedSources).not.toContain('registered.example.com'); + expect(fitted.value!.length).toBeLessThanOrEqual(MAX_EID_COOKIE_BYTES); + }); + + it('matches registered sources after lowercasing and removing a trailing dot', () => { + const unregistered = [0, 1, 2, 3, 4, 5, 6].map((index) => entry(`u${index}.example.com`)); + + const fitted = fitEidCookie( + [...unregistered, entry('Registered.Example.com.')], + new Set(['registered.example.com']) + ); + + expect(fitted.droppedSources).not.toContain('Registered.Example.com.'); + }); + + it('keeps the trim order within each group', () => { + const registered = [0, 1, 2, 3, 4, 5, 6].map((index) => entry(`r${index}.example.com`)); + const unregistered = [0, 1].map((index) => entry(`u${index}.example.com`)); + + const fitted = fitEidCookie( + [...registered.slice(0, 3), unregistered[0], ...registered.slice(3), unregistered[1]], + new Set(registered.map((item) => item.source)) + ); + + // Unregistered sources go first, last-first, then registered ones from the tail. + expect(fitted.droppedSources).toEqual([ + 'u1.example.com', + 'u0.example.com', + 'r6.example.com', + 'r5.example.com', + ]); + }); + + it('keeps the input order when the set fits', () => { + const small = [ + { source: 'u.example.com', uids: [{ id: 'u' }] }, + { source: 'r.example.com', uids: [{ id: 'r' }] }, + ]; + + const fitted = fitEidCookie(small, new Set(['r.example.com'])); + + expect(decodeCookie(fitted.value)).toEqual(small); + }); + + it('trims as before when no source is registered', () => { + const entries = [0, 1, 2, 3, 4, 5, 6].map((index) => entry(`s${index}.example.com`)); + + expect(fitEidCookie(entries, new Set())).toEqual(fitEidCookie(entries)); + expect(fitEidCookie(entries, new Set()).droppedSources).toEqual([ + 's6.example.com', + 's5.example.com', + ]); + }); + }); + it('removes EID-level ext and drops the duplicates that creates, comparing with sorted keys', () => { const withExt = { source: 'a.example.com', diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 34b7ea3239..23cf90d48a 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -235,6 +235,10 @@ interface TestAdapterSpec { ) => Array>; } +const { mockWriteEidsMirror } = vi.hoisted(() => ({ mockWriteEidsMirror: vi.fn() })); + +vi.mock('../../../src/core/eids_mirror', () => ({ writeEidsMirror: mockWriteEidsMirror })); + // Define mocks using vi.hoisted so they exist before the module under test is // imported. The shim reads Prebid.js from the `window.pbjs` global (owned by // the external bundle in production), so tests install the mock there instead @@ -2928,6 +2932,59 @@ describe('prebid/installPrebidNpm', () => { ); }); + it('interpretResponse writes the ts-eids-s mirror the response carries', () => { + mockWriteEidsMirror.mockClear(); + const spec = getAdapterSpec(); + const built = spec.buildRequests([]); + const mirror = { value: 'eyJ2IjoxfQ', max_age: 600 }; + + spec.interpretResponse({ body: { seatbid: [], ext: { ts_eids_s: mirror } } }, built); + spec.interpretResponse({ body: { seatbid: [], ext: { ts_eids_s: { clear: true } } } }, built); + + expect(mockWriteEidsMirror).toHaveBeenCalledTimes(2); + expect(mockWriteEidsMirror).toHaveBeenNthCalledWith(1, mirror); + expect(mockWriteEidsMirror).toHaveBeenNthCalledWith(2, { clear: true }); + }); + + it('interpretResponse leaves the mirror alone when the response has none', () => { + mockWriteEidsMirror.mockClear(); + const spec = getAdapterSpec(); + const built = spec.buildRequests([]); + + spec.interpretResponse({ body: { seatbid: [], ext: {} } }, built); + spec.interpretResponse({ body: { seatbid: [] } }, built); + spec.interpretResponse({ body: null }, built); + + expect(mockWriteEidsMirror).not.toHaveBeenCalled(); + }); + + it('keeps injected registered sources in ts-eids when the set must be trimmed', () => { + testWindow.__tsjs_prebid = { registeredSources: ['registered.example.com'] }; + try { + const spec = getAdapterSpec(); + const long = 'x'.repeat(400); + const unregistered = [0, 1, 2, 3, 4, 5, 6].map((index) => ({ + source: `u${index}.example.com`, + uids: [{ id: long, atype: 1 }], + })); + const registered = { source: 'registered.example.com', uids: [{ id: long, atype: 1 }] }; + + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [...unregistered, registered] } }) + ); + + expect(JSON.parse(String(result.data)).eids, 'the body keeps every entry').toHaveLength(8); + const sources = (decodedTsEidsCookie() as Array<{ source: string }>).map( + (entry) => entry.source + ); + expect(sources[0]).toBe('registered.example.com'); + expect(sources).not.toContain('u6.example.com'); + } finally { + delete testWindow.__tsjs_prebid; + } + }); + it('interpretResponse handles empty/missing seatbid', () => { const spec = getAdapterSpec(); const built = spec.buildRequests([]); From 1de9f80e37ba20b1cd88de26314d99505087e22d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 22:47:17 +0530 Subject: [PATCH 37/89] Retry a failed withdrawal tombstone once after send A tombstone whose five CAS attempts fail before send runs the same path once more after send, by which time the per-key write limit has usually passed. A second failure is logged and counted. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 90 +++++- crates/trusted-server-core/src/ec/finalize.rs | 266 +++++++++++++++++- .../src/ec/identity/test_support.rs | 27 +- crates/trusted-server-core/src/ec/mod.rs | 23 ++ 4 files changed, 383 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 8c7ecff2a0..c8293f05f8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -14,7 +14,7 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; use trusted_server_core::ec::device::DeviceSignals; -use trusted_server_core::ec::finalize::ec_finalize_response; +use trusted_server_core::ec::finalize::{ec_finalize_response, retry_failed_withdrawals}; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, @@ -514,7 +514,7 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re request_filter_effects.as_ref(), counters.as_ref(), ); - run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + run_edgezero_after_send(settings, &partner_registry, &mut ec_state); return; } Err(e) => { @@ -544,11 +544,7 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re request_filter_effects.as_ref(), counters.as_ref(), ); - run_edgezero_pull_sync_after_send( - &settings, - &partner_registry, - &ec_state, - ); + run_edgezero_after_send(&settings, &partner_registry, &mut ec_state); return; } Err(e) => { @@ -626,6 +622,44 @@ fn apply_edgezero_ec_finalize( Ok(partner_registry) } +/// Runs the work that follows the response: the withdrawal retry, then pull sync. +fn run_edgezero_after_send( + settings: &Settings, + partner_registry: &PartnerRegistry, + ec_state: &mut EcFinalizeState, +) { + run_after_send_in_order( + ec_state, + |state| !state.ec_context.failed_withdrawals().is_empty(), + |state| match require_identity_graph(settings) { + Ok(graph) => retry_failed_withdrawals(&mut state.ec_context, &graph), + Err(err) => { + log::warn!( + "withdrawal retry skipped: identity graph unavailable: {}", + err.current_context() + ); + } + }, + |state| run_edgezero_pull_sync_after_send(settings, partner_registry, state), + ); +} + +/// Runs the withdrawal retry, when one is due, before any identity work. +/// +/// A withdrawal request runs no provider dispatch or mutation flush, so the +/// order only keeps the retry's log lines ahead of the identity work's. +fn run_after_send_in_order( + state: &mut S, + has_failed_withdrawal: impl FnOnce(&S) -> bool, + retry_withdrawals: impl FnOnce(&mut S), + identity_work: impl FnOnce(&S), +) { + if has_failed_withdrawal(state) { + retry_withdrawals(state); + } + identity_work(state); +} + fn run_edgezero_pull_sync_after_send( settings: &Settings, partner_registry: &PartnerRegistry, @@ -888,6 +922,48 @@ mod tests { .expect("should parse test settings") } + #[test] + fn after_send_runs_the_withdrawal_retry_before_identity_work() { + let log_cell = std::cell::RefCell::new(Vec::<&str>::new()); + let mut state = true; + + run_after_send_in_order( + &mut state, + |failed| *failed, + |failed| { + log_cell.borrow_mut().push("retry"); + *failed = false; + }, + |failed| { + assert!(!failed, "should see the state the retry left"); + log_cell.borrow_mut().push("identity"); + }, + ); + + assert_eq!( + *log_cell.borrow(), + ["retry", "identity"], + "should retry the withdrawal before identity work" + ); + } + + #[test] + fn after_send_skips_the_retry_without_a_failed_withdrawal() { + let retried = std::cell::Cell::new(false); + let worked = std::cell::Cell::new(false); + let mut state = false; + + run_after_send_in_order( + &mut state, + |failed| *failed, + |_| retried.set(true), + |_| worked.set(true), + ); + + assert!(!retried.get(), "should not retry when nothing failed"); + assert!(worked.get(), "should still run identity work"); + } + #[test] fn pull_sync_noop_states_skip_post_send_graph_factory() { let calls = std::cell::Cell::new(0); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index b761e238c3..8b0f8f10c9 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -547,35 +547,66 @@ fn finalize_unusable_consent( // for subsequent EC behavior. if let Some(graph) = kv { apply_withdrawal_tombstones(&ids_to_withdraw, |ec_id| { - // The graph hands back the post-withdrawal snapshot rather than - // leaving the caller to rebuild it, so post-send work that reads - // the context — pull sync discloses the raw EC ID to partners — - // sees the tombstone that was just written. Only the active ID has - // a snapshot in the context to correct. - let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { - ec_context.kv_snapshot().clone() - } else { - EcKvSnapshot::NotRead - }; - let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); // The browser cookie is already cleared, so a failed tombstone // leaves a live row that server-side consumers still read as // consented. Report every failure, including the non-active cookie // ID whose outcome is not retained on the request context. - if matches!(outcome, EcKvSnapshot::Failed { .. }) { + if !tombstone_withdrawn_id(ec_context, graph, ec_id) { log::warn!( "EC withdrawal tombstone failed for '{}': the identity-graph row may \ still be live with consent granted", log_id(ec_id) ); - } - if ec_context.ec_value() == Some(ec_id) { - ec_context.set_kv_snapshot(outcome); + ec_context.record_failed_withdrawal(ec_id); } }); } } +/// Runs the tombstone path once for `ec_id` and returns whether it succeeded. +/// +/// The graph hands back the post-withdrawal snapshot rather than leaving the +/// caller to rebuild it, so post-send work that reads the context — pull sync +/// discloses the raw EC ID to partners — sees the tombstone that was just +/// written. Only the active ID has a snapshot in the context to correct. +fn tombstone_withdrawn_id( + ec_context: &mut EcContext, + graph: &KvIdentityGraph, + ec_id: &str, +) -> bool { + let initial = if ec_context.kv_snapshot().belongs_to(ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead + }; + let outcome = graph.tombstone_existing_from_snapshot(ec_id, initial); + let succeeded = !matches!(outcome, EcKvSnapshot::Failed { .. }); + if ec_context.ec_value() == Some(ec_id) { + ec_context.set_kv_snapshot(outcome); + } + succeeded +} + +/// Retries each withdrawal tombstone that failed before send, once, after send. +/// +/// Each failed ID re-reads its row (a [`EcKvSnapshot::NotRead`] start) and runs +/// the same tombstone path as before send, with its own CAS attempts and +/// completion marker. This is the last attempt: a second failure is logged and +/// the ID stays in [`EcContext::failed_withdrawals`] for the failure counters. +/// An ID that was never recorded as failed is not touched. +pub fn retry_failed_withdrawals(ec_context: &mut EcContext, graph: &KvIdentityGraph) { + let failed = ec_context.failed_withdrawals().to_vec(); + for ec_id in &failed { + if !tombstone_withdrawn_id(ec_context, graph, ec_id) { + log::warn!( + "EC withdrawal tombstone failed again after send for '{}': the identity-graph \ + row may still be live with consent granted", + log_id(ec_id) + ); + } + } +} + fn withdrawal_ec_ids(ec_context: &EcContext) -> HashSet { let mut hashes = HashSet::new(); @@ -2993,6 +3024,211 @@ mod tests { } } + mod after_send_withdrawal_retry { + use super::*; + use crate::ec::kv::MAX_CAS_RETRIES; + + fn withdrawn_context(active: &str, cookie: &str) -> EcContext { + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + make_context_with_consent(Some(active), Some(cookie), true, false, consent) + } + + fn seed(graph: &KvIdentityGraph, ec_id: &str) { + graph + .create(ec_id, &KvEntry::minimal("p.example.com", "uid", 1_000)) + .expect("should seed the row"); + } + + fn finalize(context: &mut EcContext, graph: &KvIdentityGraph) { + let mut response = empty_response(); + ec_finalize_response( + &create_test_settings(), + context, + Some(graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + } + + fn is_tombstone(graph: &KvIdentityGraph, ec_id: &str) -> bool { + graph + .get(ec_id) + .expect("should read the row") + .is_some_and(|(entry, _)| !entry.consent.ok) + } + + #[test] + fn failed_withdrawal_is_recorded_for_after_send() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = sample_ec_id("fail01"); + seed(&graph, &ec_id); + store.refuse_conditional_writes(&ec_id, usize::MAX); + let mut context = withdrawn_context(&ec_id, &ec_id); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + + finalize(&mut context, &graph); + + assert_eq!( + context.failed_withdrawals(), + std::slice::from_ref(&ec_id), + "should record the active ID whose tombstone failed" + ); + assert!( + !is_tombstone(&graph, &ec_id), + "the row should still be live" + ); + } + + #[test] + fn successful_withdrawal_records_nothing() { + let (graph, _store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = sample_ec_id("good01"); + seed(&graph, &ec_id); + let mut context = withdrawn_context(&ec_id, &ec_id); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + + finalize(&mut context, &graph); + + assert!( + context.failed_withdrawals().is_empty(), + "should record nothing for a tombstone that was written" + ); + assert!(is_tombstone(&graph, &ec_id), "should tombstone the row"); + } + + #[test] + fn nothing_failed_means_the_retry_touches_nothing() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = sample_ec_id("none01"); + seed(&graph, &ec_id); + store.reset_counts(); + let mut context = withdrawn_context(&ec_id, &ec_id); + + retry_failed_withdrawals(&mut context, &graph); + + assert_eq!( + store.writes().total(), + 0, + "should write nothing of any mode" + ); + assert_eq!(store.writes().lookups, 0, "should read nothing"); + } + + #[test] + fn after_send_retry_writes_the_tombstone_once() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = sample_ec_id("retry1"); + seed(&graph, &ec_id); + store.refuse_conditional_writes(&ec_id, MAX_CAS_RETRIES as usize); + let mut context = withdrawn_context(&ec_id, &ec_id); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + finalize(&mut context, &graph); + assert_eq!( + context.failed_withdrawals(), + std::slice::from_ref(&ec_id), + "should record the failure before send" + ); + store.reset_counts(); + + retry_failed_withdrawals(&mut context, &graph); + + assert!(is_tombstone(&graph, &ec_id), "should read as a tombstone"); + let writes = store.writes(); + assert_eq!( + writes.if_generation_match, 1, + "should write the tombstone once" + ); + assert_eq!(writes.add, 1, "should write the completion marker once"); + assert_eq!(writes.overwrite, 0, "should not overwrite"); + assert!( + context + .kv_snapshot() + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should update the active snapshot to the tombstone" + ); + } + + #[test] + fn after_send_retry_is_the_last_attempt() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let ec_id = sample_ec_id("last01"); + seed(&graph, &ec_id); + store.refuse_conditional_writes(&ec_id, usize::MAX); + let mut context = withdrawn_context(&ec_id, &ec_id); + context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + finalize(&mut context, &graph); + store.reset_counts(); + + retry_failed_withdrawals(&mut context, &graph); + + let writes = store.writes(); + assert_eq!( + writes.if_generation_match, MAX_CAS_RETRIES as usize, + "should make one run of the CAS attempts, never a second retry" + ); + assert_eq!(writes.add, 0, "should write no completion marker"); + assert_eq!(writes.overwrite, 0, "should not overwrite"); + assert_eq!( + context.failed_withdrawals(), + std::slice::from_ref(&ec_id), + "should leave the failure recorded for the counters" + ); + assert!( + !is_tombstone(&graph, &ec_id), + "the row should still be live" + ); + } + + #[test] + fn withdrawal_of_two_ids_retries_only_the_failed_one() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let active = sample_ec_id("activ4"); + let cookie = sample_ec_id("cook4e"); + seed(&graph, &active); + seed(&graph, &cookie); + store.refuse_conditional_writes(&cookie, MAX_CAS_RETRIES as usize); + let mut context = withdrawn_context(&active, &cookie); + context.set_kv_snapshot(graph.load_snapshot(&active)); + finalize(&mut context, &graph); + assert_eq!( + context.failed_withdrawals(), + std::slice::from_ref(&cookie), + "should record only the cookie ID" + ); + assert!( + is_tombstone(&graph, &active), + "the active row should be withdrawn" + ); + let active_generation = graph.get(&active).expect("should read").map(|row| row.1); + store.reset_counts(); + + retry_failed_withdrawals(&mut context, &graph); + + assert!( + is_tombstone(&graph, &cookie), + "should withdraw the cookie row" + ); + assert_eq!( + graph.get(&active).expect("should read").map(|row| row.1), + active_generation, + "should not rewrite the active row" + ); + assert_eq!( + store.writes().if_generation_match, + 1, + "should write one tombstone, for the cookie ID only" + ); + } + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 785e28c346..3bfb68f169 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -10,8 +10,8 @@ use std::collections::{BTreeMap, BTreeSet}; use std::net::IpAddr; -use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use error_stack::Report; use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; @@ -396,6 +396,7 @@ pub struct WriteCountingEcKv { if_generation_match: AtomicUsize, overwrite: AtomicUsize, lookups: AtomicUsize, + refusals: Mutex>, } impl WriteCountingEcKv { @@ -408,9 +409,24 @@ impl WriteCountingEcKv { if_generation_match: AtomicUsize::new(0), overwrite: AtomicUsize::new(0), lookups: AtomicUsize::new(0), + refusals: Mutex::new(BTreeMap::new()), } } + /// Refuses the next `count` generation-checked writes to `key` with + /// [`EcKvWriteOutcome::PreconditionFailed`], the retryable refusal. A + /// refused write is still counted. + /// + /// # Panics + /// + /// Panics when the refusal table is poisoned. + pub fn refuse_conditional_writes(&self, key: &str, count: usize) { + self.refusals + .lock() + .expect("should lock the refusals") + .insert(key.to_owned(), count); + } + /// The counts so far, seeding included. #[must_use] pub fn writes(&self) -> WriteCounts { @@ -460,6 +476,15 @@ impl EcKvStore for WriteCountingEcKv { EcKvWriteMode::Overwrite => &self.overwrite, }; count.fetch_add(1, Ordering::SeqCst); + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + let mut refusals = self.refusals.lock().expect("should lock the refusals"); + if let Some(remaining) = refusals.get_mut(key) + && *remaining > 0 + { + *remaining -= 1; + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } self.inner.insert(key, write) } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 1a9391edd2..bb21ca51aa 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -286,6 +286,9 @@ pub struct EcContext { eid_sync_source: Option, /// Identity state this request decides and collects. identity: RequestIdentity, + /// Withdrawn EC IDs whose before-send tombstone returned + /// [`EcKvSnapshot::Failed`], for the after-send retry. + failed_withdrawals: Vec, } impl EcContext { @@ -368,6 +371,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, identity: RequestIdentity::default(), + failed_withdrawals: Vec::new(), }) } @@ -616,6 +620,22 @@ impl EcContext { &self.kv_snapshot } + /// Returns the withdrawn EC IDs whose before-send tombstone failed. + /// + /// The after-send retry reads these, and the failure counters read them + /// after the retry, so the retry does not clear them. + #[must_use] + pub fn failed_withdrawals(&self) -> &[String] { + &self.failed_withdrawals + } + + /// Records a withdrawn EC ID whose before-send tombstone failed. + pub(crate) fn record_failed_withdrawal(&mut self, ec_id: &str) { + if !self.failed_withdrawals.iter().any(|id| id == ec_id) { + self.failed_withdrawals.push(ec_id.to_owned()); + } + } + /// Replaces the request-scoped identity-graph snapshot. pub fn set_kv_snapshot(&mut self, snapshot: EcKvSnapshot) { self.kv_snapshot = snapshot; @@ -732,6 +752,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), + failed_withdrawals: Vec::new(), } } @@ -757,6 +778,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), + failed_withdrawals: Vec::new(), } } @@ -785,6 +807,7 @@ impl EcContext { pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), + failed_withdrawals: Vec::new(), } } } From 8ce81cf08544b734794eee3b80342d91f33f28b6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 22:50:49 +0530 Subject: [PATCH 38/89] Clear a withdrawal from the failed list once its retry succeeds After the after-send retry the context holds only the IDs that failed twice, so the failure counter can count second failures alone. The adapter asks has_failed_withdrawals instead of reading the list. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 2 +- crates/trusted-server-core/src/ec/finalize.rs | 52 +++++++++++++++++-- crates/trusted-server-core/src/ec/mod.rs | 21 ++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index c8293f05f8..ca0e909dfc 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -630,7 +630,7 @@ fn run_edgezero_after_send( ) { run_after_send_in_order( ec_state, - |state| !state.ec_context.failed_withdrawals().is_empty(), + |state| state.ec_context.has_failed_withdrawals(), |state| match require_identity_graph(settings) { Ok(graph) => retry_failed_withdrawals(&mut state.ec_context, &graph), Err(err) => { diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 8b0f8f10c9..5c159cdd89 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -591,13 +591,16 @@ fn tombstone_withdrawn_id( /// /// Each failed ID re-reads its row (a [`EcKvSnapshot::NotRead`] start) and runs /// the same tombstone path as before send, with its own CAS attempts and -/// completion marker. This is the last attempt: a second failure is logged and -/// the ID stays in [`EcContext::failed_withdrawals`] for the failure counters. +/// completion marker. This is the last attempt: a success clears the ID from +/// the context's failed list, and a second failure is logged and leaves it +/// there for the failure counters. /// An ID that was never recorded as failed is not touched. pub fn retry_failed_withdrawals(ec_context: &mut EcContext, graph: &KvIdentityGraph) { let failed = ec_context.failed_withdrawals().to_vec(); for ec_id in &failed { - if !tombstone_withdrawn_id(ec_context, graph, ec_id) { + if tombstone_withdrawn_id(ec_context, graph, ec_id) { + ec_context.clear_failed_withdrawal(ec_id); + } else { log::warn!( "EC withdrawal tombstone failed again after send for '{}': the identity-graph \ row may still be live with consent granted", @@ -3140,6 +3143,10 @@ mod tests { retry_failed_withdrawals(&mut context, &graph); assert!(is_tombstone(&graph, &ec_id), "should read as a tombstone"); + assert!( + context.failed_withdrawals().is_empty(), + "should clear an ID whose retry succeeded" + ); let writes = store.writes(); assert_eq!( writes.if_generation_match, 1, @@ -3187,6 +3194,41 @@ mod tests { ); } + #[test] + fn only_the_id_that_fails_twice_remains_after_the_retry() { + let (graph, store) = KvIdentityGraph::write_counting("ec-store"); + let active = sample_ec_id("activ5"); + let cookie = sample_ec_id("cook5e"); + seed(&graph, &active); + seed(&graph, &cookie); + store.refuse_conditional_writes(&cookie, MAX_CAS_RETRIES as usize); + store.refuse_conditional_writes(&active, usize::MAX); + let mut context = withdrawn_context(&active, &cookie); + context.set_kv_snapshot(graph.load_snapshot(&active)); + finalize(&mut context, &graph); + assert_eq!( + context.failed_withdrawals().len(), + 2, + "should record both IDs before send" + ); + + retry_failed_withdrawals(&mut context, &graph); + + assert_eq!( + context.failed_withdrawals(), + std::slice::from_ref(&active), + "should keep only the ID that failed twice" + ); + assert!( + is_tombstone(&graph, &cookie), + "should withdraw the cookie row" + ); + assert!( + !is_tombstone(&graph, &active), + "the active row should be live" + ); + } + #[test] fn withdrawal_of_two_ids_retries_only_the_failed_one() { let (graph, store) = KvIdentityGraph::write_counting("ec-store"); @@ -3216,6 +3258,10 @@ mod tests { is_tombstone(&graph, &cookie), "should withdraw the cookie row" ); + assert!( + context.failed_withdrawals().is_empty(), + "should hold no ID once the only failed one succeeds" + ); assert_eq!( graph.get(&active).expect("should read").map(|row| row.1), active_generation, diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index bb21ca51aa..7ef46fb0db 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -620,15 +620,28 @@ impl EcContext { &self.kv_snapshot } - /// Returns the withdrawn EC IDs whose before-send tombstone failed. + /// Returns the withdrawn EC IDs whose tombstone has failed and not yet + /// succeeded. /// - /// The after-send retry reads these, and the failure counters read them - /// after the retry, so the retry does not clear them. + /// Before the after-send retry these are the before-send failures. After + /// it, only the IDs that failed a second time remain, which is what the + /// failure counters count. #[must_use] - pub fn failed_withdrawals(&self) -> &[String] { + pub(crate) fn failed_withdrawals(&self) -> &[String] { &self.failed_withdrawals } + /// Whether any withdrawn EC ID's tombstone is still unwritten. + #[must_use] + pub fn has_failed_withdrawals(&self) -> bool { + !self.failed_withdrawals.is_empty() + } + + /// Forgets a withdrawn EC ID whose retried tombstone was written. + pub(crate) fn clear_failed_withdrawal(&mut self, ec_id: &str) { + self.failed_withdrawals.retain(|id| id != ec_id); + } + /// Records a withdrawn EC ID whose before-send tombstone failed. pub(crate) fn record_failed_withdrawal(&mut self, ec_id: &str) { if !self.failed_withdrawals.iter().any(|id| id == ec_id) { From 98c543428a14418fea776f7e4c418ad91d720e20 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 23:05:15 +0530 Subject: [PATCH 39/89] Merge the server identity set into auction EIDs and apply provider denials A server value goes out as its whole entry set, one EID per stored entry, from KV only. A provider's provenance default applies only to an entry that sets no EID-level field. Browser candidates lose each UID a claiming provider invalidated or removed on opt-out, and every source of an opted-out owner or capture-only provider. Browser entries and one-UID records otherwise go out exactly as before. Servability is resolved once per request, and the bidstream and the ts-eids-s mirror read the same server set. Page-bids clears the mirror on a consent denial, as /auction does. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 742 +++++++++++++++--- crates/trusted-server-core/src/ec/eids.rs | 243 +++++- .../src/ec/identity/catalog.rs | 4 - .../src/ec/identity/denials.rs | 311 ++++++++ .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/test_support.rs | 28 +- crates/trusted-server-core/src/publisher.rs | 221 +++++- 7 files changed, 1401 insertions(+), 149 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/denials.rs diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 0e80c9c041..0135af99d3 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -9,7 +9,9 @@ use serde_json::value::RawValue; use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; +use crate::consent::{ + allows_eid_persistence, consent_allows_server_side_auction, gate_eids_by_consent, +}; use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; @@ -17,10 +19,11 @@ use crate::ec::EcKvSnapshot; use crate::ec::checked_current_timestamp; use crate::ec::client_eids::parse_client_eids; use crate::ec::current_timestamp; -use crate::ec::eids::{ResolvedPartnerId, resolve_partner_ids, to_eids}; +use crate::ec::eids::{ServerSet, server_set}; use crate::ec::identity::IdentityRuntime; use crate::ec::identity::UnixTime; use crate::ec::identity::catalog::ProviderCatalog; +use crate::ec::identity::denials::{Denials, denials, filter_browser_candidates}; use crate::ec::identity::mirror::{MirrorPayload, MirrorRead, MirrorSource, mirror_payload}; use crate::ec::identity::policy::PolicySignals; use crate::ec::identity::request::opted_out; @@ -331,8 +334,8 @@ pub async fn handle_auction( ec_context.set_kv_snapshot(auction_kv_snapshot.clone()); } let signals = PolicySignals::from_headers(http_req.headers()); - let eids = resolve_auction_eids(&auction_kv_snapshot, registry, ec_context, signals); - let mirror = mirror_member(&auction_kv_snapshot, registry, ec_context, signals); + let identity = auction_identity(&auction_kv_snapshot, registry, ec_context, signals); + let mirror = identity.mirror_member(ec_context); // Look up geo for device info. let geo = services @@ -354,9 +357,10 @@ pub async fn handle_auction( geo, )?; - // Forward the browser entries as sent, append KV entries for registered - // sources the browser did not send, then apply consent gating. - let merged_eids = merge_auction_eids(client_eids, eids); + // Forward the browser entries that pass the provider denials, append the + // server set for sources the browser did not send, then apply consent + // gating. + let merged_eids = identity.merge(client_eids.as_deref()); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); @@ -452,53 +456,168 @@ pub async fn handle_auction( Ok(conversion.response) } -/// Resolves partner EIDs from the KV identity graph for bidstream decoration. +/// What the row a request read means for the `ts-eids-s` mirror (spec 6.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MirrorState { + /// No KV read for this request's EC: no mirror member. + NotRead, + /// EC use is denied, or the row is missing, a tombstone or withdrawn. + Cleared, + /// A live row: the mirror carries the servable server set. + Live, +} + +/// What one request's KV read gives the auction, resolved once (spec 7.1). /// -/// Returns `None` when any prerequisite is missing (no KV store, no partner -/// store, no EC, consent denied). On KV or partner-resolution errors, logs a -/// warning and returns empty EIDs so the auction can proceed in degraded mode. -/// A stored record is sent only when the shared servability check passes under -/// this request's consent and `signals`. -pub(crate) fn resolve_auction_eids( +/// Servability is decided once, at one clock read, and both the bidstream +/// merge and the `ts-eids-s` mirror read the same [`ServerSet`]. +pub(crate) struct AuctionIdentity { + /// `None` when the request may not use KV IDs at all (no registry, no + /// EC, or EC use denied). + server: Option, + denials: Denials, + mirror: MirrorState, + /// The clock read servability used, when there was one. + now: Option, +} + +impl AuctionIdentity { + /// The server set's EIDs, before browser precedence. + #[cfg(test)] + #[must_use] + pub(crate) fn server_eids(&self) -> Option> { + self.server.as_ref().map(ServerSet::eids) + } + + /// The request's bidstream EIDs before consent gating: the browser's + /// candidates after the denials, then the server set (spec 7.1). + #[must_use] + pub(crate) fn merge(&self, client_eids: Option<&[Eid]>) -> Option> { + let empty = ServerSet::default(); + let merged = merge_auction_eids( + client_eids.unwrap_or_default(), + self.server.as_ref().unwrap_or(&empty), + &self.denials, + ); + if merged.is_empty() { + None + } else { + Some(merged) + } + } + + /// The mirror member of an `/auction` or `/_ts/page-bids` response + /// (spec 6.2). + /// + /// The mirror carries the same server set as the bidstream, provenance + /// defaults included, and never an ID consent would strip. + #[must_use] + pub(crate) fn mirror_member(&self, ec_context: &EcContext) -> Option { + let sources = match self.mirror { + MirrorState::NotRead | MirrorState::Cleared => Vec::new(), + MirrorState::Live if allows_eid_persistence(ec_context.consent()) => self + .server + .as_ref() + .map(|server| mirror_sources(server, ec_context)) + .unwrap_or_default(), + MirrorState::Live => Vec::new(), + }; + let read = if self.mirror == MirrorState::NotRead { + MirrorRead::NotRead + } else if sources.is_empty() { + MirrorRead::Cleared + } else { + MirrorRead::Servable(&sources) + }; + let now = self.now.unwrap_or_else(current_timestamp); + mirror_payload(read, now).map(|payload| payload.to_json()) + } +} + +/// One [`MirrorSource`] per source of `server`, ranked by its writer's +/// `cookie_priority`. +fn mirror_sources(server: &ServerSet, ec_context: &EcContext) -> Vec { + let identity = ec_context.identity_runtime().map(IdentityRuntime::identity); + server + .sources() + .iter() + .map(|source| { + let priority = match (&source.writer, identity) { + (Some(writer), Some(identity)) => identity + .providers + .get(writer.as_str()) + .map_or(0, |settings| settings.cookie_priority), + _ => 0, + }; + MirrorSource { + source: source.source_domain.clone(), + provider: source.writer.clone(), + priority, + expires_at: source.expires_at, + eids: source.eids.clone(), + } + }) + .collect() +} + +fn mirror_state( snapshot: &EcKvSnapshot, registry: Option<&PartnerRegistry>, ec_context: &EcContext, - signals: PolicySignals, -) -> Option> { - resolve_servable_ids(snapshot, registry, ec_context, signals).map(|resolved| to_eids(&resolved)) +) -> MirrorState { + if !ec_context.ec_allowed() { + return MirrorState::Cleared; + } + let (Some(ec_id), Some(_)) = (ec_context.ec_value(), registry) else { + return MirrorState::NotRead; + }; + match snapshot { + EcKvSnapshot::Missing { ec_id: read } if read == ec_id => MirrorState::Cleared, + EcKvSnapshot::Present { + ec_id: read, entry, .. + } if read == ec_id => { + if entry.consent.ok { + MirrorState::Live + } else { + MirrorState::Cleared + } + } + _ => MirrorState::NotRead, + } } -/// The servable stored records behind [`resolve_auction_eids`], before they -/// become EIDs. -fn resolve_servable_ids( +/// Resolves what the request's KV read gives the auction, once. +/// +/// The server set is `None` when any prerequisite is missing (no partner +/// store, no EC, consent denied), and empty on a KV miss, a failed read, a +/// withdrawn row or a failed clock, so the auction proceeds in degraded +/// mode. A stored record is in it only when the shared servability check +/// passes under this request's consent and `signals`. The denials come from +/// the row read, whatever its consent state. +pub(crate) fn auction_identity( snapshot: &EcKvSnapshot, registry: Option<&PartnerRegistry>, ec_context: &EcContext, signals: PolicySignals, -) -> Option> { - let registry = registry?; - - if !ec_context.ec_allowed() { - return None; - } - - let ec_id = ec_context.ec_value()?; - - let Some(entry) = snapshot.entry_for(ec_id) else { - return Some(Vec::new()); +) -> AuctionIdentity { + let mut identity = AuctionIdentity { + server: None, + denials: Denials::default(), + mirror: mirror_state(snapshot, registry, ec_context), + now: None, }; - - if !entry.consent.ok { - return Some(Vec::new()); + let Some(registry) = registry else { + return identity; + }; + if !ec_context.ec_allowed() { + return identity; } - - // A zero time would make every expiry check pass (spec 4.4), so a failed - // clock resolves no KV IDs. - let Some(now) = checked_current_timestamp() else { - return Some(Vec::new()); + let Some(ec_id) = ec_context.ec_value() else { + return identity; }; + let entry = snapshot.entry_for(ec_id); // Without a runtime no provider is configured, so only unowned records - // without a writer are servable. + // without a writer are servable, and no provider denies anything. let empty_catalog; let catalog = match ec_context.identity_runtime() { Some(runtime) => runtime.catalog(), @@ -507,7 +626,17 @@ fn resolve_servable_ids( &empty_catalog } }; - let opted_out = opted_out(entry); + if let Some(entry) = entry { + identity.denials = denials(entry, catalog); + } + // A zero time would make every expiry check pass (spec 4.4), so a failed + // clock resolves no KV IDs. + let Some(now) = checked_current_timestamp() else { + identity.server = Some(ServerSet::default()); + return identity; + }; + identity.now = Some(now); + let opted_out = entry.map(opted_out).unwrap_or_default(); let serve = ServeContext { now: UnixTime::from_secs(now), consent: ec_context.consent(), @@ -515,82 +644,34 @@ fn resolve_servable_ids( catalog, opted_out: &opted_out, }; - Some(resolve_partner_ids(registry, entry, &serve)) + identity.server = Some(server_set(entry, registry, &serve)); + identity } -/// What the request's KV read gives the `ts-eids-s` mirror (spec 6.2). -/// -/// `sources` receives the servable server set, one [`MirrorSource`] per -/// source, after `gate_eids_by_consent`: the mirror never carries an ID -/// consent filtered out. The returned [`MirrorRead`] borrows it. +/// Resolves partner EIDs from the KV identity graph for bidstream decoration. /// -/// Consent that denies EC use clears the mirror before any KV read is -/// needed, so a withdrawal reaches the cookie. Otherwise the result follows -/// the row this request read: [`MirrorRead::NotRead`] when it read none, -/// [`MirrorRead::Cleared`] for a missing row, a tombstone or `consent.ok = -/// false`, and the servable set for a live row. -pub(crate) fn mirror_read<'a>( +/// Returns `None` when any prerequisite is missing (no KV store, no partner +/// store, no EC, consent denied), and empty EIDs on a KV miss or error so +/// the auction can proceed in degraded mode. See [`auction_identity`]. +#[cfg(test)] +pub(crate) fn resolve_auction_eids( snapshot: &EcKvSnapshot, registry: Option<&PartnerRegistry>, ec_context: &EcContext, signals: PolicySignals, - sources: &'a mut Vec, -) -> MirrorRead<'a> { - if !ec_context.ec_allowed() { - return MirrorRead::Cleared; - } - let (Some(ec_id), Some(_)) = (ec_context.ec_value(), registry) else { - return MirrorRead::NotRead; - }; - match snapshot { - EcKvSnapshot::Missing { ec_id: read } if read == ec_id => return MirrorRead::Cleared, - EcKvSnapshot::Present { - ec_id: read, entry, .. - } if read == ec_id && !entry.consent.ok => { - return MirrorRead::Cleared; - } - EcKvSnapshot::Present { ec_id: read, .. } if read == ec_id => {} - _ => return MirrorRead::NotRead, - } - let resolved = - resolve_servable_ids(snapshot, registry, ec_context, signals).unwrap_or_default(); - let identity = ec_context.identity_runtime().map(IdentityRuntime::identity); - let candidates: Vec = resolved - .iter() - .map(|record| { - let priority = match (&record.writer, identity) { - (Some(writer), Some(identity)) => identity.provider(writer).cookie_priority, - _ => 0, - }; - MirrorSource { - source: record.source_domain.clone(), - provider: record.writer.clone(), - priority, - expires_at: record.expires_at, - eids: to_eids(core::slice::from_ref(record)), - } - }) - .collect(); - match gate_eids_by_consent(Some(candidates), Some(ec_context.consent())) { - Some(gated) => { - *sources = gated; - MirrorRead::Servable(sources) - } - None => MirrorRead::Cleared, - } +) -> Option> { + auction_identity(snapshot, registry, ec_context, signals).server_eids() } /// The mirror member of an `/auction` or `/_ts/page-bids` response, from the -/// row this request read (spec 6.2). +/// row this request read (spec 6.2). See [`AuctionIdentity::mirror_member`]. pub(crate) fn mirror_member( snapshot: &EcKvSnapshot, registry: Option<&PartnerRegistry>, ec_context: &EcContext, signals: PolicySignals, ) -> Option { - let mut sources = Vec::new(); - let read = mirror_read(snapshot, registry, ec_context, signals, &mut sources); - mirror_payload(read, current_timestamp()).map(|payload| payload.to_json()) + auction_identity(snapshot, registry, ec_context, signals).mirror_member(ec_context) } /// Resolves the request's browser EIDs: the `/auction` body when it has a @@ -619,23 +700,23 @@ fn parse_cookie_auction_eids(cookie_value: Option<&str>) -> Option> { } } -/// Merges the request's browser EIDs with KV-resolved partner EIDs. +/// Merges the request's browser EIDs with the server set (spec 7.1). /// -/// Browser entries are forwarded as parsed, in wire order, never altered or -/// combined: they are what Prebid sent the `trustedServer` adapter, and a -/// source may repeat with different EID-level fields. KV entries are appended -/// after them, only for registered sources no browser entry carries. Sources -/// are compared as the partner registry normalizes them (lowercase, trailing -/// dot removed). For a source the browser sends, KV's UIDs, `atype` and `ext` -/// are not used: KV keeps the first value it stored, so it is the staler copy. -/// Entries without UIDs are dropped. +/// The browser's candidates first pass the provider `denials`. Those left +/// are forwarded as parsed, in wire order, never altered beyond a denied +/// UID: they are what Prebid sent the `trustedServer` adapter, and a source +/// may repeat with different EID-level fields. A server source's whole entry +/// set is appended after them, only when no browser entry carries the source, +/// so the two are never mixed. Sources are compared as the partner registry +/// normalizes them (lowercase, trailing dot removed). Entries without UIDs are +/// dropped. pub(crate) fn merge_auction_eids( - client_eids: Option>, - resolved_eids: Option>, -) -> Option> { - let mut merged: Vec = client_eids + client_eids: &[Eid], + server: &ServerSet, + denials: &Denials, +) -> Vec { + let mut merged: Vec = filter_browser_candidates(client_eids, denials) .into_iter() - .flatten() .filter(|eid| !eid.uids.is_empty()) .collect(); let browser_sources: HashSet = merged @@ -643,22 +724,17 @@ pub(crate) fn merge_auction_eids( .filter_map(|eid| normalize_partner_source_domain(&eid.source).ok()) .collect(); - for eid in resolved_eids.into_iter().flatten() { + for eid in server.sources().iter().flat_map(|source| &source.eids) { if eid.uids.is_empty() { continue; } let carried_by_browser = normalize_partner_source_domain(&eid.source) .is_ok_and(|source| browser_sources.contains(&source)); if !carried_by_browser { - merged.push(eid); + merged.push(eid.clone()); } } - - if merged.is_empty() { - None - } else { - Some(merged) - } + merged } #[cfg(test)] @@ -670,6 +746,9 @@ mod tests { use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; + use crate::ec::identity::revision::uid_hash; + use crate::ec::identity::test_support::{ec_partner, example_entry_set, full_record}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvProviderState, KvUid}; use crate::error::IntoHttpResponse as _; use crate::openrtb::Uid; use crate::platform::test_support::{ @@ -2110,7 +2189,422 @@ mod tests { } fn merged_json(client: Option>, kv: Option>) -> JsonValue { - serde_json::to_value(merge_auction_eids(client, kv)).expect("should serialize merged EIDs") + let merged = merge_auction_eids( + &client.unwrap_or_default(), + &ServerSet::from_eids(kv.unwrap_or_default()), + &Denials::default(), + ); + let merged = (!merged.is_empty()).then_some(merged); + serde_json::to_value(merged).expect("should serialize merged EIDs") + } + + const MERGE_EC_ID_PREFIX: &str = "merge"; + + fn merge_ec_id() -> String { + format!("{}.{MERGE_EC_ID_PREFIX}", "a".repeat(64)) + } + + fn merge_registry(sources: &[&str]) -> PartnerRegistry { + let partners: Vec<_> = sources + .iter() + .map(|source| counting_test_partner(source)) + .collect(); + PartnerRegistry::from_config(&partners).expect("should build registry") + } + + fn merge_entry(records: Vec<(&str, KvPartnerId)>) -> KvEntry { + let mut entry = KvEntry::minimal("unused.example.com", "unused", 1); + entry.ids.clear(); + for (source, record) in records { + entry.ids.insert(source.to_owned(), record); + } + entry + } + + /// The request's [`AuctionIdentity`] for `entry`, with no providers. + fn merge_identity(entry: KvEntry, registry: &PartnerRegistry) -> AuctionIdentity { + let ec_id = merge_ec_id(); + let ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + let snapshot = EcKvSnapshot::Present { + ec_id, + entry: Box::new(entry), + generation: None, + }; + auction_identity( + &snapshot, + Some(registry), + &ec_context, + PolicySignals::default(), + ) + } + + fn default_atype() -> i32 { + crate::settings::EcPartner::default_openrtb_atype() + } + + fn plain_eid(source: &str, id: &str, atype: Option) -> Eid { + Eid { + source: source.to_owned(), + uids: vec![Uid { + id: id.to_owned(), + atype, + ext: None, + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + } + } + + #[test] + fn merge_browser_value_wins_for_unowned_source() { + let registry = merge_registry(&[SOURCE_S]); + let identity = merge_identity( + merge_entry(vec![(SOURCE_S, KvPartnerId::new("server"))]), + ®istry, + ); + + let merged = identity.merge(Some(&[plain_eid(SOURCE_S, "browser", Some(1))])); + + assert_eq!( + merged, + Some(vec![plain_eid(SOURCE_S, "browser", Some(1))]), + "the browser's value should win for an unowned source" + ); + } + + #[test] + fn merge_server_value_goes_out_when_the_browser_did_not_send_the_source() { + let registry = merge_registry(&[SOURCE_S, SOURCE_T]); + let identity = merge_identity( + merge_entry(vec![(SOURCE_T, KvPartnerId::new("server"))]), + ®istry, + ); + + let merged = identity.merge(Some(&[plain_eid(SOURCE_S, "browser", Some(1))])); + + assert_eq!( + merged, + Some(vec![ + plain_eid(SOURCE_S, "browser", Some(1)), + plain_eid(SOURCE_T, "server", Some(default_atype())), + ]), + "the server value should follow the browser entries" + ); + } + + #[test] + fn merge_expired_entry_is_not_sent() { + let registry = merge_registry(&[SOURCE_S, SOURCE_T]); + let expired = KvPartnerId { + exp: Some(1), + ..KvPartnerId::new("expired") + }; + let identity = merge_identity( + merge_entry(vec![ + (SOURCE_S, expired), + (SOURCE_T, KvPartnerId::new("live")), + ]), + ®istry, + ); + + assert_eq!( + identity.merge(None), + Some(vec![plain_eid(SOURCE_T, "live", Some(default_atype()))]), + "an expired record should not be sent" + ); + } + + #[test] + fn merge_stored_atype_and_ext_are_emitted() { + let registry = merge_registry(&[SOURCE_S]); + let identity = merge_identity( + merge_entry(vec![(SOURCE_S, full_record(None, example_entry_set()))]), + ®istry, + ); + + assert_eq!( + identity.merge(None), + Some(vec![ + Eid { + source: SOURCE_S.to_owned(), + uids: vec![ + Uid { + id: "example-a".to_owned(), + atype: Some(1), + ext: None, + }, + Uid { + id: "example-b".to_owned(), + atype: Some(3), + ext: Some(json!({ "stype": "ppuid" })), + }, + ], + inserter: Some("example.com".to_owned()), + matcher: None, + mm: Some(3), + ext: None, + }, + Eid { + source: SOURCE_S.to_owned(), + uids: vec![Uid { + id: "example-c".to_owned(), + atype: Some(1), + ext: None, + }], + inserter: None, + matcher: Some("partner.example.com".to_owned()), + mm: None, + ext: None, + }, + ]), + "the stored atype, UID ext and EID-level fields should go out" + ); + } + + #[test] + fn merge_entry_set_goes_out_one_eid_per_entry_in_stored_order_after_browser_entries() { + let registry = merge_registry(&[SOURCE_S, SOURCE_T]); + let identity = merge_identity( + merge_entry(vec![(SOURCE_S, full_record(None, example_entry_set()))]), + ®istry, + ); + + let merged = identity + .merge(Some(&[plain_eid(SOURCE_T, "browser", None)])) + .expect("should merge EIDs"); + + let shape: Vec<(String, Vec)> = merged + .iter() + .map(|eid| { + ( + eid.source.clone(), + eid.uids.iter().map(|uid| uid.id.clone()).collect(), + ) + }) + .collect(); + assert_eq!( + shape, + vec![ + (SOURCE_T.to_owned(), vec!["browser".to_owned()]), + ( + SOURCE_S.to_owned(), + vec!["example-a".to_owned(), "example-b".to_owned()] + ), + (SOURCE_S.to_owned(), vec!["example-c".to_owned()]), + ], + "the stored set should go out one EID per entry, in order, after the browser" + ); + } + + fn decode_eids_header(value: &str) -> Vec { + serde_json::from_slice(&BASE64.decode(value).expect("should be base64")) + .expect("should be EID JSON") + } + + #[test] + fn merge_x_ts_eids_matches_what_was_sent() { + let registry = merge_registry(&[SOURCE_S, SOURCE_T]); + let identity = merge_identity( + merge_entry(vec![(SOURCE_S, full_record(None, example_entry_set()))]), + ®istry, + ); + let merged = identity + .merge(Some(&[plain_eid(SOURCE_T, "browser", Some(1))])) + .expect("should merge EIDs"); + + let (header, truncated) = + crate::ec::eids::encode_eids_header(&merged).expect("should encode the header"); + + assert!(!truncated, "should fit without truncation"); + assert_eq!( + decode_eids_header(&header), + merged, + "x-ts-eids should carry exactly what was sent" + ); + } + + #[test] + fn merge_header_still_truncates_per_entry() { + let registry = merge_registry(&[SOURCE_S]); + let entries: Vec = (0..40) + .map(|index| KvEid { + uids: vec![KvUid { + id: format!("{index:03}-{}", "x".repeat(100)), + atype: Some(1), + ext: None, + }], + ..KvEid::default() + }) + .collect(); + let identity = merge_identity( + merge_entry(vec![(SOURCE_S, full_record(None, entries))]), + ®istry, + ); + let merged = identity.merge(None).expect("should merge EIDs"); + + let (header, truncated) = + crate::ec::eids::encode_eids_header(&merged).expect("should encode the header"); + + assert_eq!(merged.len(), 40, "user.eids should hold the whole set"); + assert!(truncated, "x-ts-eids-truncated should be set"); + let carried = decode_eids_header(&header); + assert!( + !carried.is_empty() && carried.len() < merged.len(), + "x-ts-eids should hold the entries that fit" + ); + assert_eq!( + carried, + merged[..carried.len()].to_vec(), + "the header should hold a prefix of whole entries" + ); + } + + #[test] + fn merge_denials_apply_through_the_request_identity() { + let partner = ec_partner( + "ssp.example.com", + "https://sync.ssp.example.com/pull", + "example-token-value-0123456789abcdef", + ); + let mut settings = create_test_settings(); + settings.ec.partners = vec![partner]; + let runtime = std::sync::Arc::new( + IdentityRuntime::from_settings(&settings).expect("should build the runtime"), + ); + let registry = merge_registry(&["ssp.example.com"]); + let ec_id = merge_ec_id(); + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, Some(&ec_id)); + ec_context.set_identity_runtime(runtime); + let mut entry = merge_entry(Vec::new()); + entry.ps.insert( + "pull-ssp-example-com".to_owned(), + KvProviderState { + iv: vec![uid_hash("invalidated")], + ..KvProviderState::default() + }, + ); + let snapshot = EcKvSnapshot::Present { + ec_id, + entry: Box::new(entry), + generation: None, + }; + let identity = auction_identity( + &snapshot, + Some(®istry), + &ec_context, + PolicySignals::default(), + ); + + let merged = identity.merge(Some(&[ + plain_eid("ssp.example.com", "invalidated", Some(1)), + plain_eid(SOURCE_T, "kept", Some(1)), + ])); + + assert_eq!( + merged, + Some(vec![plain_eid(SOURCE_T, "kept", Some(1))]), + "a browser UID the claiming provider invalidated should not be sent" + ); + } + + #[test] + fn golden_merge_one_uid_row_goes_out_as_on_main() { + let registry = merge_registry(&["a.example.com", "b.example.com"]); + let identity = merge_identity( + merge_entry(vec![ + ("b.example.com", KvPartnerId::new("ub")), + ("a.example.com", KvPartnerId::new("ua")), + ]), + ®istry, + ); + + // Written from `main`'s `merge_auction_eids(None, Some(to_eids(..)))`. + let expected = vec![ + Eid { + source: "a.example.com".to_owned(), + uids: vec![Uid { + id: "ua".to_owned(), + atype: Some(3), + ext: None, + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + }, + Eid { + source: "b.example.com".to_owned(), + uids: vec![Uid { + id: "ub".to_owned(), + atype: Some(3), + ext: None, + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + }, + ]; + assert_eq!( + identity.merge(None), + Some(expected), + "a one-UID row should go out exactly as on main" + ); + } + + #[test] + fn golden_merge_browser_entries_are_forwarded_unchanged() { + let client = vec![ + Eid { + source: SOURCE_S.to_owned(), + uids: vec![ + Uid { + id: "s1".to_owned(), + atype: Some(1), + ext: Some(json!({ "stype": "ppuid" })), + }, + Uid { + id: "s2".to_owned(), + atype: None, + ext: None, + }, + ], + inserter: Some("inserter-1.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(1), + ext: Some(json!({ "segments": ["a"] })), + }, + Eid { + source: SOURCE_T.to_owned(), + uids: Vec::new(), + inserter: None, + matcher: None, + mm: None, + ext: None, + }, + Eid { + source: SOURCE_S.to_owned(), + uids: vec![Uid { + id: "s1".to_owned(), + atype: Some(3), + ext: None, + }], + inserter: Some("inserter-2.example.com".to_owned()), + matcher: None, + mm: Some(2), + ext: None, + }, + ]; + + let merged = merge_auction_eids(&client, &ServerSet::default(), &Denials::default()); + + assert_eq!( + merged, + vec![client[0].clone(), client[2].clone()], + "browser entries should go out as today, minus entries without UIDs" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index a9affaa36f..976aa351f4 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -12,6 +12,7 @@ use error_stack::{Report, ResultExt as _}; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; +use super::identity::catalog::ProviderCatalog; use super::identity::servable::{ServeContext, is_servable}; use super::identity::{IdentityProviderId, SourceDomain}; use super::kv_types::{KvEid, KvEntry}; @@ -124,6 +125,124 @@ fn stored_entry_to_eid(item: &ResolvedPartnerId, entry: &KvEid) -> Eid { } } +/// One source of a [`ServerSet`]: a servable record and its outbound EIDs. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ServerSource { + /// The partner's source domain, as the registry holds it. + pub(crate) source_domain: String, + /// The provider that wrote the record, if any. + pub(crate) writer: Option, + /// Unix seconds at which the record stops being servable. + pub(crate) expires_at: Option, + /// One EID per stored entry, in stored order, provenance defaults filled. + pub(crate) eids: Vec, +} + +/// The servable server set of the KV row a request read (spec 7.1). +/// +/// Built once per request; the bidstream merge and the `ts-eids-s` mirror +/// both read it. It never holds anything from `ts-eids-s`. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct ServerSet { + sources: Vec, +} + +impl ServerSet { + /// The set's sources, ordered by source domain. + #[must_use] + pub(crate) fn sources(&self) -> &[ServerSource] { + &self.sources + } + + /// Every source's EIDs, in order. + #[cfg(test)] + #[must_use] + pub(crate) fn eids(&self) -> Vec { + self.sources + .iter() + .flat_map(|source| source.eids.iter().cloned()) + .collect() + } + + /// A set of one source per EID, for tests that start from EIDs. + #[cfg(test)] + #[must_use] + pub(crate) fn from_eids(eids: Vec) -> Self { + Self { + sources: eids + .into_iter() + .map(|eid| ServerSource { + source_domain: eid.source.clone(), + writer: None, + expires_at: None, + eids: vec![eid], + }) + .collect(), + } + } +} + +/// The servable server set of `entry` (spec 7.1). +/// +/// Without a read row, or for a row that is not live and consenting, the set +/// is empty. Each record goes through [`resolve_partner_ids`] and +/// [`to_eids`], then [`fill_provenance`]. +#[must_use] +pub(crate) fn server_set( + entry: Option<&KvEntry>, + registry: &PartnerRegistry, + serve: &ServeContext<'_>, +) -> ServerSet { + let Some(entry) = entry.filter(|entry| entry.consent.ok) else { + return ServerSet::default(); + }; + let sources = resolve_partner_ids(registry, entry, serve) + .into_iter() + .map(|record| { + let mut eids = to_eids(core::slice::from_ref(&record)); + if let Ok(source) = record.source_domain.parse::() { + fill_provenance(&mut eids, record.writer.as_ref(), &source, serve.catalog); + } + ServerSource { + source_domain: record.source_domain, + writer: record.writer, + expires_at: record.expires_at, + eids, + } + }) + .collect(); + ServerSet { sources } +} + +/// Applies the writer's `with_provenance` default for `source` to the EIDs +/// of a record it wrote (spec 4.3, 7.1). +/// +/// Only an entry that sets none of `inserter`, `matcher` and `mm` gets the +/// default; an entry that sets any of them is the vendor's complete +/// statement and is left as stored. A record without a configured writer +/// gets nothing. +pub(crate) fn fill_provenance( + eids: &mut [Eid], + writer: Option<&IdentityProviderId>, + source: &SourceDomain, + catalog: &ProviderCatalog, +) { + let Some(default) = writer + .and_then(|writer| catalog.get(writer)) + .and_then(|provider| provider.descriptor().provenance(source)) + else { + return; + }; + for eid in eids + .iter_mut() + .filter(|eid| eid.inserter.is_none() && eid.matcher.is_none() && eid.mm.is_none()) + { + eid.inserter = default.inserter().map(str::to_owned); + eid.matcher = default.matcher().map(str::to_owned); + eid.mm = default.mm(); + } +} + /// Builds a base64-encoded EID header value, truncating if needed. /// /// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds @@ -193,17 +312,17 @@ pub fn encode_eids_header(eids: &[Eid]) -> Result<(String, bool), Report ProviderCatalog { + ProviderCatalog::new( + vec![provenance_provider( + "pa", + &["a.example.com", "b.example.com"], + vec![ + ( + "a.example.com", + EidProvenance::new() + .with_inserter("default-inserter.example.com") + .with_matcher("default-matcher.example.com") + .with_mm(2), + ), + ( + "b.example.com", + EidProvenance::new().with_inserter("default-inserter.example.com"), + ), + ], + )], + BTreeMap::new(), + ) + } + + #[test] + fn merge_provenance_applies_only_to_an_entry_with_no_eid_level_fields() { + let registry = registry_with(&["a.example.com", "b.example.com"]); + let kv_uid = |id: &str| KvUid { + id: id.to_owned(), + atype: None, + ext: None, + }; + let mut entry = KvEntry::minimal("unused.example.com", "unused", 1); + entry.ids.clear(); + entry.ids.insert( + "a.example.com".to_owned(), + full_record( + Some("pa"), + vec![ + KvEid { + uids: vec![kv_uid("bare")], + ..KvEid::default() + }, + KvEid { + uids: vec![kv_uid("vendor")], + inserter: Some("vendor.example.com".to_owned()), + ..KvEid::default() + }, + ], + ), + ); + entry + .ids + .insert("b.example.com".to_owned(), KvPartnerId::new("seeded")); + let fixture = ServeFixture { + catalog: provenance_catalog(), + ..ServeFixture::new() + }; + + let set = server_set(Some(&entry), ®istry, &fixture.at(100)); + + let uid = |id: &str| Uid { + id: id.to_owned(), + atype: Some(99), + ext: None, + }; + assert_eq!( + set.eids(), + vec![ + Eid { + source: "a.example.com".to_owned(), + uids: vec![uid("bare")], + inserter: Some("default-inserter.example.com".to_owned()), + matcher: Some("default-matcher.example.com".to_owned()), + mm: Some(2), + ext: None, + }, + Eid { + source: "a.example.com".to_owned(), + uids: vec![uid("vendor")], + inserter: Some("vendor.example.com".to_owned()), + matcher: None, + mm: None, + ext: None, + }, + Eid { + source: "b.example.com".to_owned(), + uids: vec![uid("seeded")], + inserter: None, + matcher: None, + mm: None, + ext: None, + }, + ], + "should fill the default only on the provider-written entry with no EID-level field" + ); + } + + #[test] + fn merge_server_set_is_empty_without_a_live_consenting_row() { + let registry = registry_with(&["a.example.com"]); + let fixture = ServeFixture::new(); + let mut entry = KvEntry::minimal("a.example.com", "ua", 1); + entry.consent.ok = false; + + assert_eq!( + server_set(None, ®istry, &fixture.at(100)), + ServerSet::default(), + "should be empty without a KV read" + ); + assert_eq!( + server_set(Some(&entry), ®istry, &fixture.at(100)), + ServerSet::default(), + "should be empty for a withdrawn row" + ); + } } diff --git a/crates/trusted-server-core/src/ec/identity/catalog.rs b/crates/trusted-server-core/src/ec/identity/catalog.rs index 07dbc66fb0..f14a849533 100644 --- a/crates/trusted-server-core/src/ec/identity/catalog.rs +++ b/crates/trusted-server-core/src/ec/identity/catalog.rs @@ -187,8 +187,6 @@ impl ProviderCatalog { } /// Every provider, ordered by ID. - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn providers(&self) -> impl Iterator> { self.providers.values() } @@ -212,8 +210,6 @@ impl ProviderCatalog { } /// Every provider that claims `source` after narrowing, ordered by ID. - // Temporary: first used by a later identity task. - #[allow(dead_code)] pub(crate) fn claimants<'a>( &'a self, source: &'a SourceDomain, diff --git a/crates/trusted-server-core/src/ec/identity/denials.rs b/crates/trusted-server-core/src/ec/identity/denials.rs new file mode 100644 index 0000000000..1c4e10dc6c --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/denials.rs @@ -0,0 +1,311 @@ +//! Provider denials of browser EID candidates, read from KV (spec 7.1, +//! "Provider denials apply to browser candidates"). + +use std::collections::{BTreeMap, BTreeSet}; + +use super::catalog::ProviderCatalog; +use super::ids::SourceDomain; +use super::request::opted_out; +use super::revision::uid_hash; +use crate::ec::kv_types::KvEntry; +use crate::openrtb::Eid; + +/// What the providers that claim a source deny of the browser's candidates. +/// +/// Empty without a KV read. +#[derive(Debug, Default)] +pub(crate) struct Denials { + /// Per source, the `ps.iv` and `ps.ov` hashes of every claiming provider. + hashes: BTreeMap>, + /// Sources whose browser value is dropped whole: those of an opted-out + /// owner, and every source of an opted-out capture-only provider. + dropped: BTreeSet, +} + +impl Denials { + fn is_empty(&self) -> bool { + self.hashes.is_empty() && self.dropped.is_empty() + } +} + +/// The denials `entry` holds for the providers in `catalog` (spec 7.1). +#[must_use] +pub(crate) fn denials(entry: &KvEntry, catalog: &ProviderCatalog) -> Denials { + let mut result = Denials::default(); + let sources: BTreeSet<&SourceDomain> = catalog + .providers() + .flat_map(|provider| catalog.sources_of(provider.descriptor().id())) + .collect(); + for source in sources { + let hashes: BTreeSet = catalog + .claimants(source) + .filter_map(|provider| entry.ps.get(provider.as_str())) + .flat_map(|state| state.iv.iter().chain(&state.ov).cloned()) + .collect(); + if !hashes.is_empty() { + result.hashes.insert(source.clone(), hashes); + } + } + for provider in opted_out(entry) { + let capture_only = catalog.is_capture_only(&provider); + for source in catalog.sources_of(&provider) { + if capture_only || catalog.owner_of(source) == Some(&provider) { + result.dropped.insert(source.clone()); + } + } + } + result +} + +/// The browser's candidates after `denials` (spec 7.1). +/// +/// Drops each entry of a dropped-whole source; otherwise drops each UID whose +/// hash a claiming provider holds, then each entry left with no UID. Every +/// other entry goes through untouched, in order. +#[must_use] +pub(crate) fn filter_browser_candidates(client: &[Eid], denials: &Denials) -> Vec { + if denials.is_empty() { + return client.to_vec(); + } + let mut kept = Vec::with_capacity(client.len()); + for eid in client { + let Ok(source) = eid.source.parse::() else { + kept.push(eid.clone()); + continue; + }; + if denials.dropped.contains(&source) { + continue; + } + let Some(hashes) = denials.hashes.get(&source) else { + kept.push(eid.clone()); + continue; + }; + let mut filtered = eid.clone(); + filtered + .uids + .retain(|uid| !hashes.contains(&uid_hash(&uid.id))); + if !filtered.uids.is_empty() { + kept.push(filtered); + } + } + kept +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::ec::identity::test_support::{fixed_capture_provider, fixed_provider, pid, source}; + use crate::ec::kv_types::{KvOptOut, KvProviderState}; + use crate::openrtb::Uid; + + fn uid(id: &str) -> Uid { + Uid { + id: id.to_owned(), + atype: Some(1), + ext: None, + } + } + + fn browser(source: &str, ids: &[&str]) -> Eid { + Eid { + source: source.to_owned(), + uids: ids.iter().map(|id| uid(id)).collect(), + inserter: Some("inserter.example.com".to_owned()), + matcher: None, + mm: Some(1), + ext: None, + } + } + + fn entry_with(states: &[(&str, KvProviderState)]) -> KvEntry { + let mut entry = KvEntry::minimal("unused.example.com", "unused", 1); + for (provider, state) in states { + entry.ps.insert((*provider).to_owned(), state.clone()); + } + entry + } + + fn invalidated(uids: &[&str]) -> KvProviderState { + KvProviderState { + iv: uids.iter().map(|uid| uid_hash(uid)).collect(), + ..KvProviderState::default() + } + } + + fn opted_out_state() -> KvProviderState { + KvProviderState { + o: Some(KvOptOut { + at: 10, + fp: "0".to_owned(), + }), + ..KvProviderState::default() + } + } + + fn unowned_catalog() -> ProviderCatalog { + ProviderCatalog::new( + vec![ + fixed_provider("pa", &["a.example.com"]), + fixed_provider("pb", &["b.example.com"]), + ], + BTreeMap::new(), + ) + } + + #[test] + fn denial_drops_a_browser_uid_in_a_claimants_iv_for_an_unowned_source() { + let entry = entry_with(&[("pa", invalidated(&["a-old"]))]); + let denials = denials(&entry, &unowned_catalog()); + + let kept = filter_browser_candidates( + &[ + browser("a.example.com", &["a-old"]), + browser("b.example.com", &["b1"]), + ], + &denials, + ); + + assert_eq!( + kept, + vec![browser("b.example.com", &["b1"])], + "should drop the invalidated UID and its emptied entry" + ); + } + + #[test] + fn denial_drops_a_browser_uid_in_a_claimants_ov() { + let state = KvProviderState { + ov: vec![uid_hash("a-removed")], + ..KvProviderState::default() + }; + let entry = entry_with(&[("pa", state)]); + let denials = denials(&entry, &unowned_catalog()); + + let kept = filter_browser_candidates(&[browser("a.example.com", &["a-removed"])], &denials); + + assert_eq!( + kept, + Vec::::new(), + "should drop a UID the opt-out removed" + ); + } + + #[test] + fn denial_filters_uid_by_uid_and_drops_an_emptied_entry() { + let entry = entry_with(&[("pa", invalidated(&["A"]))]); + let denials = denials(&entry, &unowned_catalog()); + + let kept = filter_browser_candidates( + &[ + browser("a.example.com", &["A", "B"]), + browser("a.example.com", &["A"]), + ], + &denials, + ); + + assert_eq!( + kept, + vec![Eid { + source: "a.example.com".to_owned(), + uids: vec![uid("B")], + inserter: Some("inserter.example.com".to_owned()), + matcher: None, + mm: Some(1), + ext: None, + }], + "should keep B alone and drop the entry holding only A" + ); + } + + #[test] + fn denial_ignores_hashes_of_a_provider_that_does_not_claim_the_source() { + let entry = entry_with(&[("pb", invalidated(&["shared"]))]); + let denials = denials(&entry, &unowned_catalog()); + + let kept = filter_browser_candidates(&[browser("a.example.com", &["shared"])], &denials); + + assert_eq!( + kept, + vec![browser("a.example.com", &["shared"])], + "should not apply another source's hashes" + ); + } + + #[test] + fn denial_drops_every_owned_source_while_the_owner_is_opted_out() { + let catalog = ProviderCatalog::new( + vec![fixed_provider("owner", &["a.example.com", "b.example.com"])], + BTreeMap::from([(source("a.example.com"), pid("owner"))]), + ); + let entry = entry_with(&[("owner", opted_out_state())]); + let denials = denials(&entry, &catalog); + + let kept = filter_browser_candidates( + &[ + browser("a.example.com", &["a1"]), + browser("b.example.com", &["b1"]), + ], + &denials, + ); + + assert_eq!( + kept, + vec![browser("b.example.com", &["b1"])], + "should drop the owned source and keep the unowned one" + ); + } + + #[test] + fn denial_drops_every_source_of_an_opted_out_capture_only_provider() { + let catalog = ProviderCatalog::new( + vec![ + fixed_capture_provider("cap", &["a.example.com", "b.example.com"]), + fixed_provider("other", &["c.example.com"]), + ], + BTreeMap::from([(source("a.example.com"), pid("cap"))]), + ); + let client = [ + browser("a.example.com", &["a1"]), + browser("b.example.com", &["b1"]), + browser("c.example.com", &["c1"]), + ]; + let entry = entry_with(&[("cap", opted_out_state())]); + + let kept = filter_browser_candidates(&client, &denials(&entry, &catalog)); + + assert_eq!( + kept, + vec![browser("c.example.com", &["c1"])], + "should drop owned and unowned sources of the capture-only provider" + ); + + let cleared = KvProviderState { + oc: Some(20), + ..opted_out_state() + }; + let entry = entry_with(&[("cap", cleared)]); + let kept = filter_browser_candidates(&client, &denials(&entry, &catalog)); + assert_eq!( + kept, + client.to_vec(), + "control: after a later oc the browser values go out again" + ); + } + + #[test] + fn denial_mirror_id_fed_back_through_the_body_is_filtered() { + let entry = entry_with(&[("pa", invalidated(&["mirror-only"]))]); + let denials = denials(&entry, &unowned_catalog()); + + let kept = + filter_browser_candidates(&[browser("a.example.com", &["mirror-only"])], &denials); + + assert_eq!( + kept, + Vec::::new(), + "a stale mirror ID fed back as a browser candidate should not be sent" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 856ba8a4dc..0bb5614cf9 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod catalog; pub(crate) mod coordinator; +pub(crate) mod denials; pub mod descriptor; pub mod exchange; pub(crate) mod generic_pull; diff --git a/crates/trusted-server-core/src/ec/identity/test_support.rs b/crates/trusted-server-core/src/ec/identity/test_support.rs index 3bfb68f169..847770cab1 100644 --- a/crates/trusted-server-core/src/ec/identity/test_support.rs +++ b/crates/trusted-server-core/src/ec/identity/test_support.rs @@ -26,7 +26,7 @@ use crate::consent::types::TcfConsent; use crate::ec::identity::catalog::ProviderCatalog; use crate::ec::identity::coordinator::{OutcomeContext, RequestFacts}; use crate::ec::identity::descriptor::{ - Captures, ConsentPolicy, Input, ProviderDescriptor, ResolveKind, Resolves, + Captures, ConsentPolicy, EidProvenance, Input, ProviderDescriptor, ResolveKind, Resolves, }; use crate::ec::identity::exchange::{ProviderRequest, ProviderResponse}; use crate::ec::identity::ids::{IdentityProviderId, SourceDomain, UnixTime}; @@ -163,6 +163,32 @@ pub fn fixed_capture_provider(id: &str, sources: &[&str]) -> Arc, +) -> Arc { + let mut builder = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_resolves(Resolves::new().with_endpoint("https://api.example.com")) + .with_consent_policy(ConsentPolicy::eid_persistence_only()); + for source in sources { + builder = builder.with_source(source.parse().expect("should parse the source")); + } + for (source, default) in provenance { + builder = + builder.with_provenance(&source.parse().expect("should parse the source"), default); + } + provider_from(builder.build().expect("should build the descriptor")) +} + /// A descriptor-only provider for `descriptor`, for tests that need more /// than [`fixed_provider`] declares. #[must_use] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 28b587df95..335347eb55 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -38,7 +38,7 @@ use sha2::Digest as _; use web_time::Instant; use crate::auction::endpoints::{ - merge_auction_eids, mirror_member, resolve_auction_eids, resolve_client_auction_eids, + AuctionIdentity, auction_identity, mirror_member, resolve_client_auction_eids, }; use crate::auction::formats::sanitize_publisher_page_url; use crate::auction::orchestrator::{ @@ -66,6 +66,7 @@ use crate::creative_opportunities::{ AdStackGateInput, AssemblyMode, CreativeOpportunitiesConfig, RuntimeAdStackExpected, evaluate_ad_stack_gate, }; +use crate::ec::identity::mirror::MirrorPayload; use crate::ec::identity::policy::PolicySignals; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -5543,11 +5544,12 @@ struct AuctionEidTargeting<'a> { /// /// Shared verbatim by the initial-page and page-bids dispatch paths so the EID /// resolution and consent gating live in one place; `path_label` only varies -/// the consent-stripped warning message. +/// the consent-stripped warning message. Returns the request's +/// [`AuctionIdentity`], so the page-bids mirror reads the same server set. fn apply_auction_eids_and_device( auction_request: &mut AuctionRequest, targeting: &AuctionEidTargeting<'_>, -) { +) -> AuctionIdentity { let ts_eids_value = targeting .cookie_jar .and_then(|j| j.get(COOKIE_TS_EIDS)) @@ -5557,13 +5559,13 @@ fn apply_auction_eids_and_device( } else { None }; - let kv_eids = resolve_auction_eids( + let identity = auction_identity( targeting.kv_snapshot, targeting.partner_registry, targeting.ec_context, targeting.signals, ); - let merged_eids = merge_auction_eids(client_eids, kv_eids); + let merged_eids = identity.merge(client_eids.as_deref()); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = gate_eids_by_consent(merged_eids, auction_request.user.consent.as_ref()); @@ -5587,6 +5589,7 @@ fn apply_auction_eids_and_device( device.ip = client_ip; device.geo = targeting.geo.cloned(); } + identity } /// Build an [`AuctionRequest`] from matched creative opportunity slots. @@ -7058,9 +7061,9 @@ pub async fn handle_page_bids( // unchanged) but skip the live auction, matching the existing behavior. let ad_stack_enabled = ad_templates_enabled && auction_enabled && consent_allows_auction; - // The row the auction read, for the `ts-eids-s` mirror. It stays - // `NotRead` when no auction runs, so the mirror is then absent. - let mut mirror_snapshot = crate::ec::EcKvSnapshot::NotRead; + // What the auction's KV read gave it, for the `ts-eids-s` mirror. It stays + // `None` when no auction runs, so the mirror is then absent. + let mut auction_identity_read: Option = None; let (winning_bids, prebuilt_bid_map) = if matched_slots.is_empty() { (std::collections::HashMap::new(), None) } else { @@ -7107,7 +7110,7 @@ pub async fn handle_page_bids( &settings.publisher.domain, user_agent, ); - apply_auction_eids_and_device( + let identity = apply_auction_eids_and_device( &mut auction_request, &AuctionEidTargeting { cookie_jar: cookie_jar.as_ref(), @@ -7121,7 +7124,7 @@ pub async fn handle_page_bids( path_label: "Page-bids", }, ); - mirror_snapshot = page_bids_kv_snapshot; + auction_identity_read = Some(identity); let timeout_ms = co_config .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); @@ -7237,12 +7240,22 @@ pub async fn handle_page_bids( "slots": slots_json, "bids": bid_map, }); - if let Some(mirror) = mirror_member( - &mirror_snapshot, - auction.registry, - ec_context, - PolicySignals::from_headers(req.headers()), - ) { + // A consent denial clears the mirror, as `/auction` does, so a + // withdrawal reaches the cookie (spec 5.6). + let mirror = if consent_allows_auction { + match &auction_identity_read { + Some(identity) => identity.mirror_member(ec_context), + None => mirror_member( + &crate::ec::EcKvSnapshot::NotRead, + None, + ec_context, + PolicySignals::from_headers(req.headers()), + ), + } + } else { + Some(MirrorPayload::Clear.to_json()) + }; + if let Some(mirror) = mirror { body["ts_eids_s"] = mirror; } let body = serde_json::to_string(&body).change_context(TrustedServerError::Proxy { @@ -8051,6 +8064,151 @@ mod tests { ); } + /// The navigation and page-bids merge for `snapshot` and a `ts-eids` + /// cookie holding `cookie_entries`, plus any `extra_cookie`. + fn publisher_merge( + snapshot: &EcKvSnapshot, + registry: &PartnerRegistry, + ec_context: &EcContext, + cookie_entries: &serde_json::Value, + extra_cookie: Option<&str>, + ) -> Option> { + let cookie = + BASE64.encode(serde_json::to_vec(cookie_entries).expect("should serialize cookie")); + let mut header = format!("{COOKIE_TS_EIDS}={cookie}"); + if let Some(extra) = extra_cookie { + header.push_str("; "); + header.push_str(extra); + } + let jar = parse_cookies_to_jar(&header); + let services = noop_services(); + let mut auction_request = canonical_parity_auction_request(); + auction_request.user.consent = None; + auction_request.user.eids = None; + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: Some(&jar), + ec_id: ec_context.ec_value(), + kv_snapshot: snapshot, + partner_registry: Some(registry), + ec_context, + signals: PolicySignals::default(), + services: &services, + geo: None, + path_label: "Test", + }, + ); + auction_request.user.eids + } + + fn publisher_merge_fixture(entry: KvEntry) -> (EcKvSnapshot, PartnerRegistry, EcContext) { + let ec_id = format!("{}.navmrg", "a".repeat(64)); + let registry = PartnerRegistry::from_config(&[ + marker_partner("ids.example.com"), + marker_partner("other.example.com"), + ]) + .expect("should build partner registry"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + let ec_context = EcContext::new_for_test( + Some(ec_id), + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..ConsentContext::default() + }, + ); + (snapshot, registry, ec_context) + } + + fn publisher_eid(source: &str, id: &str, atype: i32) -> crate::openrtb::Eid { + crate::openrtb::Eid { + source: source.to_owned(), + uids: vec![crate::openrtb::Uid { + id: id.to_owned(), + atype: Some(atype), + ext: None, + }], + inserter: None, + matcher: None, + mm: None, + ext: None, + } + } + + #[test] + fn merge_publisher_never_reads_the_mirror_cookie() { + let entry = KvEntry::minimal("other.example.com", "kv-other", 1000); + let (snapshot, registry, ec_context) = publisher_merge_fixture(entry); + let cookie_entries = serde_json::json!([{ "source": "ids.example.com", "uids": [{ "id": "b", "atype": 1 }] }]); + let mirror = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&serde_json::json!({ "v": 1, "eids": [ + { "source": "mirror-only.example.com", "uids": [{ "id": "m", "atype": 1 }] } + ] })) + .expect("should serialize the mirror"), + ); + + let without = publisher_merge(&snapshot, ®istry, &ec_context, &cookie_entries, None); + let with = publisher_merge( + &snapshot, + ®istry, + &ec_context, + &cookie_entries, + Some(&format!("__Host-ts-eids-s={mirror}")), + ); + + assert_eq!( + with, + Some(vec![ + publisher_eid("ids.example.com", "b", 1), + publisher_eid( + "other.example.com", + "kv-other", + crate::settings::EcPartner::default_openrtb_atype() + ), + ]), + "should merge the cookie and KV only" + ); + assert_eq!(with, without, "the mirror cookie should change nothing"); + } + + #[test] + fn merge_publisher_denial_drops_a_browser_uid_in_a_claimants_iv_for_an_unowned_source() { + let mut entry = KvEntry::minimal("other.example.com", "kv-other", 1000); + entry.ps.insert( + "pull-ids-example-com".to_owned(), + crate::ec::kv_types::KvProviderState { + iv: vec![crate::ec::identity::revision::uid_hash("invalidated")], + ..Default::default() + }, + ); + let (snapshot, registry, mut ec_context) = publisher_merge_fixture(entry); + let mut settings = create_test_settings(); + settings.ec.partners = vec![marker_partner("ids.example.com")]; + ec_context.set_identity_runtime(Arc::new( + crate::ec::identity::IdentityRuntime::from_settings(&settings) + .expect("should build the runtime"), + )); + let cookie_entries = serde_json::json!([ + { "source": "ids.example.com", "uids": [{ "id": "invalidated", "atype": 1 }] } + ]); + + let merged = publisher_merge(&snapshot, ®istry, &ec_context, &cookie_entries, None); + + assert_eq!( + merged, + Some(vec![publisher_eid( + "other.example.com", + "kv-other", + crate::settings::EcPartner::default_openrtb_atype() + )]), + "the invalidated browser UID should not be sent" + ); + } + async fn run_marker_lookup_probe( probe: MarkerProbe, has_eid_cookie: bool, @@ -24315,6 +24473,37 @@ mod tests { ); } + #[tokio::test] + async fn page_bids_clears_the_mirror_when_consent_denies_the_auction() { + let settings = auction_settings(); + let graph = seeded_graph(&[(MIRROR_SOURCE, "uid-mirror")]); + // GDPR applies without a TC string: EC use stays allowed outside a + // GDPR jurisdiction, but the server-side auction gate denies. + let mut ec_context = EcContext::new_for_test( + Some(mirror_ec_id()), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + gdpr_applies: true, + ..Default::default() + }, + ); + assert!(ec_context.ec_allowed(), "should allow EC use"); + + let body = page_bids_body( + &settings, + &graph, + &mut ec_context, + make_page_bids_request("/2024/01/my-article/"), + ) + .await; + + assert_eq!( + body["ts_eids_s"], + serde_json::json!({ "clear": true }), + "should clear the mirror as /auction does on a consent denial" + ); + } + #[tokio::test] async fn page_bids_without_an_auction_omits_the_mirror() { let settings = settings_with_co_auction_disabled(); From 2c1a32b6c0000460b499baacda022aaa968a5926 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 26 Oct 2026 23:57:11 +0530 Subject: [PATCH 40/89] Flush identity changes once after send and run pull partners there Phase 4 replaces the pull sync after-send hook. It reuses the phase 1 snapshot when no after-send provider is due, otherwise re-reads the row, dispatches the due providers, and applies the whole mutation in one IfGenerationMatch write. A failed flush is not retried, but its removals are written alone with the shared CAS loop, and any that still fail are counted as lost. Every returning writer navigation now reads KV in phase 1, so the pull-complete marker cookie is removed. Pull partners run only on a returning visitor's writer navigation and back off through ps. evaluate_cluster is removed. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 40 +- .../trusted-server-adapter-fastly/src/main.rs | 461 ++++--- crates/trusted-server-core/src/constants.rs | 2 - crates/trusted-server-core/src/ec/finalize.rs | 190 +-- .../src/ec/identity/flush.rs | 1229 +++++++++++++++++ .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/mutation.rs | 48 + .../src/ec/identity/removal.rs | 4 - crates/trusted-server-core/src/ec/kv.rs | 126 +- .../trusted-server-core/src/ec/kv_backend.rs | 5 +- crates/trusted-server-core/src/ec/mod.rs | 77 +- .../trusted-server-core/src/ec/pull_sync.rs | 1068 +------------- .../src/ec/pull_sync_marker.rs | 689 --------- crates/trusted-server-core/src/publisher.rs | 252 ++-- 14 files changed, 1722 insertions(+), 2470 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/flush.rs delete mode 100644 crates/trusted-server-core/src/ec/pull_sync_marker.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 2894f7afdd..7732dac68f 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -193,6 +193,9 @@ pub(crate) struct AppState { pub(crate) auction_telemetry_sink: Arc, /// Identity providers, keys and policy, built once per settings load. pub(crate) identity: Arc, + /// The identity graph over `ec.ec_store`, built once. `None` when no + /// store is configured. + pub(crate) identity_graph: Option, } /// Build the application state, loading settings and constructing all per-application components. @@ -243,17 +246,33 @@ pub(crate) fn build_state_from_settings( let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + let identity_graph = crate::maybe_identity_graph(&settings); Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + identity_graph, auction_telemetry_sink, identity, })) } +/// Builds application state from `settings` with `graph` as its identity +/// graph, so routing, finalize and the after-send flush all use one store. +#[cfg(test)] +pub(crate) fn app_state_with_graph_for_test( + settings: Settings, + graph: KvIdentityGraph, +) -> Arc { + let state = build_state_from_settings(settings).expect("should build app state from settings"); + let mut state = Arc::try_unwrap(state) + .unwrap_or_else(|_| unreachable!("a freshly built state should have one owner")); + state.identity_graph = Some(graph); + Arc::new(state) +} + fn warn_if_certificate_check_disabled(settings: &Settings) { if !settings.proxy.certificate_check { log::warn!( @@ -330,21 +349,20 @@ fn uses_dynamic_tsjs_fallback(method: &Method, path: &str) -> bool { /// /// `edgezero_main` pops this from the response after dispatch and runs /// [`trusted_server_core::ec::finalize::ec_finalize_response`] plus the -/// pull-sync hook on the converted fastly response — the same EC response +/// after-send identity flush on the converted fastly response — the same EC response /// lifecycle the legacy path drives through `RouteResult`. #[derive(Clone)] pub(crate) struct EcFinalizeState { pub(crate) ec_context: EcContext, /// Whether EC finalization may write to the KV identity graph. - /// `KvIdentityGraph` wraps a non-`Sync` `dyn EcKvStore` and cannot ride - /// in response extensions, so `edgezero_main` rebuilds the graph from - /// settings when this is set. + /// `edgezero_main` takes the graph from application state when this is + /// set. pub(crate) use_finalize_kv: bool, pub(crate) eids_cookie: Option, pub(crate) sharedid_cookie: Option, pub(crate) is_real_browser: bool, - /// Per-request services carried to the entry point so the pull-sync - /// dispatcher can reuse the same platform HTTP client. + /// Per-request services carried to the entry point so the after-send + /// providers reuse the same platform HTTP client. pub(crate) services: RuntimeServices, } @@ -491,7 +509,7 @@ fn build_ec_request_state( // Bot gate: suppress KV-backed EC writes for unrecognized clients, except // consent withdrawals. Revocations keep the write path so tombstones stay // authoritative even for privacy-extension-heavy clients. - let kv_graph = crate::maybe_identity_graph(settings); + let kv_graph = state.identity_graph.clone(); let finalize_kv_graph = if setup_error.is_none() && (is_real_browser || ec_consent_withdrawn(ec_context.consent())) { @@ -626,7 +644,7 @@ async fn execute_named( // Deliberately do not use an EC request-state graph: that // copy is bot-gated, while operators use curl for this // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); + let kv = state.identity_graph.clone(); handle_admin_ec_lookup( kv.as_ref(), ®istry, @@ -784,8 +802,7 @@ async fn run_named_route( /// graph + partner registry + rate limiter, with a default EC context for /// response finalization. fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> Response { - let device_signals = device_signals_for(&req); - let is_real_browser = device_signals.looks_like_browser(); + let is_real_browser = device_signals_for(&req).looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); @@ -1348,7 +1365,7 @@ impl TrustedServerApp { (Self::routes_for_state(&state), Some(state)) } - fn routes_for_state(state: &Arc) -> RouterService { + pub(crate) fn routes_for_state(state: &Arc) -> RouterService { let mut router = RouterService::builder() .middleware(FinalizeResponseMiddleware::new( Arc::clone(&state.settings), @@ -1642,6 +1659,7 @@ mod tests { registry: Arc::new(registry), default_kv_store, identity, + identity_graph: None, }); TrustedServerApp::routes_for_state(&state) } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index ca0e909dfc..d675e6206f 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -15,10 +15,12 @@ use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_h use trusted_server_core::constants::HEADER_X_TS_VERSION; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::{ec_finalize_response, retry_failed_withdrawals}; -use trusted_server_core::ec::kv::KvIdentityGraph; -use trusted_server_core::ec::pull_sync::{ - PullSyncContext, build_pull_sync_context, dispatch_pull_sync, +use trusted_server_core::ec::identity::IdentityRuntime; +use trusted_server_core::ec::identity::flush::{ + AfterSendInput, FlushOutcome, may_flush, run_after_send, }; +use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::rate_limiter::RateLimiter; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::RequestFilterEffects; @@ -506,57 +508,56 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re } if let Some(mut ec_state) = ec_state { - if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { - Ok(partner_registry) => { - send_edgezero_response( + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let mut send = |response: HttpResponse| { + send_edgezero_response(response, request_filter_effects.as_ref(), counters.as_ref()); + }; + if let Some(state) = app_state.as_deref() { + let graph = || state.identity_graph.clone(); + finalize_send_and_flush( + PostRouteInput { + runtime: &state.identity, + settings: &state.settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &limiter, + }, + response, + &mut send, + ); + return; + } + match load_settings_from_config_store(&runtime_stores) { + Ok(settings) => match build_identity_runtime(&settings) { + Ok(identity) => { + // No application state was built, so the identity runtime + // comes from the reloaded settings. + ec_state + .ec_context + .set_identity_runtime(Arc::clone(&identity)); + let graph = || maybe_identity_graph(&settings); + finalize_send_and_flush( + PostRouteInput { + runtime: &identity, + settings: &settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &limiter, + }, response, - request_filter_effects.as_ref(), - counters.as_ref(), + &mut send, ); - run_edgezero_after_send(settings, &partner_registry, &mut ec_state); return; } Err(e) => { - log::error!( - "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" + log::warn!( + "EdgeZero EC finalize skipped: identity runtime unavailable after settings reload: {}", + e.current_context() ); } - } - } else { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) => { - // No application state was built, so build the identity - // runtime from the reloaded settings. - match build_identity_runtime(&settings) { - Ok(identity) => ec_state.ec_context.set_identity_runtime(identity), - Err(e) => { - log::warn!( - "identity runtime unavailable after settings reload: {}", - e.current_context() - ); - } - } - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { - Ok(partner_registry) => { - send_edgezero_response( - response, - request_filter_effects.as_ref(), - counters.as_ref(), - ); - run_edgezero_after_send(&settings, &partner_registry, &mut ec_state); - return; - } - Err(e) => { - log::error!( - "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" - ); - } - } - } - Err(e) => { - log::warn!("EdgeZero EC finalize skipped: failed to reload settings: {e:?}"); - } + }, + Err(e) => { + log::warn!("EdgeZero EC finalize skipped: failed to reload settings: {e:?}"); } } } @@ -599,14 +600,48 @@ fn apply_entry_point_finalize_headers( apply_finalize_headers(settings, geo_info.as_ref(), response); } -fn apply_edgezero_ec_finalize( - settings: &Settings, - ec_state: &mut EcFinalizeState, - response: &mut HttpResponse, -) -> Result> { - let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; +/// What the post-route work reads. It takes the runtime and settings rather +/// than [`AppState`], because the settings-reload branch has none. +struct PostRouteInput<'a> { + runtime: &'a IdentityRuntime, + settings: &'a Settings, + ec_state: &'a mut EcFinalizeState, + /// Builds the identity graph. Called only when finalize or the after-send + /// work needs one, and at most once. + graph: &'a dyn Fn() -> Option, + rate_limiter: &'a dyn RateLimiter, +} + +/// Finalizes the response, hands it to `send`, then runs the after-send work: +/// the withdrawal retry first, then phase 4's one flush (spec 5.2). +/// +/// `send` is a callback so phase 4 runs only after the response is sent. +/// Returns phase 4's outcome, or `None` when finalize could not run. +fn finalize_send_and_flush( + input: PostRouteInput<'_>, + mut response: HttpResponse, + send: &mut dyn FnMut(HttpResponse), +) -> Option { + let PostRouteInput { + runtime, + settings, + ec_state, + graph, + rate_limiter, + } = input; + let partner_registry = match PartnerRegistry::from_config(&settings.ec.partners) { + Ok(registry) => registry, + Err(e) => { + log::error!("EdgeZero EC finalize skipped: failed to build partner registry: {e:?}"); + send(response); + return None; + } + }; + let mut built: Option> = None; + let mut graph_once = || built.get_or_insert_with(graph).clone(); + let finalize_kv_graph = if ec_state.use_finalize_kv { - maybe_identity_graph(settings) + graph_once() } else { None }; @@ -617,92 +652,33 @@ fn apply_edgezero_ec_finalize( &partner_registry, ec_state.eids_cookie.as_deref(), ec_state.sharedid_cookie.as_deref(), - response, + &mut response, ); - Ok(partner_registry) -} + let status = response.status(); + send(response); -/// Runs the work that follows the response: the withdrawal retry, then pull sync. -fn run_edgezero_after_send( - settings: &Settings, - partner_registry: &PartnerRegistry, - ec_state: &mut EcFinalizeState, -) { - run_after_send_in_order( - ec_state, - |state| state.ec_context.has_failed_withdrawals(), - |state| match require_identity_graph(settings) { - Ok(graph) => retry_failed_withdrawals(&mut state.ec_context, &graph), - Err(err) => { - log::warn!( - "withdrawal retry skipped: identity graph unavailable: {}", - err.current_context() - ); - } - }, - |state| run_edgezero_pull_sync_after_send(settings, partner_registry, state), - ); -} - -/// Runs the withdrawal retry, when one is due, before any identity work. -/// -/// A withdrawal request runs no provider dispatch or mutation flush, so the -/// order only keeps the retry's log lines ahead of the identity work's. -fn run_after_send_in_order( - state: &mut S, - has_failed_withdrawal: impl FnOnce(&S) -> bool, - retry_withdrawals: impl FnOnce(&mut S), - identity_work: impl FnOnce(&S), -) { - if has_failed_withdrawal(state) { - retry_withdrawals(state); + if ec_state.ec_context.has_failed_withdrawals() { + match graph_once() { + Some(graph) => retry_failed_withdrawals(&mut ec_state.ec_context, &graph), + None => log::warn!("withdrawal retry skipped: identity graph unavailable"), + } } - identity_work(state); -} - -fn run_edgezero_pull_sync_after_send( - settings: &Settings, - partner_registry: &PartnerRegistry, - ec_state: &EcFinalizeState, -) { - if !ec_state.is_real_browser { - return; + if !ec_state.is_real_browser || !may_flush(&ec_state.ec_context) { + return Some(FlushOutcome::Skipped); } - - let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); - let Some((context, kv)) = - prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) - else { - return; + let Some(graph) = graph_once() else { + log::debug!("identity flush skipped: identity graph unavailable"); + return Some(FlushOutcome::Skipped); }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync( + Some(run_after_send(AfterSendInput { + runtime, settings, - &kv, - partner_registry, - &limiter, - &context, - &ec_state.services, - ); -} - -fn prepare_pull_sync_after_send( - context: Option, - graph_factory: F, -) -> Option<(PullSyncContext, KvIdentityGraph)> -where - F: FnOnce() -> Result>, -{ - let context = context?; - let kv = match graph_factory() { - Ok(kv) => kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return None; - } - }; - Some((context, kv)) + services: &ec_state.services, + graph: &graph, + context: &mut ec_state.ec_context, + status, + rate_limiter, + })) } /// Sends a finalized `EdgeZero` response to the client. @@ -890,6 +866,7 @@ pub(crate) fn derive_device_signals(req: &FastlyRequest) -> DeviceSignals { #[cfg(test)] mod tests { use super::*; + use crate::app::TrustedServerApp; use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; @@ -922,63 +899,199 @@ mod tests { .expect("should parse test settings") } - #[test] - fn after_send_runs_the_withdrawal_retry_before_identity_work() { - let log_cell = std::cell::RefCell::new(Vec::<&str>::new()); - let mut state = true; - - run_after_send_in_order( - &mut state, - |failed| *failed, - |failed| { - log_cell.borrow_mut().push("retry"); - *failed = false; - }, - |failed| { - assert!(!failed, "should see the state the retry left"); - log_cell.borrow_mut().push("identity"); + const EC_ID: &str = "0000000000000000000000000000000000000000000000000000000000000000.ABC123"; + + /// Allows every call. + struct AllowAll; + + impl RateLimiter for AllowAll { + fn exceeded(&self, _key: &str, _limit: u32) -> Result> { + Ok(false) + } + } + + /// A browser document navigation carrying `cookie`. + fn navigation(cookie: Option<&str>) -> HttpRequest { + let mut builder = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::GET) + .uri("https://test-publisher.com/some-page") + .header("sec-fetch-dest", "document"); + if let Some(cookie) = cookie { + builder = builder.header("cookie", cookie); + } + let mut request = builder + .body(EdgeBody::empty()) + .expect("should build request"); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + request + } + + /// Routes `request`, then finalizes, sends and flushes it as + /// `edgezero_main` does, keeping the sent response. + fn drive_request_for_test( + state: &Arc, + request: HttpRequest, + ) -> (HttpResponse, Option) { + let router = TrustedServerApp::routes_for_state(state); + let mut response = + futures::executor::block_on(router.oneshot(request)).expect("should route the request"); + let Some(mut ec_state) = response.extensions_mut().remove::() else { + return (response, None); + }; + let mut sent = None; + let graph = || state.identity_graph.clone(); + let outcome = finalize_send_and_flush( + PostRouteInput { + runtime: &state.identity, + settings: &state.settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &AllowAll, }, + response, + &mut |response| sent = Some(response), + ); + (sent.expect("should send the response"), outcome) + } + + /// A returning visitor's live row, without a cluster size so the flush + /// has one change to write. + fn seeded_state() -> ( + Arc, + Arc, + ) { + let (graph, store) = KvIdentityGraph::write_counting("drive-store"); + let consent = trusted_server_core::ec::identity::test_support::allowing_consent(); + let entry = trusted_server_core::ec::kv_types::KvEntry::new( + &consent, + None, + 1_000, + "test-publisher.com", ); + graph.create(EC_ID, &entry).expect("should seed the row"); + store.reset_counts(); + let state = crate::app::app_state_with_graph_for_test(test_settings(), graph); + (state, store) + } + #[test] + fn drive_request_for_test_counts_routing_finalize_and_flush_against_one_store() { + let (state, store) = seeded_state(); + + let (_, outcome) = + drive_request_for_test(&state, navigation(Some(&format!("ts-ec={EC_ID}")))); + + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush once"); + let writes = store.writes(); + assert_eq!(writes.lookups, 1, "should read the row once, in phase 1"); assert_eq!( - *log_cell.borrow(), - ["retry", "identity"], - "should retry the withdrawal before identity work" + writes.if_generation_match, 1, + "should write once with a CAS" ); + assert_eq!(writes.total(), 1, "should make no other write"); } #[test] - fn after_send_skips_the_retry_without_a_failed_withdrawal() { - let retried = std::cell::Cell::new(false); - let worked = std::cell::Cell::new(false); - let mut state = false; - - run_after_send_in_order( - &mut state, - |failed| *failed, - |_| retried.set(true), - |_| worked.set(true), + fn finalize_send_and_flush_sends_before_it_flushes() { + let (state, store) = seeded_state(); + let router = TrustedServerApp::routes_for_state(&state); + let mut response = futures::executor::block_on( + router.oneshot(navigation(Some(&format!("ts-ec={EC_ID}")))), + ) + .expect("should route the request"); + let mut ec_state = response + .extensions_mut() + .remove::() + .expect("should carry EC state"); + let writes_at_send = std::cell::Cell::new(None); + let graph = || state.identity_graph.clone(); + + let outcome = finalize_send_and_flush( + PostRouteInput { + runtime: &state.identity, + settings: &state.settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &AllowAll, + }, + response, + &mut |_| writes_at_send.set(Some(store.writes().total())), ); - assert!(!retried.get(), "should not retry when nothing failed"); - assert!(worked.get(), "should still run identity work"); + assert_eq!( + writes_at_send.get(), + Some(0), + "should send before any write" + ); + assert_eq!(outcome, Some(FlushOutcome::Written), "should then flush"); + assert_eq!(store.writes().total(), 1, "should write once"); } #[test] - fn pull_sync_noop_states_skip_post_send_graph_factory() { + fn after_send_noop_states_skip_post_send_graph_factory() { + let (state, _) = seeded_state(); + let router = TrustedServerApp::routes_for_state(&state); + // A named route is never the writer, so phase 4 has nothing to do. + let request = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::GET) + .uri("https://test-publisher.com/.well-known/trusted-server.json") + .body(EdgeBody::empty()) + .expect("should build request"); + let mut response = + futures::executor::block_on(router.oneshot(request)).expect("should route"); + let mut ec_state = response + .extensions_mut() + .remove::() + .expect("should carry EC state"); + ec_state.use_finalize_kv = false; let calls = std::cell::Cell::new(0); - let result = prepare_pull_sync_after_send(None, || { + let graph = || { calls.set(calls.get() + 1); - Err(Report::new(TrustedServerError::KvStore { - store_name: "unexpected".to_owned(), - message: "graph factory should not run".to_owned(), - })) - }); + None + }; + + let outcome = finalize_send_and_flush( + PostRouteInput { + runtime: &state.identity, + settings: &state.settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &AllowAll, + }, + response, + &mut drop, + ); + + assert_eq!(outcome, Some(FlushOutcome::Skipped), "should skip phase 4"); + assert_eq!(calls.get(), 0, "should not build the graph"); + } + + #[test] + fn no_response_sets_or_reads_the_pull_complete_cookie() { + let (state, store) = seeded_state(); + let cookie = format!("ts-ec={EC_ID}; ts-ec-pull-complete=v1.example"); + + let (response, _) = drive_request_for_test(&state, navigation(Some(&cookie))); + assert!( - result.is_none(), - "a skipped pull-sync plan should return none" + response + .headers() + .get_all("set-cookie") + .iter() + .filter_map(|value| value.to_str().ok()) + .all(|value| !value.starts_with("ts-ec-pull-complete")), + "should set no pull-complete cookie" + ); + assert_eq!( + store.writes().lookups, + 1, + "should read the row whatever the old marker cookie says" ); - assert_eq!(calls.get(), 0, "should not invoke the graph factory"); } #[test] diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index 458786db01..41c5644113 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -1,8 +1,6 @@ use http::header::HeaderName; pub const COOKIE_TS_EC: &str = "ts-ec"; -/// Short-lived signed proof that the current EC row has every pull-partner UID. -pub const COOKIE_TS_EC_PULL_COMPLETE: &str = "ts-ec-pull-complete"; /// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 5c159cdd89..21c7f49b45 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -23,7 +23,6 @@ use super::kv::{ }; use super::kv_types::{KvEntry, KvPartnerId}; use super::prebid_eids::collect_eid_updates; -use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; use super::{EcKvSnapshot, EidSyncSource, checked_current_timestamp, current_timestamp, log_id}; @@ -60,17 +59,10 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { - ec_context.validate_pull_sync_marker(settings, registry); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); if !consent_allows_ec { - // Expire the request-local marker independently of the EC cookie: a - // withdrawal must stop any pending pull-sync disclosure window. - if consent_withdrawn && ec_context.pull_sync_marker().was_present() { - expire_marker(ec_context.pull_sync_marker_mut(), response); - } - finalize_unusable_consent( settings, ec_context, @@ -126,8 +118,6 @@ pub fn ec_finalize_response( } } - reconcile_pull_sync_marker(settings, registry, ec_context, response); - // Ordinary returning-user page views no longer refresh the browser // cookie, emit the EC header, or update KV TTL. return; @@ -139,7 +129,6 @@ pub fn ec_finalize_response( if ec_context.ec_generated() { let (Some(_), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { log::info!("Skipping generated EC response write because KV graph is unavailable"); - reconcile_pull_sync_marker(settings, registry, ec_context, response); return; }; @@ -164,8 +153,6 @@ pub fn ec_finalize_response( log::warn!("Skipping generated EC cookie because backing row is not authoritative"); } } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); } /// The browser ingest of one request, ready to seed a new identity row. @@ -302,24 +289,6 @@ fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome ); } -fn reconcile_pull_sync_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_context: &mut EcContext, - response: &mut Response, -) { - let ec_id = ec_context.ec_value().map(str::to_owned); - let snapshot = ec_context.kv_snapshot().clone(); - reconcile_marker( - settings, - registry, - ec_id.as_deref(), - &snapshot, - ec_context.pull_sync_marker_mut(), - response, - ); -} - /// Withholds EID-derived KV updates when consent does not allow EID /// persistence, on top of the Purpose 1 (EC) gate `ec_finalize_response` /// already enforces before reaching this point. @@ -1269,58 +1238,6 @@ mod tests { ); } - #[test] - fn valid_marker_with_unread_snapshot_defers_orphan_recovery() { - let settings = create_test_settings(); - let orphaned_ec = sample_ec_id("orphn2"); - let consent = ConsentContext { - jurisdiction: Jurisdiction::NonRegulated, - source: ConsentSource::Cookie, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test_with_ip( - Some(orphaned_ec.clone()), - consent, - Some("192.0.2.10".to_owned()), - ); - ec_context.set_recovery_eligible(true); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Valid { expires_at: 4_600 }, - ); - let mut partner = make_partner("pull.example.com"); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; - partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); - let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - let graph = KvIdentityGraph::in_memory("test_store"); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - ®istry, - None, - None, - &mut response, - ); - - assert_eq!( - ec_context.ec_value(), - Some(orphaned_ec.as_str()), - "an unread snapshot should defer orphan rotation until marker expiry" - ); - assert!( - matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead), - "a marker-skipped request should leave the snapshot unread" - ); - assert!( - response.headers().get(http::header::SET_COOKIE).is_none(), - "bounded orphan deferral should not rewrite browser identity state" - ); - } - #[test] fn finalize_returning_user_subresource_does_not_persist_eid_updates() { let settings = create_test_settings(); @@ -2353,9 +2270,6 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, - ); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2386,69 +2300,10 @@ mod tests { .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), "KV failure should not prevent EC cookie deletion" ); - assert!( - cookies.iter().any(|cookie| { - cookie.starts_with("ts-ec-pull-complete=;") && cookie.contains("Max-Age=0") - }), - "KV failure should not prevent marker deletion" - ); } #[test] - fn finalize_sets_marker_for_complete_pull_partner_snapshot() { - let settings = create_test_settings(); - let ec_id = sample_ec_id("compl1"); - let mut partner = make_partner("ssp.example.com"); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; - partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); - let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - let mut ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::NonRegulated, - ); - let mut entry = live_entry(); - entry.ids.insert( - "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::new("partner-uid"), - ); - ec_context.set_kv_snapshot(EcKvSnapshot::Present { - ec_id, - entry: Box::new(entry), - generation: Some(1), - }); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - None, - ®istry, - None, - None, - &mut response, - ); - - let cookies = response - .headers() - .get_all(http::header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect::>(); - assert!( - cookies - .iter() - .any(|cookie| cookie.starts_with("ts-ec-pull-complete=v1.")), - "complete snapshot should issue the marker" - ); - } - - #[test] - fn explicit_withdrawal_without_marker_or_ec_cookie_does_not_set_cookie() { + fn explicit_withdrawal_without_ec_cookie_does_not_set_cookie() { let settings = create_test_settings(); let consent = ConsentContext { jurisdiction: Jurisdiction::UsState("CA".to_owned()), @@ -2475,49 +2330,6 @@ mod tests { ); } - #[test] - fn explicit_withdrawal_expires_marker_without_ec_cookie() { - let settings = create_test_settings(); - let consent = ConsentContext { - jurisdiction: Jurisdiction::UsState("CA".to_owned()), - gpc: true, - source: ConsentSource::Cookie, - ..Default::default() - }; - let mut ec_context = make_context_with_consent(None, None, false, false, consent); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, - ); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - None, - &PartnerRegistry::empty(), - None, - None, - &mut response, - ); - - let cookies = response - .headers() - .get_all(http::header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect::>(); - assert!( - cookies.iter().any(|cookie| { - cookie.starts_with("ts-ec-pull-complete=;") && cookie.contains("Max-Age=0") - }), - "withdrawal should expire the marker independently of EC cookie state" - ); - assert!( - cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), - "missing EC cookie should not add an EC-cookie expiry" - ); - } - mod identity_flow { use serde_json::json; diff --git a/crates/trusted-server-core/src/ec/identity/flush.rs b/crates/trusted-server-core/src/ec/identity/flush.rs new file mode 100644 index 0000000000..fa83f82fdf --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/flush.rs @@ -0,0 +1,1229 @@ +//! Phase 4: the writer's one flush after send (spec 5.2). +//! +//! The writer request stages every identity change in one +//! [`IdentityMutation`]. After the response is sent, this module dispatches +//! the after-send providers and writes the whole mutation once, with +//! `IfGenerationMatch`. A failed flush is not retried as a whole; only its +//! removals are written again, alone (spec 5.2 phase 4 step 5). + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::IpAddr; + +use http::StatusCode; + +use super::coordinator::{ + Call, Eligibility, EligibilityInput, EligibilityView, OutcomeChanges, OutcomeContext, Phase, + RequestFacts, dispatch, eligibility, fingerprint_bits, outcome_to_changes, +}; +use super::descriptor::{Input, Resolves}; +use super::ids::{IdentityProviderId, SourceDomain, UnixTime}; +use super::input::{ProviderConsent, Region, ResolveSignals, StoredId, ThreadRandom}; +use super::mutation::{ApplyContext, IdentityMutation, apply}; +use super::policy::PolicySignals; +use super::provider::IdentityProvider; +use super::removal::{log_removal_lost, write_removals}; +use super::revision::StateRevision; +use super::runtime::IdentityRuntime; +use crate::consent::allows_eid_persistence; +use crate::ec::kv::KvIdentityGraph; +use crate::ec::kv_backend::EcKvWriteOutcome; +use crate::ec::kv_types::KvEntry; +use crate::ec::rate_limiter::RateLimiter; +use crate::ec::{EcContext, EcKvSnapshot, current_timestamp, log_id}; +use crate::platform::RuntimeServices; +use crate::settings::Settings; + +pub use super::removal::RemovalWriteOutcome; + +/// What phase 4 needs. The catalog and keys come from `runtime`. +pub struct AfterSendInput<'a> { + /// The identity runtime of this settings load. + pub runtime: &'a IdentityRuntime, + /// The settings the request ran with. + pub settings: &'a Settings, + /// The request's platform services, for the after-send calls. + pub services: &'a RuntimeServices, + /// The identity graph to read and write. + pub graph: &'a KvIdentityGraph, + /// The request's EC state, holding the writer decision and the mutation. + pub context: &'a mut EcContext, + /// The status of the response that was sent. + pub status: StatusCode, + /// The per-visitor burst cap. + pub rate_limiter: &'a dyn RateLimiter, +} + +/// How phase 4 ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FlushOutcome { + /// Not a returning EC's writer, or nothing to do. Nothing was read. + Skipped, + /// The mutation changed nothing in the row, so nothing was written. + NoChange, + /// The one write landed. + Written, + /// The row was a tombstone, missing or unreadable, or the merged row was + /// invalid. The mutation was dropped and no provider was called after it. + Dropped(&'static str), + /// The one write failed. `removals` is how the removal-only retry ended. + WriteFailed { + /// The removal-only retry's outcome. + removals: RemovalWriteOutcome, + }, +} + +/// Whether phase 4 can have work for this request: it is a returning EC's +/// writer. A caller checks this before building the identity graph. +#[must_use] +pub fn may_flush(context: &EcContext) -> bool { + context.identity().writer() && context.ec_was_present() && !context.ec_generated() +} + +/// Runs phase 4 for the writer request (spec 5.2, steps 1 to 5). +/// +/// It reads the row at most once, calls the due after-send providers only +/// after that read confirms a live, consenting row, and writes at most once +/// with the generation it read. Only a failed write that holds removals adds +/// the removal-only write of spec 5.3. +#[must_use] +pub fn run_after_send(input: AfterSendInput<'_>) -> FlushOutcome { + let AfterSendInput { + runtime, + settings, + services, + graph, + context, + status, + rate_limiter, + } = input; + + let returning_writer = may_flush(context); + let Some(ec_id) = context.ec_value().map(str::to_owned) else { + return FlushOutcome::Skipped; + }; + if !returning_writer { + return FlushOutcome::Skipped; + } + + let mut mutation = core::mem::take(context.identity_mut().mutation_mut()); + let status_ok = status.is_success() || status == StatusCode::NOT_MODIFIED; + if !status_ok { + mutation.retain_for_error_status(); + } + + let now = current_timestamp(); + let facts = request_facts(context, now); + let phase_one = context.kv_snapshot().clone(); + let phase_one_entry = phase_one.entry_for(&ec_id); + let phase_one_view = + phase_one_entry.map_or_else(EligibilityView::empty, EligibilityView::from_kv); + let due = if status_ok { + due_providers(runtime, settings, context, &phase_one_view, &facts) + } else { + Vec::new() + }; + let needs_cluster_size = phase_one_entry.is_some_and(lacks_cluster_size); + + if mutation.is_empty() && due.is_empty() && !needs_cluster_size { + return FlushOutcome::Skipped; + } + + // Phase 1's read decides when no provider is due (spec 5.2 phase 4 + // step 2). A failed or absent phase 1 read counts as no read. + let reusable = match &phase_one { + EcKvSnapshot::Present { + entry, + generation: Some(generation), + .. + } if due.is_empty() && phase_one.belongs_to(&ec_id) => { + Some((entry.as_ref().clone(), *generation)) + } + EcKvSnapshot::Missing { .. } if due.is_empty() && phase_one.belongs_to(&ec_id) => { + return drop_mutation(&ec_id, "missing"); + } + _ => None, + }; + let (live, generation) = match reusable { + Some(snapshot) => snapshot, + None => match graph.get(&ec_id) { + Ok(Some(row)) => row, + Ok(None) => return drop_mutation(&ec_id, "missing"), + Err(error) => { + log::warn!( + "Identity flush for '{}' could not read the row: {}", + log_id(&ec_id), + error.current_context() + ); + return drop_mutation(&ec_id, "read failed"); + } + }, + }; + if !live.consent.ok { + return drop_mutation(&ec_id, "tombstone"); + } + + // The row is live and consenting: only now may the EC ID leave TS. + let live_view = EligibilityView::from_kv(&live); + let calls: Vec = due + .iter() + .filter_map(|id| live_call(runtime, settings, context, &live_view, &facts, id)) + .collect(); + if !calls.is_empty() { + add_dispatch_results( + &mut mutation, + &DispatchInput { + runtime, + settings, + services, + context, + rate_limiter, + ec_id: &ec_id, + live: &live, + view: &live_view, + facts: &facts, + }, + calls, + ); + } + + if lacks_cluster_size(&live) { + match graph.compute_cluster_size(&ec_id) { + Ok(size) => mutation.set_cluster_size(size), + Err(error) => log::warn!( + "Identity flush for '{}' could not compute the cluster size: {}", + log_id(&ec_id), + error.current_context() + ), + } + } + + let random = ThreadRandom; + let apply_context = ApplyContext { + now, + random: &random, + catalog: runtime.catalog(), + }; + let Some(applied) = apply(&mutation, &live, &apply_context) else { + return drop_mutation(&ec_id, "invalid"); + }; + if !applied.changed { + return FlushOutcome::NoChange; + } + + match graph.write_if_generation(&ec_id, &applied.entry, generation) { + Ok(EcKvWriteOutcome::Written) => FlushOutcome::Written, + Ok(EcKvWriteOutcome::PreconditionFailed) => { + log::warn!( + "Identity flush for '{}' lost its generation check; not retried", + log_id(&ec_id) + ); + write_failed(graph, context, &ec_id, &mutation, &apply_context) + } + Err(error) => { + log::warn!( + "Identity flush for '{}' failed to write: {}", + log_id(&ec_id), + error.current_context() + ); + write_failed(graph, context, &ec_id, &mutation, &apply_context) + } + } +} + +fn drop_mutation(ec_id: &str, reason: &'static str) -> FlushOutcome { + log::debug!( + "Identity flush for '{}' dropped its mutation: {reason}", + log_id(ec_id) + ); + FlushOutcome::Dropped(reason) +} + +fn lacks_cluster_size(entry: &KvEntry) -> bool { + entry.consent.ok + && entry + .network + .as_ref() + .and_then(|network| network.cluster_size) + .is_none() +} + +/// Writes the failed flush's removals alone, and counts each one every +/// attempt failed to write (spec 5.2 phase 4 step 5). +fn write_failed( + graph: &KvIdentityGraph, + context: &mut EcContext, + ec_id: &str, + mutation: &IdentityMutation, + apply_context: &ApplyContext<'_>, +) -> FlushOutcome { + let removals = mutation.removals(); + if !removals.has_removals() { + return FlushOutcome::WriteFailed { + removals: RemovalWriteOutcome::NothingToWrite, + }; + } + let outcome = write_removals(graph, ec_id, &removals, apply_context); + if outcome == RemovalWriteOutcome::Failed { + log_removal_lost(ec_id, &removals); + for _ in removals.removal_labels() { + context.identity_mut().record_removal_lost(); + } + } + FlushOutcome::WriteFailed { removals: outcome } +} + +/// What eligibility reads about the request itself. +fn request_facts(context: &EcContext, now: u64) -> RequestFacts { + let identity = context.identity(); + let mut available_inputs = BTreeSet::new(); + for (input, present) in [ + (Input::ClientIp, client_ip(context).is_some()), + (Input::UserAgent, identity.user_agent().is_some()), + (Input::PageUrl, identity.page_url().is_some()), + (Input::Region, region(context).is_some()), + ] { + if present { + available_inputs.insert(input); + } + } + RequestFacts { + is_real_browser: identity.is_real_browser(), + ec_persistence_allowed: context.ec_allowed() && allows_eid_persistence(context.consent()), + policy_signals: PolicySignals { + dnt: identity.dnt(), + }, + writer: identity.writer(), + new_visitor: context.ec_generated(), + // Hold dispatch before the auction is not wired yet, so no hold-mode + // provider has run before phase 4: each runs here, as on a navigation + // that runs no auction (spec 4.7 step 7). + runs_auction: false, + now: UnixTime::from_secs(now), + available_inputs, + } +} + +fn client_ip(context: &EcContext) -> Option { + context.client_ip().and_then(|ip| ip.parse().ok()) +} + +fn region(context: &EcContext) -> Option { + context + .geo_info() + .and_then(|geo| Region::from_geo(&geo.country, geo.region.as_deref())) +} + +/// The eligibility of `provider` for phase 4 against `view`. +fn after_send_eligibility( + provider: &dyn IdentityProvider, + runtime: &IdentityRuntime, + settings: &Settings, + context: &EcContext, + view: &EligibilityView, + facts: &RequestFacts, +) -> Eligibility { + let provider_settings = settings.ec.identity.provider(provider.descriptor().id()); + eligibility( + &EligibilityInput { + provider, + settings: &provider_settings, + identity: &settings.ec.identity, + catalog: runtime.catalog(), + consent: context.consent(), + }, + view, + facts, + Phase::AfterSend, + ) +} + +/// The after-send providers due on the phase 1 view. +fn due_providers( + runtime: &IdentityRuntime, + settings: &Settings, + context: &EcContext, + view: &EligibilityView, + facts: &RequestFacts, +) -> Vec { + runtime + .catalog() + .providers() + .filter(|provider| { + matches!( + after_send_eligibility(provider.as_ref(), runtime, settings, context, view, facts), + Eligibility::Eligible { .. } + ) + }) + .map(|provider| provider.descriptor().id().clone()) + .collect() +} + +/// The call of a provider still due on the live row, with its live `ps.st`. +fn live_call( + runtime: &IdentityRuntime, + settings: &Settings, + context: &EcContext, + view: &EligibilityView, + facts: &RequestFacts, + id: &IdentityProviderId, +) -> Option { + let provider = runtime.catalog().get(id)?; + let Eligibility::Eligible { kinds } = + after_send_eligibility(provider.as_ref(), runtime, settings, context, view, facts) + else { + return None; + }; + Some(Call { + provider: id.clone(), + kinds, + state_json: view + .state(id) + .and_then(|state| serde_json::to_string(state).ok()), + }) +} + +/// What dispatching the due providers reads. +struct DispatchInput<'a> { + runtime: &'a IdentityRuntime, + settings: &'a Settings, + services: &'a RuntimeServices, + context: &'a EcContext, + rate_limiter: &'a dyn RateLimiter, + ec_id: &'a str, + live: &'a KvEntry, + view: &'a EligibilityView, + facts: &'a RequestFacts, +} + +/// Dispatches `calls` after send and adds their outcomes to `mutation`. +fn add_dispatch_results( + mutation: &mut IdentityMutation, + input: &DispatchInput<'_>, + calls: Vec, +) { + let random = ThreadRandom; + let resolving: BTreeSet<&SourceDomain> = + calls.iter().flat_map(|call| call.kinds.keys()).collect(); + let stored: BTreeMap = input + .view + .records() + .filter(|(source, _)| resolving.contains(source)) + .filter_map(|(source, record)| { + StoredId::from_record(source, record).map(|stored| (source.clone(), stored)) + }) + .collect(); + let identity = input.context.identity(); + let signals = ResolveSignals { + client_ip: client_ip(input.context), + user_agent: identity.user_agent(), + page_url: identity.page_url().cloned(), + region: region(input.context), + hashed_email: None, + consent: ProviderConsent::from_context(input.context.consent()), + publisher_domain: &input.settings.publisher.domain, + now: input.facts.now, + random: &random, + ec_id: Some(input.ec_id.to_owned()), + stored, + }; + let deps = input.runtime.deps( + input.settings, + input.services, + &random, + Some(input.rate_limiter), + ); + let catalog = input.runtime.catalog(); + for (id, outcome) in dispatch(&deps, calls, &signals, None) { + let Some(provider) = catalog.get(&id) else { + continue; + }; + let provider: &dyn IdentityProvider = provider.as_ref(); + let descriptor = provider.descriptor(); + let provider_settings = input.settings.ec.identity.provider(&id); + let fingerprint = fingerprint_bits( + descriptor.consent_policy(), + input.context.consent(), + input.facts.policy_signals, + ); + let stateful = descriptor + .resolves() + .is_some_and(Resolves::has_private_state); + let outcome_context = OutcomeContext { + now: input.facts.now, + catalog, + settings: &provider_settings, + identity: &input.settings.ec.identity, + expected_state: stateful.then(|| { + StateRevision::of( + input + .live + .ps + .get(id.as_str()) + .and_then(|state| state.st.as_ref()), + ) + }), + fingerprint: &fingerprint, + }; + let changes = outcome_to_changes(provider, outcome, input.view, &outcome_context); + add_outcome(mutation, &id, changes); + } +} + +/// Adds one outcome's changes to the mutation. +fn add_outcome(mutation: &mut IdentityMutation, id: &IdentityProviderId, changes: OutcomeChanges) { + let OutcomeChanges { + changes, + result, + state, + } = changes; + for change in changes { + mutation.add_change(change); + } + if let Some(result) = result { + mutation.add_result(result); + } + mutation.merge_state(id.clone(), state); +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::sync::Arc; + + use error_stack::Report; + + use super::*; + use crate::ec::identity::mutation::{PartnerIdChange, ProviderStateChange}; + use crate::ec::identity::request::WriterDecision; + use crate::ec::identity::revision::{record_revision, visitor_key}; + use crate::ec::identity::test_support::fake_http::{ + FakeBackend, FakeHttpClient, fake_services, + }; + use crate::ec::identity::test_support::{ + TEST_EC_ID, WriteCountingEcKv, allowing_consent, ec_partner, example_entry_set, + full_record, pid, record_for_test, source, + }; + use crate::ec::kv_types::{KvNetwork, KvOptOut, KvPartnerId}; + use crate::error::TrustedServerError; + use crate::settings::{ProviderMode, ProviderSettings}; + use crate::test_support::tests::create_test_settings; + + const PULL_HOST: &str = "sync.ssp.example.com"; + const PULL_SOURCE: &str = "ssp.example.com"; + const PULL_ID: &str = "pull-ssp-example-com"; + + /// Answers every burst-cap read with `exceeded`, and records the keys. + struct ScriptedLimiter { + exceeded: bool, + keys: RefCell>, + } + + impl ScriptedLimiter { + fn allowing() -> Self { + Self { + exceeded: false, + keys: RefCell::new(Vec::new()), + } + } + + fn capped() -> Self { + Self { + exceeded: true, + keys: RefCell::new(Vec::new()), + } + } + } + + impl RateLimiter for ScriptedLimiter { + fn exceeded(&self, key: &str, _limit: u32) -> Result> { + self.keys.borrow_mut().push(key.to_owned()); + Ok(self.exceeded) + } + } + + struct Fixture { + settings: Settings, + runtime: IdentityRuntime, + client: Arc, + services: RuntimeServices, + graph: KvIdentityGraph, + store: Arc, + context: EcContext, + } + + impl Fixture { + /// A returning writer whose phase 1 read found `row`. With + /// `pull_partner`, one generic pull partner is configured. + fn new(row: Option<&KvEntry>, pull_partner: bool) -> Self { + Self::with_settings(row, Self::settings(pull_partner)) + } + + fn settings(pull_partner: bool) -> Settings { + let mut settings = create_test_settings(); + settings.ec.partners = if pull_partner { + vec![ec_partner( + PULL_SOURCE, + &format!("https://{PULL_HOST}/pull"), + "example-token-value-0123456789abcdef", + )] + } else { + Vec::new() + }; + settings + } + + fn with_settings(row: Option<&KvEntry>, settings: Settings) -> Self { + let runtime = IdentityRuntime::from_settings(&settings).expect("should build runtime"); + let client = Arc::new(FakeHttpClient::new()); + let backend = Arc::new(FakeBackend::new()); + let services = fake_services(&client, &backend); + let (graph, store) = KvIdentityGraph::write_counting("flush-store"); + let mut context = EcContext::new_for_test_with_ip( + Some(TEST_EC_ID.to_owned()), + allowing_consent(), + Some("192.0.2.10".to_owned()), + ); + context.identity_mut().set_writer_decision(WriterDecision { + writer: true, + is_real_browser: true, + user_agent: Some("ExampleBrowser/1.0".to_owned()), + dnt: false, + page_url: None, + }); + if let Some(row) = row { + graph.create(TEST_EC_ID, row).expect("should seed the row"); + context.set_kv_snapshot(graph.load_snapshot(TEST_EC_ID)); + } + store.reset_counts(); + Self { + settings, + runtime, + client, + services, + graph, + store, + context, + } + } + + fn run(&mut self, limiter: &ScriptedLimiter) -> FlushOutcome { + self.run_with_status(limiter, StatusCode::OK) + } + + fn run_with_status( + &mut self, + limiter: &ScriptedLimiter, + status: StatusCode, + ) -> FlushOutcome { + run_after_send(AfterSendInput { + runtime: &self.runtime, + settings: &self.settings, + services: &self.services, + graph: &self.graph, + context: &mut self.context, + status, + rate_limiter: limiter, + }) + } + + fn stage(&mut self, change: PartnerIdChange) { + self.context + .identity_mut() + .mutation_mut() + .add_change(change); + } + + fn stage_sharedid(&mut self) { + self.stage(PartnerIdChange::FillMissing { + source: source("sharedid.org"), + record: KvPartnerId::browser("shared-uid", Some(current_timestamp())), + }); + } + + fn row(&self) -> KvEntry { + self.graph + .get(TEST_EC_ID) + .expect("should read the row") + .expect("should hold the row") + .0 + } + } + + /// A live, consenting row; `cluster_size` is set unless `sized` is false. + fn live_row(sized: bool) -> KvEntry { + let mut entry = KvEntry::new(&allowing_consent(), None, 1_000, "test-publisher.com"); + if sized { + entry.network = Some(KvNetwork { + cluster_size: Some(1), + }); + } + entry + } + + fn pull_answers(fixture: &Fixture, uid: &str) { + fixture + .client + .respond(PULL_HOST, 200, format!(r#"{{"uid":"{uid}"}}"#).as_bytes()); + } + + #[test] + fn writer_flushes_mutation_once_with_generation_match() { + let mut fixture = Fixture::new(Some(&live_row(true)), false); + fixture.stage_sharedid(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the mutation"); + let writes = fixture.store.writes(); + assert_eq!( + writes.if_generation_match, 1, + "should write once with a CAS" + ); + assert_eq!(writes.total(), 1, "should make no other write"); + assert_eq!(writes.lookups, 0, "should reuse the phase 1 read"); + assert!( + fixture.row().ids.contains_key("sharedid.org"), + "should store the sharedId fill" + ); + } + + #[test] + fn writer_with_nothing_to_do_reads_and_writes_nothing() { + let mut fixture = Fixture::new(Some(&live_row(true)), false); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Skipped, "should have nothing to do"); + let writes = fixture.store.writes(); + assert_eq!( + (writes.lookups, writes.total()), + (0, 0), + "should touch no KV" + ); + } + + #[test] + fn non_2xx_or_304_status_drops_ingest_and_dispatches_nothing() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + pull_answers(&fixture, "pulled-uid"); + fixture.stage_sharedid(); + + let outcome = fixture.run_with_status( + &ScriptedLimiter::allowing(), + StatusCode::INTERNAL_SERVER_ERROR, + ); + + assert_eq!(outcome, FlushOutcome::Skipped, "should drop the ingest"); + assert!( + fixture.client.sent_hosts().is_empty(), + "should call no provider" + ); + let writes = fixture.store.writes(); + assert_eq!( + (writes.lookups, writes.total()), + (0, 0), + "should touch no KV" + ); + } + + #[test] + fn not_modified_status_still_flushes() { + let mut fixture = Fixture::new(Some(&live_row(true)), false); + fixture.stage_sharedid(); + + let outcome = + fixture.run_with_status(&ScriptedLimiter::allowing(), StatusCode::NOT_MODIFIED); + + assert_eq!( + outcome, + FlushOutcome::Written, + "should treat 304 as success" + ); + } + + #[test] + fn tombstone_on_reread_drops_mutation_and_calls_no_provider() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + pull_answers(&fixture, "leaked-uid"); + fixture.stage_sharedid(); + let tombstoned = fixture + .graph + .write_withdrawal_tombstone(TEST_EC_ID, drop) + .expect("should tombstone the row"); + assert_eq!( + tombstoned, + crate::ec::kv::TombstoneOutcome::Written, + "should tombstone the live row" + ); + fixture.store.reset_counts(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!( + outcome, + FlushOutcome::Dropped("tombstone"), + "should drop the mutation on a tombstone" + ); + assert!( + fixture.client.sent_hosts().is_empty(), + "should disclose the EC ID to no provider" + ); + assert_eq!(fixture.store.writes().total(), 0, "should write nothing"); + assert!( + fixture.row().ids.is_empty(), + "should leave the tombstone bare" + ); + } + + #[test] + fn failed_phase_one_read_rereads_before_writing() { + let mut fixture = Fixture::new(Some(&live_row(true)), false); + fixture.context.set_kv_snapshot(EcKvSnapshot::Failed { + ec_id: TEST_EC_ID.to_owned(), + }); + fixture.stage_sharedid(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!( + outcome, + FlushOutcome::Written, + "should write after the re-read" + ); + let writes = fixture.store.writes(); + assert_eq!(writes.lookups, 1, "should re-read once"); + assert_eq!(writes.if_generation_match, 1, "should write once"); + } + + #[test] + fn ingest_queued_against_an_unread_snapshot_is_written_after_a_read() { + let mut fixture = Fixture::new(Some(&live_row(true)), false); + fixture.context.set_kv_snapshot(EcKvSnapshot::NotRead); + fixture.stage_sharedid(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the ingest"); + assert_eq!( + fixture.store.writes().lookups, + 1, + "should read before writing" + ); + assert!( + fixture.row().ids.contains_key("sharedid.org"), + "should store the sharedId the writer queued" + ); + } + + #[test] + fn due_generic_pull_provider_dispatches_after_reread_and_its_uid_lands_in_the_same_write() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + pull_answers(&fixture, "pulled-uid"); + fixture.stage_sharedid(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write once"); + let writes = fixture.store.writes(); + assert_eq!(writes.lookups, 1, "should re-read before dispatch"); + assert_eq!(writes.total(), 1, "should make one write"); + assert_eq!( + fixture.client.sent_hosts(), + [PULL_HOST], + "should call the partner" + ); + let row = fixture.row(); + let record = row + .ids + .get(PULL_SOURCE) + .expect("should store the pulled UID"); + assert_eq!(record.eids.len(), 1, "should store a one-entry record"); + assert_eq!( + record.first_uid(), + Some("pulled-uid"), + "should store the UID" + ); + assert_eq!( + record.w.as_deref(), + Some(PULL_ID), + "should name the provider" + ); + assert!( + row.ids.contains_key("sharedid.org"), + "should land the ingest too" + ); + } + + #[test] + fn generic_pull_results_join_the_single_flush() { + let mut settings = Fixture::settings(true); + settings.ec.partners.push(ec_partner( + "dsp.example.com", + "https://sync.dsp.example.com/pull", + "example-token-value-0123456789abcdef", + )); + settings.ec.identity.max_concurrency = 1; + let mut fixture = Fixture::with_settings(Some(&live_row(true)), settings); + pull_answers(&fixture, "pulled-uid"); + fixture + .client + .respond("sync.dsp.example.com", 200, br#"{"uid":"other-uid"}"#); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write once"); + assert_eq!(fixture.store.writes().total(), 1, "should make one write"); + let row = fixture.row(); + assert!( + row.ids.contains_key(PULL_SOURCE) && row.ids.contains_key("dsp.example.com"), + "should land both partners' UIDs in the one write" + ); + } + + #[test] + fn hold_provider_on_a_navigation_without_an_auction_runs_in_phase_four() { + let mut settings = Fixture::settings(true); + settings.ec.identity.providers.insert( + PULL_ID.to_owned(), + ProviderSettings { + mode: ProviderMode::Hold, + ..ProviderSettings::default() + }, + ); + let mut fixture = Fixture::with_settings(Some(&live_row(true)), settings); + pull_answers(&fixture, "pulled-uid"); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the result"); + assert_eq!( + fixture.client.sent_hosts(), + [PULL_HOST], + "should call the provider" + ); + } + + #[test] + fn cas_conflict_is_not_retried_as_a_whole() { + let mut fixture = Fixture::new(Some(&live_row(true)), false); + fixture.stage_sharedid(); + fixture.store.refuse_conditional_writes(TEST_EC_ID, 1); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!( + outcome, + FlushOutcome::WriteFailed { + removals: RemovalWriteOutcome::NothingToWrite + }, + "should fail without a removal retry" + ); + assert_eq!( + fixture.store.writes().if_generation_match, + 1, + "should attempt the flush once" + ); + } + + /// A row holding `PULL_SOURCE`, and a mutation with a fill, a removal of + /// that record and an opt-out of the pull provider. + fn removal_fixture() -> (Fixture, u64) { + let mut row = live_row(true); + let record = record_for_test("old-uid", Some(PULL_ID), 900); + let expected = record_revision(&record); + row.ids.insert(PULL_SOURCE.to_owned(), record); + let mut fixture = Fixture::new(Some(&row), true); + fixture.stage_sharedid(); + fixture.stage(PartnerIdChange::OwnerRemove { + source: source(PULL_SOURCE), + provider: pid(PULL_ID), + expected, + }); + let at = current_timestamp(); + fixture.context.identity_mut().mutation_mut().merge_state( + pid(PULL_ID), + ProviderStateChange { + opt_out: Some(KvOptOut { + at, + fp: "0".to_owned(), + }), + ..ProviderStateChange::default() + }, + ); + (fixture, at) + } + + #[test] + fn failed_flush_writes_its_removals_alone() { + let (mut fixture, at) = removal_fixture(); + fixture.store.refuse_conditional_writes(TEST_EC_ID, 1); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!( + outcome, + FlushOutcome::WriteFailed { + removals: RemovalWriteOutcome::Written + }, + "should write the removals alone" + ); + assert_eq!( + fixture.store.writes().if_generation_match, + 2, + "should make one flush attempt and one removal write" + ); + let row = fixture.row(); + assert!( + !row.ids.contains_key(PULL_SOURCE), + "should apply the removal" + ); + assert!( + !row.ids.contains_key("sharedid.org"), + "should drop the fill" + ); + assert_eq!( + row.ps + .get(PULL_ID) + .and_then(|state| state.o.as_ref()) + .map(|o| o.at), + Some(at), + "should store the opt-out" + ); + } + + #[test] + fn failed_removal_retry_counts_removal_lost() { + let (mut fixture, _) = removal_fixture(); + fixture + .store + .refuse_conditional_writes(TEST_EC_ID, usize::MAX); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!( + outcome, + FlushOutcome::WriteFailed { + removals: RemovalWriteOutcome::Failed + }, + "should lose the removals" + ); + assert_eq!( + fixture.store.writes().if_generation_match, + 1 + 5, + "should make one flush attempt and five removal attempts" + ); + assert_eq!( + fixture.context.identity().removal_lost(), + 2, + "should count the removal and the opt-out" + ); + } + + #[test] + fn burst_capped_provider_gets_sixty_second_backoff_in_the_write() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + pull_answers(&fixture, "pulled-uid"); + let before = current_timestamp(); + + let outcome = fixture.run(&ScriptedLimiter::capped()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the backoff"); + assert!( + fixture.client.sent_hosts().is_empty(), + "should not call the provider" + ); + let next = fixture + .row() + .ps + .get(PULL_ID) + .and_then(|state| state.n) + .expect("should store n"); + assert!( + (before + 60..=current_timestamp() + 60).contains(&next), + "should wait one 60-second window" + ); + } + + #[test] + fn missing_cluster_size_is_added_to_the_one_write() { + let mut fixture = Fixture::new(Some(&live_row(false)), false); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the size"); + assert_eq!(fixture.store.writes().total(), 1, "should write once"); + assert_eq!( + fixture + .row() + .network + .and_then(|network| network.cluster_size), + Some(1), + "should store the cluster size" + ); + } + + #[test] + fn untouched_entry_set_is_preserved_by_the_flush() { + let mut row = live_row(true); + let multi = full_record(Some("other"), example_entry_set()); + row.ids.insert("multi.example.com".to_owned(), multi); + let mut fixture = Fixture::new(Some(&row), false); + let live = fixture + .graph + .get(TEST_EC_ID) + .expect("should read") + .expect("should hold the row") + .0; + fixture.store.reset_counts(); + fixture.stage_sharedid(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the fill"); + assert_eq!( + fixture.row().ids.get("multi.example.com"), + live.ids.get("multi.example.com"), + "should keep the untouched record field for field" + ); + } + + #[test] + fn generic_pull_runs_only_on_returning_writer() { + let mut stager = Fixture::new(Some(&live_row(true)), true); + pull_answers(&stager, "pulled-uid"); + stager + .context + .identity_mut() + .set_writer_decision(WriterDecision { + writer: false, + is_real_browser: true, + user_agent: None, + dnt: false, + page_url: None, + }); + + assert_eq!( + stager.run(&ScriptedLimiter::allowing()), + FlushOutcome::Skipped, + "should skip a stager" + ); + assert!( + stager.client.sent_hosts().is_empty(), + "should call no partner" + ); + } + + #[test] + fn new_visitor_reaches_pull_partners_from_second_page_load() { + let mut first = Fixture::new(None, true); + pull_answers(&first, "pulled-uid"); + let snapshot = EcKvSnapshot::Present { + ec_id: TEST_EC_ID.to_owned(), + entry: Box::new(live_row(true)), + generation: None, + }; + first + .context + .replace_with_generated(TEST_EC_ID.to_owned(), snapshot); + + assert_eq!( + first.run(&ScriptedLimiter::allowing()), + FlushOutcome::Skipped, + "should run no provider on the first navigation" + ); + assert!( + first.client.sent_hosts().is_empty(), + "should call no partner" + ); + + let mut second = Fixture::new(Some(&live_row(true)), true); + pull_answers(&second, "pulled-uid"); + assert_eq!( + second.run(&ScriptedLimiter::allowing()), + FlushOutcome::Written, + "should pull on the second navigation" + ); + assert_eq!( + second.client.sent_hosts(), + [PULL_HOST], + "should call the partner" + ); + } + + #[test] + fn failing_pull_partner_backs_off_through_ps() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + fixture.client.respond(PULL_HOST, 503, b""); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Written, "should write the backoff"); + let row = fixture.row(); + let state = row.ps.get(PULL_ID).expect("should store provider state"); + assert_eq!(state.f, 1, "should count the failure"); + assert!( + state.n.is_some_and(|next| next > current_timestamp()), + "should push n into the future" + ); + + let mut again = Fixture::new(Some(&row), true); + again.client.respond(PULL_HOST, 503, b""); + assert_eq!( + again.run(&ScriptedLimiter::allowing()), + FlushOutcome::Skipped, + "should not retry the partner before n" + ); + assert!( + again.client.sent_hosts().is_empty(), + "should call no partner" + ); + } + + #[test] + fn rate_limit_key_is_per_visitor_not_per_ip() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + pull_answers(&fixture, "pulled-uid"); + let limiter = ScriptedLimiter::allowing(); + + let _ = fixture.run(&limiter); + + let expected = format!( + "identity:{PULL_ID}:{}", + visitor_key(fixture.runtime.keys(), TEST_EC_ID) + ); + assert_eq!(*limiter.keys.borrow(), [expected], "should key by visitor"); + } + + #[test] + fn eid_persistence_denied_dispatches_no_pull_partner() { + let mut fixture = Fixture::new(Some(&live_row(true)), true); + pull_answers(&fixture, "pulled-uid"); + *fixture.context.consent_mut() = crate::test_support::tests::purpose_one_only_consent(); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Skipped, "should have nothing to do"); + assert!( + fixture.client.sent_hosts().is_empty(), + "should call no partner" + ); + } + + #[test] + fn pull_partner_with_a_stored_uid_is_not_called() { + let mut row = live_row(true); + row.ids + .insert(PULL_SOURCE.to_owned(), KvPartnerId::new("already-known")); + let mut fixture = Fixture::new(Some(&row), true); + pull_answers(&fixture, "pulled-uid"); + + let outcome = fixture.run(&ScriptedLimiter::allowing()); + + assert_eq!(outcome, FlushOutcome::Skipped, "should have nothing to do"); + assert!( + fixture.client.sent_hosts().is_empty(), + "should call no partner" + ); + assert_eq!(fixture.store.writes().lookups, 0, "should not re-read"); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 0bb5614cf9..00076b73d1 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod coordinator; pub(crate) mod denials; pub mod descriptor; pub mod exchange; +pub mod flush; pub(crate) mod generic_pull; pub mod ids; pub mod input; diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs index 760b465e06..2fdc9444ae 100644 --- a/crates/trusted-server-core/src/ec/identity/mutation.rs +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -288,6 +288,22 @@ impl IdentityMutation { copy } + /// Cuts the mutation down for a navigation whose origin status was not + /// 2xx or 304 (spec 5.1). + /// + /// Keeps every removal, opt-out and clear, the stand-alone state changes + /// (the `n` and `f` of every outcome other than `found`) and the cluster + /// size. Drops browser ingest and every other positive change, and each + /// provider result whole, keeping only its opt-out and clear. + pub(crate) fn retain_for_error_status(&mut self) { + self.partner_ids.retain(PartnerIdChange::is_removal); + for result in core::mem::take(&mut self.results) { + if result.state.has_removal() { + self.merge_state(result.provider, result.state.removals()); + } + } + } + /// Every provider state change that applies regardless of its result: /// the standalone ones, and each result's. // Temporary: first used by a later identity task. @@ -915,6 +931,38 @@ mod tests { } } + #[test] + fn retain_for_error_status_keeps_removals_and_backoff() { + let removal = PartnerIdChange::OwnerRemove { + source: src("a.example.com"), + provider: pid("owner"), + expected: Revision::Unset, + }; + let backoff = ProviderStateChange { + n: Some(NOW + 86_400), + ..ProviderStateChange::default() + }; + let mut mutation = IdentityMutation::default(); + mutation.add_change(fill("b.example.com", KvPartnerId::browser("u", Some(NOW)))); + mutation.add_change(PartnerIdChange::OwnerWrite { + source: src("a.example.com"), + record: written("owner", "found-uid", "1111111111111111"), + expected: None, + }); + mutation.add_change(removal.clone()); + mutation.merge_state(pid("other"), backoff.clone()); + + mutation.retain_for_error_status(); + + let mut expected = IdentityMutation::default(); + expected.add_change(removal); + expected.merge_state(pid("other"), backoff); + assert_eq!( + mutation, expected, + "should keep only the removal and the no_match backoff" + ); + } + #[test] fn tombstone_drops_everything() { let mut mutation = IdentityMutation::default(); diff --git a/crates/trusted-server-core/src/ec/identity/removal.rs b/crates/trusted-server-core/src/ec/identity/removal.rs index 4bdede3d78..9868aa2e60 100644 --- a/crates/trusted-server-core/src/ec/identity/removal.rs +++ b/crates/trusted-server-core/src/ec/identity/removal.rs @@ -1,10 +1,6 @@ //! The shared removal write: a removal-only mutation written with the //! withdrawal path's CAS loop (spec 5.2 phase 4, 5.3). -// Temporary: the capture removal write and the flush retry first use this in -// later identity tasks. Remove this when they land. -#![allow(dead_code)] - use super::ids::SourceDomain; use super::mutation::{ApplyContext, IdentityMutation, apply_removals_only}; use crate::ec::kv::{KvIdentityGraph, MAX_CAS_RETRIES}; diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 4eb82d1c5d..80d80330c6 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,7 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; -use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; +use super::kv_types::{KvEntry, KvMetadata}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -536,6 +536,9 @@ impl KvIdentityGraph { /// later chance to retry. /// /// [`generate_if_needed`]: super::generate_if_needed + // No production caller since phase 4 replaced pull sync's own write; its + // tests still pin the merge rules. Remove with them in a cleanup task. + #[allow(dead_code)] pub(crate) fn upsert_partner_ids_from_snapshot( &self, ec_id: &str, @@ -662,9 +665,6 @@ impl KvIdentityGraph { /// /// Returns an error when `entry` does not serialize or validate, or the /// store write fails for a reason other than a generation mismatch. - // Temporary: the removal write and the flush are its first callers, in - // later identity tasks. Remove this when they land. - #[allow(dead_code)] pub(crate) fn write_if_generation( &self, ec_id: &str, @@ -690,6 +690,9 @@ impl KvIdentityGraph { /// update is logged. Write failures are *not* routed here: they are real /// operation failures and stay [`EcKvSnapshot::Failed`] so callers can /// report them. + // No production caller since phase 4 replaced pull sync's own write; its + // tests still pin the merge rules. Remove with them in a cleanup task. + #[allow(dead_code)] fn keep_proven( ec_id: &str, downgraded: EcKvSnapshot, @@ -1320,85 +1323,6 @@ impl KvIdentityGraph { self.count_hash_prefix_keys(ec_hash(ec_id)) } - /// Evaluates the cluster size for an EC entry. - /// - /// Returns the stored `cluster_size` when it has already been evaluated - /// for a live entry. Tombstone entries return `None` without store I/O so - /// their 24-hour withdrawal TTL is not extended. Otherwise, counts the - /// number of keys sharing the same hash prefix via - /// [`count_hash_prefix_keys`](Self::count_hash_prefix_keys) and writes the - /// result back to the entry. The CAS write is best-effort — on conflict - /// or write failure, the computed value is still returned. - /// - /// # Errors - /// - /// Returns [`TrustedServerError::KvStore`] on store or list failure. - pub fn evaluate_cluster( - &self, - ec_id: &str, - entry: &KvEntry, - generation: u64, - ) -> Result, Report> { - if !entry.consent.ok { - log::trace!("evaluate_cluster: skipping tombstone entry"); - return Ok(None); - } - - if let Some(cluster_size) = entry - .network - .as_ref() - .and_then(|network| network.cluster_size) - { - log::trace!("evaluate_cluster: using stored cluster_size"); - return Ok(Some(cluster_size)); - } - - // Compute cluster size via prefix list. - let cluster_size = self.compute_cluster_size(ec_id)?; - - log::debug!( - "evaluate_cluster: computed cluster_size={cluster_size} for '{}'", - log_id(ec_id) - ); - - // Best-effort CAS write-back — update only the cluster size so any - // future `network` fields are preserved across this lazy write. - let mut updated_entry = entry.clone(); - let mut network = updated_entry - .network - .unwrap_or(KvNetwork { cluster_size: None }); - network.cluster_size = Some(cluster_size); - updated_entry.network = Some(network); - - let (body, meta_str) = Self::serialize_entry(&updated_entry, self.store_name())?; - - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => {} - Ok(EcKvWriteOutcome::PreconditionFailed) => { - log::debug!( - "evaluate_cluster: CAS conflict writing cluster_size for '{}', \ - returning computed value anyway", - log_id(ec_id), - ); - } - Err(report) => { - // Log but don't fail — the computed value is still valid. - log::warn!( - "evaluate_cluster: failed to write cluster_size for '{}': {report}", - log_id(ec_id) - ); - } - } - - Ok(Some(cluster_size)) - } - /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). Consent @@ -2025,42 +1949,6 @@ mod tests { assert_eq!(entry.ids["ssp_x"].first_uid(), Some("original")); } - #[test] - fn evaluate_cluster_returns_stored_value_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut entry = live_entry(); - entry.network = Some(KvNetwork { - cluster_size: Some(5), - }); - - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store when cluster_size is already known"); - - assert_eq!( - cluster_size, - Some(5), - "should return stored cluster_size without re-listing keys" - ); - } - - #[test] - fn evaluate_cluster_skips_tombstone_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = KvEntry::tombstone(1000); - - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store for tombstone entries"); - - assert_eq!( - cluster_size, None, - "should not evaluate or write cluster_size for tombstones" - ); - } - #[test] fn create_then_get_roundtrips_entry() { let kv = KvIdentityGraph::in_memory("test_store"); diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index 79d581e140..46d589cd45 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -70,7 +70,10 @@ pub enum EcKvWriteOutcome { /// Infrastructure failures are reported as [`TrustedServerError::KvStore`]; /// write precondition failures are part of the normal control flow and are /// returned as [`EcKvWriteOutcome::PreconditionFailed`] instead of errors. -pub trait EcKvStore { +/// +/// `Send + Sync` so a [`super::kv::KvIdentityGraph`] can be built once and +/// held in shared application state. +pub trait EcKvStore: Send + Sync { /// Returns the platform store name, used in log and error messages. fn store_name(&self) -> &str; diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 7ef46fb0db..71330910c0 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -48,7 +48,6 @@ pub mod kv_types; pub mod partner; pub mod prebid_eids; pub mod pull_sync; -pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; @@ -78,7 +77,7 @@ use http::{Request, header}; use url::Url; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; -use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE}; +use crate::constants::COOKIE_TS_EC; use crate::cookies::handle_request_cookies; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; @@ -93,7 +92,6 @@ use self::identity::request::{RequestIdentity, WriterDecision}; use self::identity::runtime::IdentityRuntime; use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; use self::kv_types::KvEntry; -use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. /// @@ -190,8 +188,6 @@ pub use generation::{ struct RequestEc { /// EC ID from the `ts-ec` cookie, if present. cookie_ec: Option, - /// Pull-sync completeness marker, if present. - pull_sync_marker: Option, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -208,17 +204,7 @@ fn parse_ec_from_request(req: &Request) -> Result Option { @@ -280,8 +266,6 @@ pub struct EcContext { kv_snapshot: EcKvSnapshot, /// Whether this request may rotate an orphaned EC identity. recovery_eligible: bool, - /// Browser-carried proof of recent pull-partner completeness. - pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Identity state this request decides and collects. @@ -368,7 +352,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, identity: RequestIdentity::default(), failed_withdrawals: Vec::new(), @@ -464,7 +447,6 @@ impl EcContext { self.ec_value = Some(ec_id); self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); return Ok(()); } @@ -676,43 +658,11 @@ impl EcContext { self.recovery_eligible } - /// Validates a browser completeness marker against the active EC and partner set. - pub(crate) fn validate_pull_sync_marker( - &mut self, - settings: &Settings, - registry: ®istry::PartnerRegistry, - ) { - validate_marker_state( - &mut self.pull_sync_marker, - settings, - registry, - self.ec_value.as_deref(), - ); - } - - /// Returns the current pull-sync marker state. - #[must_use] - pub(crate) fn pull_sync_marker(&self) -> &PullSyncMarkerState { - &self.pull_sync_marker - } - - /// Returns mutable pull-sync marker state for response reconciliation. - pub(crate) fn pull_sync_marker_mut(&mut self) -> &mut PullSyncMarkerState { - &mut self.pull_sync_marker - } - - /// Sets pull-sync marker state in focused unit tests. - #[cfg(test)] - pub(crate) fn set_pull_sync_marker_for_test(&mut self, state: PullSyncMarkerState) { - self.pull_sync_marker = state; - } - /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); self.ec_generated = true; self.kv_snapshot = snapshot; - self.pull_sync_marker.invalidate_for_replaced_ec(); } /// Returns whether EC creation is permitted by consent for this request. @@ -762,13 +712,32 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), failed_withdrawals: Vec::new(), } } + /// Creates a test-only [`EcContext`] for a real browser's writer + /// navigation, with explicit client IP. + #[cfg(test)] + #[must_use] + pub fn new_writer_for_test_with_ip( + ec_value: Option, + consent: ConsentContext, + client_ip: Option, + ) -> Self { + let mut context = Self::new_for_test_with_ip(ec_value, consent, client_ip); + context.identity.set_writer_decision(WriterDecision { + writer: true, + is_real_browser: true, + user_agent: None, + dnt: false, + page_url: None, + }); + context + } + /// Creates a test-only [`EcContext`] with explicit client IP. #[cfg(test)] #[must_use] @@ -788,7 +757,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), failed_withdrawals: Vec::new(), @@ -817,7 +785,6 @@ impl EcContext { device_signals: None, kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, - pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, identity: RequestIdentity::default(), failed_withdrawals: Vec::new(), diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 3a0940a672..b0b2100643 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -1,321 +1,11 @@ -//! Pull sync background dispatch. +//! Pull sync URL helpers. //! -//! Launches partner pull-sync requests for organic traffic after the client -//! response has been sent. Dispatch is best-effort and never affects client -//! response status. -//! -//! Pull sync currently fills missing partner UIDs only. Once a partner UID is -//! present in the EC identity entry, it is not periodically refreshed because -//! the entry no longer stores per-partner sync timestamps. +//! Pull partners are called as generic pull identity providers +//! (`identity::generic_pull`), dispatched by the writer's phase 4 +//! flush. These helpers validate and build the partner URL they call. -use edgezero_core::body::Body as EdgeBody; -use http::{Method, StatusCode}; -use serde::Deserialize; use url::Url; -use crate::consent::allows_eid_persistence; -use crate::platform::{ - DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, RuntimeServices, -}; -use crate::settings::Settings; - -use super::generation::{ec_hash, is_valid_ec_id}; -use super::identity::coordinator::{Waited, buffered_body, drain_in_flight}; -use super::identity::generic_pull::generic_pull_id; -use super::kv::{KvIdentityGraph, PartnerIdUpdate}; -use super::kv_types::KvEntry; -use super::rate_limiter::RateLimiter; -use super::registry::{PartnerConfig, PartnerRegistry}; - -// `current_timestamp` is defined in the parent `ec` module. -use super::EcContext; -use super::EcKvSnapshot; -use super::current_timestamp; - -/// Inputs needed to dispatch pull sync after response flush. -#[derive(Debug, Clone)] -pub struct PullSyncContext { - ec_id: String, - snapshot: EcKvSnapshot, -} - -impl PullSyncContext { - /// Returns the EC ID for the request. - #[must_use] - pub fn ec_id(&self) -> &str { - &self.ec_id - } -} - -#[derive(Debug, Deserialize)] -struct PullSyncResponse { - uid: Option, -} - -/// Builds post-send pull-sync context from the route EC context. -/// -/// Returns `None` when there are no pull-enabled partners, when consent denies -/// EC, when there is no valid active EC ID, or when the request snapshot holds -/// no consented row that is still missing at least one pull-partner UID. -#[must_use] -pub fn build_pull_sync_context( - ec_context: &EcContext, - registry: &PartnerRegistry, -) -> Option { - // Without EID-persistence consent (TCF Purpose 1 + 4), finalization - // withholds browser-supplied partner UIDs. That leaves partner slots empty, - // which pull sync would otherwise read as eligible and disclose the EC ID - // to partners for exactly the users who opted out. - if registry.pull_enabled_partners().is_empty() - || !ec_context.ec_allowed() - || !allows_eid_persistence(ec_context.consent()) - { - return None; - } - - let ec_id_ref = ec_context.ec_value()?; - if !is_valid_ec_id(ec_id_ref) { - log::debug!("Pull sync: skipping dispatch because active EC ID is invalid format"); - return None; - } - - let entry = ec_context.kv_snapshot().entry_for(ec_id_ref)?; - if !entry.consent.ok || entry_is_pull_complete(entry, registry) { - return None; - } - - let ec_id = ec_id_ref.to_owned(); - let snapshot = ec_context.kv_snapshot().clone(); - Some(PullSyncContext { ec_id, snapshot }) -} - -/// Dispatches partner pull-sync requests in the background. -/// -/// This function is best-effort: all errors are logged and swallowed. -/// -/// # Consent revalidation -/// -/// Pull sync runs *after* the page response has been flushed to the browser and -/// discloses the raw `ec_id` to partners, so it cannot be authorized by the -/// request snapshot alone: a concurrent CMP withdrawal can tombstone the row in -/// the window between snapshot capture and dispatch. The live row is re-read -/// immediately before the first partner request and anything short of a live, -/// consenting entry cancels the dispatch. The read is deferred until after the -/// snapshot-based eligibility filter, so a request with nothing to pull still -/// costs no KV operation, and the fresh snapshot is reused for the write-back -/// instead of the stale one. -/// -/// # Panics -/// -/// Panics if the HTTP request builder produces an invalid request, which -/// cannot happen with the hardcoded method and well-formed URI used here. -pub fn dispatch_pull_sync( - settings: &Settings, - kv: &KvIdentityGraph, - registry: &PartnerRegistry, - rate_limiter: &dyn RateLimiter, - context: &PullSyncContext, - services: &RuntimeServices, -) { - let now = current_timestamp(); - let mut pull_partners = registry.pull_enabled_partners(); - - if pull_partners.is_empty() { - return; - } - - let Some(request_entry) = context.snapshot.entry_for(context.ec_id()) else { - log::debug!( - "Pull sync: skipping dispatch for '{}' because the request captured no usable \ - identity snapshot", - super::log_id(context.ec_id()) - ); - return; - }; - if !request_entry.consent.ok { - return; - } - - // Sort by source domain for deterministic ordering, then apply a rotating - // hourly offset so that different partners get dispatch priority (§10.3). - pull_partners.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - - log::debug!( - "Pull sync: {} pull-enabled partners after filtering", - pull_partners.len(), - ); - - // Rotate the partner list so that the starting partner changes each - // hour. This ensures fair distribution when max_concurrency limits - // how many partners are dispatched per request. - let offset = (now / 3600) as usize % pull_partners.len(); - pull_partners.rotate_left(offset); - - // Drop partners the request snapshot already has a UID for before paying - // for the revalidation read: a dispatch with nothing to pull must not add a - // KV operation. - pull_partners.retain(|partner| is_partner_pull_eligible(partner, Some(request_entry))); - if pull_partners.is_empty() { - return; - } - - let live_snapshot = kv.load_snapshot(context.ec_id()); - let Some(live_entry) = live_snapshot.entry_for(context.ec_id()) else { - log::warn!( - "Pull sync: skipping dispatch for '{}' because the live identity row could not be \ - confirmed", - super::log_id(context.ec_id()) - ); - return; - }; - if !live_entry.consent.ok { - log::info!( - "Pull sync: skipping dispatch for '{}' because consent was withdrawn after the \ - request snapshot was captured", - super::log_id(context.ec_id()) - ); - return; - } - - let max_concurrency = settings.ec.identity.max_concurrency.max(1); - let mut in_flight: Vec<(String, PlatformPendingRequest)> = Vec::new(); - let mut updates = Vec::new(); - - for partner in pull_partners { - // Re-checked against the live row: a concurrent request may have filled - // this partner's UID since the request snapshot was captured. - if !is_partner_pull_eligible(partner, Some(live_entry)) { - continue; - } - - let Some(url) = validated_pull_sync_url( - partner.pull_sync_url.as_deref(), - &partner.pull_sync_allowed_domains, - ) else { - continue; - }; - - let rate_key = pull_rate_limit_key(&partner.source_domain, context.ec_id()); - let provider_settings = settings - .ec - .identity - .provider(&generic_pull_id(&partner.source_domain)); - match rate_limiter.exceeded_per_minute(&rate_key, provider_settings.max_calls_per_minute) { - Ok(true) => { - log::debug!( - "Pull sync: rate-limited partner '{}' for ec_id '{}'", - partner.source_domain, - super::log_id(context.ec_id()) - ); - continue; - } - Ok(false) => {} - Err(err) => { - log::warn!( - "Pull sync: failed to read rate limit for partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - } - - let Some(token) = partner.ts_pull_token.as_ref() else { - log::warn!( - "Pull sync: partner '{}' enabled but missing ts_pull_token", - partner.source_domain - ); - continue; - }; - - let request_url = build_pull_request_url(url, context.ec_id()); - let scheme = request_url.scheme().to_string(); - let host = request_url.host_str().unwrap_or_default().to_string(); - let port = request_url.port(); - - let backend_name = match services.backend().ensure(&PlatformBackendSpec { - scheme, - host, - port, - host_header_override: None, - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) { - Ok(name) => name, - Err(err) => { - log::warn!( - "Pull sync: failed to resolve backend for partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - let request = http::Request::builder() - .method(Method::GET) - .uri(request_url.as_str()) - .header("authorization", format!("Bearer {}", token.expose())) - .body(EdgeBody::empty()) - .expect("should build pull sync request"); - - let pending = match futures::executor::block_on( - services - .http_client() - .send_async(PlatformHttpRequest::new(request, backend_name)), - ) { - Ok(pending) => pending, - Err(err) => { - log::warn!( - "Pull sync: failed to dispatch partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - in_flight.push((partner.source_domain.clone(), pending)); - - if in_flight.len() >= max_concurrency { - drain_pull_batch(&mut in_flight, services, &mut updates); - } - } - - drain_pull_batch(&mut in_flight, services, &mut updates); - if !updates.is_empty() { - // Write back from the revalidated snapshot, not the request one: its - // generation is current, so the first CAS attempt is not spent losing a - // conflict against the read that authorized this dispatch. - let outcome = - kv.upsert_partner_ids_from_snapshot(context.ec_id(), &updates, live_snapshot.clone()); - if matches!(outcome, EcKvSnapshot::Failed { .. }) { - log::warn!( - "Pull sync: failed to persist partner updates for '{}'", - super::log_id(context.ec_id()) - ); - } - } -} - -/// Returns whether a consented live entry holds an ID for every pull-enabled -/// partner. Always returns `false` when there are no pull-enabled partners. -#[must_use] -pub(crate) fn entry_is_pull_complete(entry: &KvEntry, registry: &PartnerRegistry) -> bool { - let pull_partners = registry.pull_enabled_partners(); - !pull_partners.is_empty() - && entry.consent.ok - && pull_partners - .iter() - .all(|partner| !is_partner_pull_eligible(partner, Some(entry))) -} - -fn is_partner_pull_eligible(partner: &PartnerConfig, kv_entry: Option<&KvEntry>) -> bool { - kv_entry - .and_then(|entry| entry.ids.get(&partner.source_domain)) - .is_none() -} - /// Returns the pull URL when it is HTTPS and its host is allowlisted. /// /// The host comparison ignores case and a trailing dot on both sides. @@ -367,125 +57,11 @@ pub(crate) fn build_pull_request_url(mut base_url: Url, ec_id: &str) -> Url { base_url } -fn pull_rate_limit_key(source_domain: &str, ec_id: &str) -> String { - format!("pull:{source_domain}:{}", ec_hash(ec_id)) -} - -fn drain_pull_batch( - in_flight: &mut Vec<(String, PlatformPendingRequest)>, - services: &RuntimeServices, - updates: &mut Vec, -) { - // All requests were dispatched up front via send_async, so waiting on - // each in turn does not change concurrency. - for (source_domain, waited) in drain_in_flight(in_flight, services, || true) { - let Waited::Response(response) = waited else { - continue; - }; - - let Some(uid) = extract_pull_uid(response, &source_domain) else { - continue; - }; - - updates.push(PartnerIdUpdate::new(source_domain, uid)); - } -} - -/// Maximum response body size accepted from pull sync partners (64 KiB). -/// -/// The expected response is `{"uid":""}`, so 64 KiB is generous. -/// This prevents a misbehaving partner from exhausting WASM memory. -const MAX_PULL_RESPONSE_BYTES: usize = 64 * 1024; - -fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option { - let status = response.response.status(); - - if status == StatusCode::NOT_FOUND { - log::debug!( - "Pull sync: partner '{}' returned 404, treating as no-op", - source_domain - ); - return None; - } - - if !status.is_success() { - log::warn!( - "Pull sync: partner '{}' returned non-success status {}", - source_domain, - status - ); - return None; - } - - let label = format!("Pull sync partner {source_domain}"); - let (_, body) = buffered_body(response, MAX_PULL_RESPONSE_BYTES, &label)?; - let payload = match serde_json::from_slice::(&body) { - Ok(payload) => payload, - Err(err) => { - log::warn!( - "Pull sync: partner '{}' returned invalid JSON body: {err}", - source_domain - ); - return None; - } - }; - - use super::kv_types::MAX_UID_LENGTH; - - let uid = payload.uid.filter(|value| !value.trim().is_empty()); - match uid { - None => { - log::debug!( - "Pull sync: partner '{}' returned null/empty uid, treating as no-op", - source_domain - ); - None - } - Some(ref value) if value.len() > MAX_UID_LENGTH => { - log::warn!( - "Pull sync: partner '{}' returned uid exceeding {} bytes (got {}), rejecting", - source_domain, - MAX_UID_LENGTH, - value.len() - ); - None - } - _ => uid, - } -} - #[cfg(test)] mod tests { use super::*; - use crate::consent::types::ConsentContext; - use crate::ec::kv_types::KvEntry; - use crate::platform::PlatformResponse; + use crate::ec::registry::PartnerConfig; use crate::redacted::Redacted; - use crate::test_support::tests::purpose_one_only_consent; - use http::header; - - fn make_response(status: u16, body: &[u8]) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - - fn make_response_with_content_length( - status: u16, - content_length: usize, - body: &[u8], - ) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .header(header::CONTENT_LENGTH, content_length.to_string()) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } fn pull_partner() -> PartnerConfig { PartnerConfig { @@ -502,152 +78,6 @@ mod tests { } } - #[test] - fn build_pull_sync_context_returns_context_when_valid() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - let context = build_pull_sync_context(&ec_context, ®istry) - .expect("should build pull sync context for valid EC"); - assert_eq!( - context.ec_id(), - ec_context.ec_value().expect("ec should be present"), - "should capture the EC ID from context" - ); - } - - #[test] - fn build_pull_sync_context_rejects_invalid_ec_id() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_context = EcContext::new_for_test(Some("invalid-ec".to_owned()), consent); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - let context = build_pull_sync_context(&ec_context, ®istry); - assert!( - context.is_none(), - "should reject pull sync context when EC ID format is invalid" - ); - } - - #[test] - fn build_pull_sync_context_skips_when_eid_persistence_is_denied() { - // Purpose 1 granted keeps the EC, but Purpose 4 denied withholds EID - // writes and leaves partner slots empty. Pull sync must not read those - // empty slots as eligible and disclose the EC ID to partners. - let consent = purpose_one_only_consent(); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - assert!( - ec_context.ec_allowed(), - "should keep the EC allowed when Purpose 1 is granted" - ); - - let context = build_pull_sync_context(&ec_context, ®istry); - - assert!( - context.is_none(), - "should skip pull sync when TCF Purpose 4 is denied" - ); - } - - #[test] - fn build_pull_sync_context_skips_empty_registry_and_complete_snapshot() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - let mut snapshot = seed_present_snapshot(&graph, &ec_id); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - assert!( - build_pull_sync_context(&ec_context, &PartnerRegistry::empty()).is_none(), - "no pull partners should skip before graph construction" - ); - assert!( - build_pull_sync_context(&ec_context, ®istry).is_none(), - "an unread snapshot should skip before graph construction" - ); - - if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { - entry.ids.insert( - "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::new("uid"), - ); - } - ec_context.set_kv_snapshot(snapshot); - assert!( - build_pull_sync_context(&ec_context, ®istry).is_none(), - "a complete snapshot should skip post-send work" - ); - } - - #[test] - fn partner_is_eligible_when_missing_from_entry() { - let partner = pull_partner(); - let entry = KvEntry::minimal("other_partner", "uid-1", 100); - - assert!( - is_partner_pull_eligible(&partner, Some(&entry)), - "should dispatch when partner has no stored UID" - ); - } - - #[test] - fn partner_is_not_eligible_when_already_present() { - let partner = pull_partner(); - let entry = KvEntry::minimal("ssp.example.com", "uid-1", 1000); - - assert!( - !is_partner_pull_eligible(&partner, Some(&entry)), - "should skip dispatch when partner already has a stored UID" - ); - } - - #[test] - fn completeness_requires_all_pull_partner_ids() { - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("a.example.com"), - pull_enabled_ec_partner("b.example.com"), - ]) - .expect("should build registry"); - let mut entry = KvEntry::minimal("a.example.com", "uid-a", 1_000); - - assert!( - !entry_is_pull_complete(&entry, ®istry), - "entry missing a pull partner ID should be incomplete" - ); - - entry.ids.insert( - "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::new("uid-b"), - ); - - assert!( - entry_is_pull_complete(&entry, ®istry), - "entry with every pull partner ID should be complete" - ); - } - #[test] fn validated_pull_sync_url_rejects_http_scheme() { let mut partner = pull_partner(); @@ -707,492 +137,4 @@ mod tests { "should not forward client IP to partners" ); } - - #[test] - fn pull_rate_limit_key_uses_ec_hash_only() { - let first_ec_id = format!("{}.ABC123", "a".repeat(64)); - let second_ec_id = format!("{}.XYZ789", "a".repeat(64)); - - let first_key = pull_rate_limit_key("ssp.example.com", &first_ec_id); - let second_key = pull_rate_limit_key("ssp.example.com", &second_ec_id); - - assert_eq!( - first_key, second_key, - "should bucket different suffixes for the same EC hash together" - ); - assert_eq!( - first_key, - format!("pull:ssp.example.com:{}", "a".repeat(64)), - "should key pull-sync rate limiting by source domain and EC hash" - ); - } - - #[test] - fn extract_pull_uid_treats_404_as_noop() { - let response = make_response(404, b""); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should treat 404 as no-op"); - } - - #[test] - fn extract_pull_uid_treats_uid_null_as_noop() { - let response = make_response(200, b"{\"uid\":null}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should treat uid=null as no-op"); - } - - #[test] - fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(513); - let body = format!("{{\"uid\":\"{long_uid}\"}}"); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding 512 bytes"); - } - - #[test] - fn extract_pull_uid_reads_uid_from_success_body() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should parse uid from 200 body" - ); - } - - #[test] - fn extract_pull_uid_rejects_oversized_content_length_before_body_read() { - let response = make_response_with_content_length( - 200, - MAX_PULL_RESPONSE_BYTES + 1, - b"{\"uid\":\"abc123\"}", - ); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!( - uid.is_none(), - "should reject oversized Content-Length before parsing body" - ); - } - - #[test] - fn extract_pull_uid_accepts_small_body_without_content_length() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should accept small valid response without Content-Length" - ); - } - - #[test] - fn extract_pull_uid_rejects_body_larger_than_limit() { - let body = format!("{{\"uid\":\"{}\"}}", "x".repeat(MAX_PULL_RESPONSE_BYTES)); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject body larger than limit"); - } - - #[test] - fn rotating_offset_distributes_partners_across_hours() { - // Simulate 3 partners sorted by source domain: alpha, beta, gamma. - let ids = vec!["alpha.example.com", "beta.example.com", "gamma.example.com"]; - - // Hour 0: offset = 0 % 3 = 0 → [alpha, beta, gamma] - let ts_h0: u64 = 100; // within hour 0 - let offset_h0 = (ts_h0 / 3600) as usize % ids.len(); - assert_eq!(offset_h0, 0, "hour 0 should start at index 0"); - - // Hour 1: offset = (3600 / 3600) % 3 = 1 → [beta, gamma, alpha] - let offset_h1 = (3600u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h1, 1, "hour 1 should start at index 1"); - - // Hour 2: offset = (7200 / 3600) % 3 = 2 → [gamma, alpha, beta] - let offset_h2 = (7200u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h2, 2, "hour 2 should start at index 2"); - - // Hour 3: offset = (10800 / 3600) % 3 = 0 → wraps back to [alpha, beta, gamma] - let offset_h3 = (10800u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h3, 0, "hour 3 should wrap back to index 0"); - - // Verify rotate_left produces expected ordering - let mut rotated = ids.clone(); - rotated.rotate_left(offset_h1); - assert_eq!( - rotated, - vec!["beta.example.com", "gamma.example.com", "alpha.example.com"], - "hour 1 rotation should move beta to front" - ); - } - - // ----------------------------------------------------------------------- - // Snapshot-driven eligibility and request-wide aggregation - // ----------------------------------------------------------------------- - - use crate::ec::kv::TombstoneOutcome; - use crate::error::TrustedServerError; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::settings::EcPartner; - use crate::test_support::tests::create_test_settings; - use error_stack::Report; - use std::sync::Arc; - - struct AllowAllRateLimiter; - - impl RateLimiter for AllowAllRateLimiter { - fn exceeded( - &self, - _key: &str, - _hourly_limit: u32, - ) -> Result> { - Ok(false) - } - } - - fn pull_enabled_ec_partner(source_domain: &str) -> EcPartner { - EcPartner { - name: format!("Partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(Redacted::new(format!( - "{source_domain}-api-token-32-bytes-minimum" - ))), - batch_rate_limit: EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://{source_domain}/sync")), - pull_sync_allowed_domains: vec![source_domain.to_owned()], - removed_pull_sync_ttl_sec: None, - removed_pull_sync_rate_limit: None, - ts_pull_token: Some(Redacted::new("outbound-token".to_owned())), - } - } - - fn snapshot_ec_id() -> String { - format!("{}.ABC123", "a".repeat(64)) - } - - fn seed_present_snapshot(graph: &KvIdentityGraph, ec_id: &str) -> EcKvSnapshot { - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - graph.create(ec_id, &entry).expect("should seed live entry"); - graph.load_snapshot(ec_id) - } - - #[test] - fn dispatch_pull_sync_aggregates_batches_into_one_bulk_write() { - let mut settings = create_test_settings(); - // Force one partner per concurrency batch so responses span batches. - settings.ec.identity.max_concurrency = 1; - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("alpha.example.com"), - pull_enabled_ec_partner("beta.example.com"), - ]) - .expect("should build pull registry"); - - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = seed_present_snapshot(&graph, &ec_id); - - let stub = Arc::new(StubHttpClient::new()); - // One JSON response per partner, drained across two concurrency batches. - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - let (entry, generation) = graph - .get(&ec_id) - .expect("should read store") - .expect("entry should exist"); - assert_eq!( - entry - .ids - .get("alpha.example.com") - .and_then(|id| id.first_uid()), - Some("synced-uid"), - "first partner UID should persist" - ); - assert_eq!( - entry - .ids - .get("beta.example.com") - .and_then(|id| id.first_uid()), - Some("synced-uid"), - "second partner UID should persist" - ); - assert_eq!( - generation, 2, - "two partner responses across batches must persist in exactly one bulk write" - ); - } - - #[test] - fn dispatch_pull_sync_calls_and_persists_only_missing_partner() { - let mut settings = create_test_settings(); - settings.ec.identity.max_concurrency = 4; - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("alpha.example.com"), - pull_enabled_ec_partner("beta.example.com"), - ]) - .expect("should build pull registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - graph - .create( - &ec_id, - &KvEntry::minimal("alpha.example.com", "existing-alpha", 1_000), - ) - .expect("should seed partial entry"); - let snapshot = graph.load_snapshot(&ec_id); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"new-beta"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert_eq!( - stub.recorded_backend_names().len(), - 1, - "should call only the missing partner" - ); - let (entry, generation) = graph - .get(&ec_id) - .expect("should read store") - .expect("entry should exist"); - assert_eq!( - entry - .ids - .get("alpha.example.com") - .and_then(|partner_id| partner_id.first_uid()), - Some("existing-alpha"), - "existing UID should remain unchanged" - ); - assert_eq!( - entry - .ids - .get("beta.example.com") - .and_then(|partner_id| partner_id.first_uid()), - Some("new-beta"), - "missing UID should persist" - ); - assert_eq!(generation, 2, "missing UID should use one bulk CAS write"); - } - - #[test] - fn dispatch_pull_sync_skips_non_present_snapshots() { - let mut settings = create_test_settings(); - settings.ec.identity.max_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - for snapshot in [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ] { - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - } - - assert!( - stub.recorded_backend_names().is_empty(), - "not-read, missing, and failed snapshots must not dispatch pull sync" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "no snapshot state should create a missing root" - ); - } - - #[test] - fn dispatch_pull_sync_skips_tombstone_snapshot() { - let mut settings = create_test_settings(); - settings.ec.identity.max_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1000)), - generation: Some(1), - }; - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a tombstone snapshot must not dispatch pull sync" - ); - } - #[test] - fn dispatch_pull_sync_skips_dispatch_when_ec_is_tombstoned_after_snapshot_capture() { - // The request snapshot authorized pull sync, then a concurrent request - // completed a CMP withdrawal while the page response was in flight. - // Pull sync runs post-send and discloses the raw `ec_id` to partners, so - // it must revalidate the live row and cancel rather than leak an - // identity the user just withdrew. - let mut settings = create_test_settings(); - settings.ec.identity.max_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = seed_present_snapshot(&graph, &ec_id); - - // Concurrent withdrawal lands after the snapshot was captured. - assert_eq!( - graph - .write_withdrawal_tombstone(&ec_id, drop) - .expect("should tombstone the row"), - TombstoneOutcome::Written, - "should tombstone the row the snapshot still reports as present" - ); - - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"leaked-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a withdrawal that lands after snapshot capture must cancel post-send pull sync" - ); - let (entry, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("tombstone should remain"); - assert!( - entry.ids.is_empty(), - "no partner UID may be written back onto a tombstoned row" - ); - } - - #[test] - fn dispatch_pull_sync_skips_revalidation_read_when_no_partner_is_eligible() { - // Every pull-enabled partner already has a UID in the request snapshot, - // so there is nothing to dispatch. The revalidation read exists to - // authorize outbound calls; with no calls to authorize it must not cost - // a KV operation. - let mut settings = create_test_settings(); - settings.ec.identity.max_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::new("already-known"), - ); - graph - .create(&ec_id, &entry) - .expect("should seed live entry"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: Some(1), - }; - - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a fully synced entry must not dispatch pull sync" - ); - } } diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs deleted file mode 100644 index efa96233f3..0000000000 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ /dev/null @@ -1,689 +0,0 @@ -//! Short-lived browser marker for complete pull-sync partner state. - -use core::fmt; - -use edgezero_core::body::Body as EdgeBody; -use hmac::{Hmac, Mac as _}; -use http::{HeaderValue, Response, header}; -use sha2::{Digest as _, Sha256}; - -use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; -use crate::redacted::Redacted; -use crate::settings::Settings; - -use super::pull_sync::entry_is_pull_complete; -use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, current_timestamp}; - -type HmacSha256 = Hmac; - -const MARKER_VERSION: &str = "v1"; -const MARKER_KEY_LABEL: &[u8] = b"trusted-server/ec-pull-complete/key/v1"; -const MARKER_MAX_AGE_SECS: u64 = 60 * 60; -const MARKER_CLOCK_SKEW_SECS: u64 = 60; -const MAX_MARKER_LENGTH: usize = 256; - -/// Request-local validation state for the pull-sync completeness marker. -#[derive(Clone, Default)] -pub(crate) enum PullSyncMarkerState { - /// No marker cookie was present. - #[default] - Absent, - /// A marker was present but has not been checked against the active EC and partner set. - Unvalidated(Redacted), - /// A present marker failed validation or was disproved by authoritative KV state. - Invalid, - /// The marker is valid until the given Unix timestamp. - Valid { expires_at: u64 }, -} - -impl fmt::Debug for PullSyncMarkerState { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Absent => formatter.write_str("Absent"), - Self::Unvalidated(_) => formatter.write_str("Unvalidated()"), - Self::Invalid => formatter.write_str("Invalid"), - Self::Valid { expires_at } => formatter - .debug_struct("Valid") - .field("expires_at", expires_at) - .finish(), - } - } -} - -impl PullSyncMarkerState { - /// Creates marker state from an optional request cookie value. - #[must_use] - pub(crate) fn from_cookie(value: Option) -> Self { - value - .map(Redacted::new) - .map_or(Self::Absent, Self::Unvalidated) - } - - /// Returns whether a marker cookie was present on the request. - #[must_use] - pub(crate) fn was_present(&self) -> bool { - !matches!(self, Self::Absent) - } - - /// Returns whether the marker is currently valid. - #[must_use] - pub(crate) fn is_valid(&self) -> bool { - matches!(self, Self::Valid { .. }) - } - - /// Invalidates state bound to a replaced active EC ID. - pub(crate) fn invalidate_for_replaced_ec(&mut self) { - if self.was_present() { - *self = Self::Invalid; - } - } -} - -/// Validates an unvalidated marker against the active EC and current pull-partner set. -pub(crate) fn validate_marker_state( - state: &mut PullSyncMarkerState, - settings: &Settings, - registry: &PartnerRegistry, - ec_id: Option<&str>, -) { - let PullSyncMarkerState::Unvalidated(value) = state else { - return; - }; - let valid_until = ec_id.and_then(|ec_id| { - validate_marker( - value.expose(), - settings, - registry, - ec_id, - current_timestamp(), - ) - }); - *state = valid_until.map_or(PullSyncMarkerState::Invalid, |expires_at| { - PullSyncMarkerState::Valid { expires_at } - }); -} - -/// Reconciles browser marker state against finalized authoritative KV state. -pub(crate) fn reconcile_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: Option<&str>, - snapshot: &EcKvSnapshot, - state: &mut PullSyncMarkerState, - response: &mut Response, -) { - let pull_partners = sorted_pull_partner_domains(registry); - if pull_partners.is_empty() { - expire_if_present(state, response); - return; - } - - let Some(ec_id) = ec_id else { - expire_if_present(state, response); - return; - }; - - if !matches!(snapshot, EcKvSnapshot::NotRead) && !snapshot.belongs_to(ec_id) { - expire_if_present(state, response); - return; - } - - match snapshot { - EcKvSnapshot::Present { .. } => { - let entry = snapshot - .entry_for(ec_id) - .expect("snapshot binding should be checked before marker reconciliation"); - if !entry.consent.ok { - expire_if_present(state, response); - } else if entry_is_pull_complete(entry, registry) { - if !state.is_valid() { - set_marker(settings, registry, ec_id, state, response); - } - } else { - expire_if_present(state, response); - } - } - EcKvSnapshot::Missing { .. } => { - expire_if_present(state, response); - } - EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => { - if matches!(state, PullSyncMarkerState::Invalid) { - expire_if_present(state, response); - } - } - } -} - -/// Expires the completeness marker regardless of KV state. -pub(crate) fn expire_marker(state: &mut PullSyncMarkerState, response: &mut Response) { - append_cookie(response, &format_marker_cookie("", 0)); - *state = PullSyncMarkerState::Absent; -} - -fn set_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - state: &mut PullSyncMarkerState, - response: &mut Response, -) { - let now = current_timestamp(); - let expires_at = now.saturating_add(MARKER_MAX_AGE_SECS); - let Some(value) = create_marker(settings, registry, ec_id, expires_at) else { - return; - }; - append_cookie(response, &format_marker_cookie(&value, MARKER_MAX_AGE_SECS)); - *state = PullSyncMarkerState::Valid { expires_at }; -} - -fn expire_if_present(state: &mut PullSyncMarkerState, response: &mut Response) { - if state.was_present() { - expire_marker(state, response); - } -} - -fn append_cookie(response: &mut Response, value: &str) { - match HeaderValue::from_str(value) { - Ok(value) => { - response.headers_mut().append(header::SET_COOKIE, value); - } - Err(err) => { - log::warn!("Skipping pull-sync marker cookie: invalid header value: {err}"); - } - } -} - -fn format_marker_cookie(value: &str, max_age: u64) -> String { - // Keep this cookie host-only. The signed payload does not include the - // request host, so a Domain attribute would allow replay across sibling - // hosts that share the domain-wide EC cookie and passphrase. - format!( - "{COOKIE_TS_EC_PULL_COMPLETE}={value}; Path=/; Secure; SameSite=Lax; Max-Age={max_age}; HttpOnly" - ) -} - -#[cfg(test)] -pub(crate) fn create_marker_for_test( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, -) -> String { - create_marker_for_test_with_expiry( - settings, - registry, - ec_id, - current_timestamp().saturating_add(MARKER_MAX_AGE_SECS), - ) -} - -#[cfg(test)] -pub(crate) fn create_marker_for_test_with_expiry( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - expires_at: u64, -) -> String { - create_marker(settings, registry, ec_id, expires_at) - .expect("should create marker for non-empty test registry") -} - -fn create_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - expires_at: u64, -) -> Option { - let fingerprint = partner_set_fingerprint(registry)?; - let payload = marker_payload(ec_id, expires_at, &fingerprint); - let key = marker_key(settings); - let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); - mac.update(payload.as_bytes()); - let tag = hex::encode(mac.finalize().into_bytes()); - Some(format!("{MARKER_VERSION}.{expires_at}.{fingerprint}.{tag}")) -} - -fn validate_marker( - value: &str, - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - now: u64, -) -> Option { - if value.len() > MAX_MARKER_LENGTH { - return None; - } - - let mut segments = value.split('.'); - let version = segments.next()?; - let expires = segments.next()?; - let fingerprint = segments.next()?; - let tag = segments.next()?; - if segments.next().is_some() || version != MARKER_VERSION { - return None; - } - - let expires_at = expires.parse::().ok()?; - let max_future = now - .saturating_add(MARKER_MAX_AGE_SECS) - .saturating_add(MARKER_CLOCK_SKEW_SECS); - if expires_at <= now || expires_at > max_future { - return None; - } - - let expected_fingerprint = partner_set_fingerprint(registry)?; - if fingerprint != expected_fingerprint { - return None; - } - - let tag = hex::decode(tag).ok()?; - if tag.len() != 32 { - return None; - } - - let payload = marker_payload(ec_id, expires_at, fingerprint); - let key = marker_key(settings); - let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); - mac.update(payload.as_bytes()); - mac.verify_slice(&tag).ok()?; - Some(expires_at) -} - -fn marker_key(settings: &Settings) -> [u8; 32] { - let mut mac = HmacSha256::new_from_slice(settings.ec.passphrase.expose().as_bytes()) - .expect("should create marker key HMAC"); - mac.update(MARKER_KEY_LABEL); - mac.finalize().into_bytes().into() -} - -fn marker_payload(ec_id: &str, expires_at: u64, fingerprint: &str) -> String { - format!("{MARKER_VERSION}\0{ec_id}\0{expires_at}\0{fingerprint}") -} - -fn partner_set_fingerprint(registry: &PartnerRegistry) -> Option { - let domains = sorted_pull_partner_domains(registry); - if domains.is_empty() { - return None; - } - - let mut hasher = Sha256::new(); - hasher.update(b"trusted-server/ec-pull-partners/v1\0"); - for domain in domains { - hasher.update((domain.len() as u64).to_be_bytes()); - hasher.update(domain.as_bytes()); - } - Some(hex::encode(hasher.finalize())) -} - -fn sorted_pull_partner_domains(registry: &PartnerRegistry) -> Vec { - let mut domains = registry - .pull_enabled_partners() - .into_iter() - .map(|partner| partner.source_domain.clone()) - .collect::>(); - domains.sort(); - domains -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ec::kv_types::KvEntry; - use crate::redacted::Redacted; - use crate::settings::{EcPartner, Settings}; - use crate::test_support::tests::create_test_settings; - - const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; - - fn pull_partner(source_domain: &str) -> EcPartner { - EcPartner { - name: format!("Partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(Redacted::new(format!( - "token-{source_domain}-32-bytes-minimum-value" - ))), - batch_rate_limit: EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), - pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - removed_pull_sync_ttl_sec: None, - removed_pull_sync_rate_limit: None, - ts_pull_token: Some(Redacted::new("pull-token".to_owned())), - } - } - - fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { - let mut settings = create_test_settings(); - settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); - (settings, registry) - } - - fn empty_response() -> Response { - Response::builder() - .status(200) - .body(EdgeBody::empty()) - .expect("should build response") - } - - fn live_snapshot(ec_id: &str, domains: &[&str]) -> EcKvSnapshot { - let mut entry = KvEntry::tombstone(1_000); - entry.consent.ok = true; - for domain in domains { - entry.ids.insert( - (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId::new(format!("uid-{domain}")), - ); - } - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: Some(1), - } - } - - fn marker_cookies(response: &Response) -> Vec<&str> { - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect() - } - - fn assert_expired(state: &PullSyncMarkerState, response: &Response) { - assert!(matches!(state, PullSyncMarkerState::Absent)); - assert_eq!( - marker_cookies(response), - vec!["ts-ec-pull-complete=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly"], - "should emit the exact host-only expiration cookie" - ); - } - - #[test] - fn marker_round_trip_binds_ec_and_partner_set() { - let (settings, registry) = settings_and_registry(&["a.example.com", "b.example.com"]); - let now = 1_000; - let marker = - create_marker(&settings, ®istry, EC_ID, now + 3_600).expect("should create marker"); - - assert_eq!( - validate_marker(&marker, &settings, ®istry, EC_ID, now), - Some(now + 3_600), - "should validate the issued marker" - ); - assert!( - validate_marker(&marker, &settings, ®istry, "wrong", now).is_none(), - "should reject a marker for another EC" - ); - } - - #[test] - fn marker_fingerprint_is_order_independent_and_set_sensitive() { - let (settings, first) = settings_and_registry(&["a.example.com", "b.example.com"]); - let (_, reordered) = settings_and_registry(&["b.example.com", "a.example.com"]); - let (_, changed) = settings_and_registry(&["a.example.com", "c.example.com"]); - let marker = create_marker(&settings, &first, EC_ID, 4_600).expect("should create marker"); - - assert!( - validate_marker(&marker, &settings, &reordered, EC_ID, 1_000).is_some(), - "config ordering should not change the marker" - ); - assert!( - validate_marker(&marker, &settings, &changed, EC_ID, 1_000).is_none(), - "partner-set changes should invalidate the marker" - ); - } - - #[test] - fn fingerprint_changes_for_enable_disable_add_and_remove() { - let (_, enabled_one) = settings_and_registry(&["a.example.com"]); - let (_, enabled_two) = settings_and_registry(&["a.example.com", "b.example.com"]); - let mut disabled_config = pull_partner("a.example.com"); - disabled_config.pull_sync_enabled = false; - let disabled = PartnerRegistry::from_config(&[disabled_config]) - .expect("should build disabled registry"); - let removed = PartnerRegistry::empty(); - - let base = partner_set_fingerprint(&enabled_one); - assert_ne!( - base, - partner_set_fingerprint(&enabled_two), - "adding a pull partner should change the fingerprint" - ); - assert_ne!( - base, - partner_set_fingerprint(&disabled), - "disabling a pull partner should change the fingerprint" - ); - assert_ne!( - base, - partner_set_fingerprint(&removed), - "removing a pull partner should change the fingerprint" - ); - assert_ne!( - partner_set_fingerprint(&disabled), - partner_set_fingerprint(&enabled_one), - "enabling a partner should change the fingerprint" - ); - } - - #[test] - fn reconcile_rejects_snapshot_bound_to_different_ec() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let snapshot = live_snapshot("different-ec", &["a.example.com"]); - let mut valid_state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut valid_response = empty_response(); - - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut valid_state, - &mut valid_response, - ); - assert_expired(&valid_state, &valid_response); - - let mut absent_state = PullSyncMarkerState::Absent; - let mut absent_response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut absent_state, - &mut absent_response, - ); - assert!(marker_cookies(&absent_response).is_empty()); - assert!(matches!(absent_state, PullSyncMarkerState::Absent)); - } - - #[test] - fn authoritative_incomplete_states_expire_valid_marker() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let mut tombstone = KvEntry::tombstone(1_000); - tombstone.ids.insert( - "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::new("stale"), - ); - let snapshots = [ - live_snapshot(EC_ID, &[]), - EcKvSnapshot::Missing { - ec_id: EC_ID.to_owned(), - }, - EcKvSnapshot::Present { - ec_id: EC_ID.to_owned(), - entry: Box::new(tombstone), - generation: Some(1), - }, - ]; - - for snapshot in snapshots { - let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut state, - &mut response, - ); - assert_expired(&state, &response); - } - } - - #[test] - fn non_authoritative_states_preserve_valid_fixed_expiry_marker() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let snapshots = [ - EcKvSnapshot::Failed { - ec_id: EC_ID.to_owned(), - }, - EcKvSnapshot::NotRead, - ]; - - for snapshot in snapshots { - let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut state, - &mut response, - ); - assert!(matches!( - state, - PullSyncMarkerState::Valid { expires_at: 4_600 } - )); - assert!(marker_cookies(&response).is_empty()); - } - } - - #[test] - fn steady_states_leave_the_marker_cookie_untouched() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let complete = live_snapshot(EC_ID, &["a.example.com"]); - let incomplete = live_snapshot(EC_ID, &[]); - let cases = [ - (PullSyncMarkerState::Valid { expires_at: 4_600 }, &complete), - (PullSyncMarkerState::Absent, &incomplete), - ]; - - for (mut state, snapshot) in cases { - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - snapshot, - &mut state, - &mut response, - ); - assert!( - marker_cookies(&response).is_empty(), - "a steady marker state should not add a Set-Cookie header" - ); - } - } - - #[test] - fn invalid_marker_is_cleared_without_authoritative_snapshot() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let mut state = PullSyncMarkerState::Invalid; - let mut response = empty_response(); - - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &EcKvSnapshot::NotRead, - &mut state, - &mut response, - ); - - assert_expired(&state, &response); - } - - #[test] - fn marker_allows_sixty_seconds_of_issuer_clock_skew() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - - assert_eq!( - validate_marker(&marker, &settings, ®istry, EC_ID, 940), - Some(4_600), - "a marker from an issuer sixty seconds ahead should remain valid" - ); - } - - #[test] - fn marker_rejects_expired_overlong_and_tampered_values() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - let mut tampered = marker.clone(); - tampered.push('0'); - - assert!( - validate_marker(&marker, &settings, ®istry, EC_ID, 4_600).is_none(), - "expired marker should fail" - ); - assert!( - validate_marker(&marker, &settings, ®istry, EC_ID, 939).is_none(), - "marker beyond the lifetime and clock-skew allowance should fail" - ); - assert!( - validate_marker(&tampered, &settings, ®istry, EC_ID, 1_000).is_none(), - "tampering should fail" - ); - assert!( - validate_marker( - &"x".repeat(MAX_MARKER_LENGTH + 1), - &settings, - ®istry, - EC_ID, - 1_000 - ) - .is_none(), - "overlong marker should fail" - ); - } - - #[test] - fn marker_rejects_wrong_passphrase_and_empty_partner_set() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - let mut changed_settings = settings.clone(); - changed_settings.ec.passphrase = - Redacted::new("different-secret-key-32-bytes-minimum".to_owned()); - let empty = PartnerRegistry::empty(); - - assert!( - validate_marker(&marker, &changed_settings, ®istry, EC_ID, 1_000).is_none(), - "passphrase rotation should invalidate the marker" - ); - assert!( - create_marker(&settings, &empty, EC_ID, 4_600).is_none(), - "empty partner sets should not produce a marker" - ); - } - - #[test] - fn marker_cookie_is_host_only_and_secure() { - let cookie = format_marker_cookie("value", MARKER_MAX_AGE_SECS); - assert_eq!( - cookie, - "ts-ec-pull-complete=value; Path=/; Secure; SameSite=Lax; Max-Age=3600; HttpOnly" - ); - assert!(!cookie.contains("Domain="), "marker should be host-only"); - } -} diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 335347eb55..ba9094cc22 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -57,7 +57,7 @@ use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::template_cache_policy::{TemplateCookieDecision, evaluate_cookie_policy}; use crate::cookies::{ TS_ORIGIN_STRIPPED_COOKIE_NAMES, handle_request_cookies, strip_cookie_headers, @@ -3107,19 +3107,16 @@ struct EcSnapshotPreloadInput { is_get: bool, has_ec_id: bool, has_kv: bool, - marker_valid: bool, + /// A returning EC's writer navigation: phase 1 reads the row on every + /// one (spec 5.2, 6.3). + returning_writer: bool, auction_needs_row: bool, - eid_cookie_may_need_persistence: bool, privacy_needs_row: bool, } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; - let marker_can_skip = input.marker_valid - && !input.auction_needs_row - && !input.eid_cookie_may_need_persistence - && !input.privacy_needs_row; - eligible && !marker_can_skip + eligible && (input.returning_writer || input.auction_needs_row || input.privacy_needs_row) } /// Rewrites a downstream request into an outbound publisher-origin request. @@ -4488,9 +4485,6 @@ pub async fn handle_publisher_request( let ec_id_owned = active_ec_id_owned.clone().filter(|_| ec_allowed); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; - if let Some(registry) = auction.registry { - ec_context.validate_pull_sync_marker(settings, registry); - } let geo = ec_context.geo_info().cloned(); let parsed_origin = url::Url::parse(&settings.publisher.origin_url).change_context( @@ -4806,24 +4800,19 @@ pub async fn handle_publisher_request( let request_method = req.method().clone(); let mut origin_request = Some(req); - let eid_cookie_may_need_persistence = cookie_jar - .as_ref() - .is_some_and(|jar| jar.get(COOKIE_TS_EIDS).is_some() || jar.get(COOKIE_SHAREDID).is_some()); - // This decision also gates the concurrent origin send below. A marker skip - // cannot currently delay the origin behind an auction because marker - // validation requires a non-empty registry, and every such auction sets - // `auction_needs_row` and retains the preload. + // This decision also gates the concurrent origin send below. let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { is_navigation, is_get, has_ec_id: active_ec_id.is_some(), has_kv: kv.is_some(), - marker_valid: ec_context.pull_sync_marker().is_valid(), + returning_writer: ec_context.identity().writer() + && ec_context.ec_was_present() + && !ec_context.ec_generated(), auction_needs_row: should_run_auction && auction .registry .is_some_and(|registry| !registry.is_empty()), - eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), }); let mut pending_origin = None; @@ -7539,9 +7528,8 @@ mod tests { is_get: true, has_ec_id: true, has_kv: true, - marker_valid: false, + returning_writer: true, auction_needs_row: false, - eid_cookie_may_need_persistence: false, privacy_needs_row: false, }; assert!(should_preload_ec_snapshot(&baseline)); @@ -7564,29 +7552,24 @@ mod tests { } #[test] - fn valid_marker_skips_only_when_no_other_consumer_needs_snapshot() { - let marker_only = EcSnapshotPreloadInput { + fn stager_navigation_preloads_only_for_an_auction_or_a_withdrawal() { + let stager = EcSnapshotPreloadInput { is_navigation: true, is_get: true, has_ec_id: true, has_kv: true, - marker_valid: true, + returning_writer: false, auction_needs_row: false, - eid_cookie_may_need_persistence: false, privacy_needs_row: false, }; - assert!(!should_preload_ec_snapshot(&marker_only)); + assert!(!should_preload_ec_snapshot(&stager)); assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { auction_needs_row: true, - ..marker_only - })); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - eid_cookie_may_need_persistence: true, - ..marker_only + ..stager })); assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { privacy_needs_row: true, - ..marker_only + ..stager })); } use crate::auction::test_support::canonical_parity_auction_request; @@ -7839,11 +7822,21 @@ mod tests { Arc::clone(&http) as Arc ); let ec_id = format!("{}.CkId01", "b".repeat(64)); - let mut ec_context = EcContext::new_for_test_with_ip( + let mut ec_context = EcContext::new_writer_for_test_with_ip( Some(ec_id), scheduling_consent(), Some("203.0.113.7".to_owned()), ); + // A returning visitor's writer navigation, which phase 1 always reads. + ec_context.identity_mut().set_writer_decision( + crate::ec::identity::request::WriterDecision { + writer: true, + is_real_browser: true, + user_agent: None, + dnt: false, + page_url: None, + }, + ); assert!( ec_context.ec_allowed() && ec_context.ec_value().is_some(), "test precondition: an active, consent-allowed EC must exist" @@ -7964,40 +7957,12 @@ mod tests { } } - #[derive(Clone, Copy)] - enum MarkerProbe { - Valid, - Malformed, - WrongEc, - Expired, - PartnerSetMismatch, - } - - fn marker_partner(source_domain: &str) -> crate::settings::EcPartner { - crate::settings::EcPartner { - name: format!("Marker partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(crate::redacted::Redacted::new(format!( - "marker-{source_domain}-api-token-32-bytes-minimum" - ))), - batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), - pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - removed_pull_sync_ttl_sec: None, - removed_pull_sync_rate_limit: None, - ts_pull_token: Some(crate::redacted::Redacted::new("pull-token".to_owned())), - } - } - #[test] fn body_less_auction_eids_put_cookie_entries_first_and_kv_only_sources_after() { let ec_id = format!("{}.navmrg", "a".repeat(64)); let registry = PartnerRegistry::from_config(&[ - marker_partner("ids.example.com"), - marker_partner("other.example.com"), + pull_partner_config("ids.example.com"), + pull_partner_config("other.example.com"), ]) .expect("should build partner registry"); let mut entry = KvEntry::minimal("ids.example.com", "kv-stale", 1000); @@ -8105,8 +8070,8 @@ mod tests { fn publisher_merge_fixture(entry: KvEntry) -> (EcKvSnapshot, PartnerRegistry, EcContext) { let ec_id = format!("{}.navmrg", "a".repeat(64)); let registry = PartnerRegistry::from_config(&[ - marker_partner("ids.example.com"), - marker_partner("other.example.com"), + pull_partner_config("ids.example.com"), + pull_partner_config("other.example.com"), ]) .expect("should build partner registry"); let snapshot = EcKvSnapshot::Present { @@ -8187,7 +8152,7 @@ mod tests { ); let (snapshot, registry, mut ec_context) = publisher_merge_fixture(entry); let mut settings = create_test_settings(); - settings.ec.partners = vec![marker_partner("ids.example.com")]; + settings.ec.partners = vec![pull_partner_config("ids.example.com")]; ec_context.set_identity_runtime(Arc::new( crate::ec::identity::IdentityRuntime::from_settings(&settings) .expect("should build the runtime"), @@ -8209,25 +8174,34 @@ mod tests { ); } - async fn run_marker_lookup_probe( - probe: MarkerProbe, - has_eid_cookie: bool, - has_matched_auction_slot: bool, - ) -> usize { - let mut settings = if has_matched_auction_slot { - scheduling_settings() - } else { - create_test_settings() - }; - settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + fn pull_partner_config(source_domain: &str) -> crate::settings::EcPartner { + crate::settings::EcPartner { + name: format!("Partner {source_domain}"), + source_domain: source_domain.to_owned(), + openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(crate::redacted::Redacted::new(format!( + "partner-{source_domain}-api-token-32-bytes-minimum" + ))), + batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), + pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], + removed_pull_sync_ttl_sec: None, + removed_pull_sync_rate_limit: None, + ts_pull_token: Some(crate::redacted::Redacted::new("pull-token".to_owned())), + } + } + + /// Runs a navigation without an auction and counts its KV reads. + async fn run_navigation_lookup_probe(writer: bool) -> usize { + let settings = create_test_settings(); let http = Arc::new(StubHttpClient::new()); http.set_concurrent_fanout(true); http.push_response(200, b"ok".to_vec()); let lookups = Arc::new(AtomicUsize::new(0)); let graph = KvIdentityGraph::new(OrderRecordingKv { - inner: InMemoryEcKv::new("marker-store"), + inner: InMemoryEcKv::new("probe-store"), http: Arc::clone(&http), http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), lookups: Arc::clone(&lookups), @@ -8236,54 +8210,21 @@ mod tests { Arc::clone(&http) as Arc ); let ec_id = format!("{}.CkId01", "b".repeat(64)); - let marker = match probe { - MarkerProbe::Valid => { - crate::ec::pull_sync_marker::create_marker_for_test(&settings, ®istry, &ec_id) - } - MarkerProbe::Malformed => "not-a-marker".to_owned(), - MarkerProbe::WrongEc => crate::ec::pull_sync_marker::create_marker_for_test( - &settings, - ®istry, - &format!("{}.Other1", "c".repeat(64)), - ), - MarkerProbe::Expired => { - crate::ec::pull_sync_marker::create_marker_for_test_with_expiry( - &settings, - ®istry, - &ec_id, - crate::ec::current_timestamp().saturating_sub(1), - ) - } - MarkerProbe::PartnerSetMismatch => { - let other_registry = - PartnerRegistry::from_config(&[marker_partner("other.example.com")]) - .expect("should build alternate registry"); - crate::ec::pull_sync_marker::create_marker_for_test( - &settings, - &other_registry, - &ec_id, - ) - } - }; - let mut ec_context = EcContext::new_for_test_with_ip( + let mut ec_context = EcContext::new_writer_for_test_with_ip( Some(ec_id), scheduling_consent(), Some("203.0.113.7".to_owned()), ); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), + ec_context.identity_mut().set_writer_decision( + crate::ec::identity::request::WriterDecision { + writer, + is_real_browser: true, + user_agent: None, + dnt: false, + page_url: None, + }, ); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let slots = has_matched_auction_slot - .then(scheduling_slot) - .into_iter() - .collect::>(); - let mut request = navigation_request(); - if has_eid_cookie { - request - .headers_mut() - .insert(header::COOKIE, HeaderValue::from_static("ts-eids=present")); - } let result = handle_publisher_request( &settings, @@ -8292,10 +8233,10 @@ mod tests { &mut ec_context, AuctionDispatch { orchestrator: &orchestrator, - slots: &slots, - registry: Some(®istry), + slots: &[], + registry: None, }, - request, + navigation_request(), EdgeCacheHeader::SurrogateControl, ) .await; @@ -8305,42 +8246,16 @@ mod tests { } #[tokio::test] - async fn valid_completeness_marker_skips_pull_only_snapshot_lookup() { + async fn returning_writer_navigation_reads_kv_without_an_auction() { assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, false, false).await, - 0 - ); - } - - #[tokio::test] - async fn invalid_markers_fall_back_to_snapshot_lookup() { - for probe in [ - MarkerProbe::Malformed, - MarkerProbe::WrongEc, - MarkerProbe::Expired, - MarkerProbe::PartnerSetMismatch, - ] { - assert_eq!( - run_marker_lookup_probe(probe, false, false).await, - 1, - "invalid marker should retain the normal preload" - ); - } - } - - #[tokio::test] - async fn valid_marker_does_not_skip_eid_cookie_persistence_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, true, false).await, - 1 + run_navigation_lookup_probe(true).await, + 1, + "should read the row once in phase 1" ); - } - - #[tokio::test] - async fn valid_marker_does_not_skip_auction_snapshot_lookup() { assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, false, true).await, - 1 + run_navigation_lookup_probe(false).await, + 0, + "control: a stager navigation without an auction should read nothing" ); } @@ -11704,7 +11619,7 @@ mod tests { queue_shareable_html(&stub); for (index, cache_status) in ["miss-stored", "hit"].into_iter().enumerate() { - let mut ec_context = EcContext::new_for_test_with_ip( + let mut ec_context = EcContext::new_writer_for_test_with_ip( Some(format!("{}.CkId01", "b".repeat(64))), scheduling_consent(), Some("203.0.113.7".to_owned()), @@ -17260,12 +17175,23 @@ mod tests { .map(|(_, value)| value.as_str()) } + /// A returning visitor's writer navigation, which phase 1 reads. fn active_ec_context() -> EcContext { - EcContext::new_for_test_with_ip( + let mut context = EcContext::new_writer_for_test_with_ip( Some(format!("{}.CkId01", "b".repeat(64))), scheduling_consent(), Some("203.0.113.7".to_owned()), - ) + ); + context.identity_mut().set_writer_decision( + crate::ec::identity::request::WriterDecision { + writer: true, + is_real_browser: true, + user_agent: None, + dnt: false, + page_url: None, + }, + ); + context } #[tokio::test] @@ -18500,7 +18426,7 @@ mod tests { ..Default::default() }; let mut ec_context = - EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + EcContext::new_writer_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); assert!( ec_context.ec_allowed(), "test precondition: consent must allow EC creation" From 5cfe1529557aee4eb464359c539b644d1c47230f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 00:03:37 +0530 Subject: [PATCH 41/89] Finalize the response when the reloaded identity runtime is unavailable The settings-reload branch now runs finalize and the withdrawal retry without an identity runtime and skips only phase 4, so withdrawal cookies and tombstones are still enforced. A missing identity graph is logged with its cause. The error-status cut keeps only n, f and removals of stand-alone state changes, as documented. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/main.rs | 174 ++++++++++++++---- .../src/ec/identity/mutation.rs | 17 +- 2 files changed, 153 insertions(+), 38 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index d675e6206f..98df4bf87c 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -513,10 +513,15 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re send_edgezero_response(response, request_filter_effects.as_ref(), counters.as_ref()); }; if let Some(state) = app_state.as_deref() { - let graph = || state.identity_graph.clone(); + let graph = || { + if state.identity_graph.is_none() { + log::warn!("identity graph unavailable: ec.ec_store is not configured"); + } + state.identity_graph.clone() + }; finalize_send_and_flush( PostRouteInput { - runtime: &state.identity, + runtime: Some(&state.identity), settings: &state.settings, ec_state: &mut ec_state, graph: &graph, @@ -528,34 +533,45 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re return; } match load_settings_from_config_store(&runtime_stores) { - Ok(settings) => match build_identity_runtime(&settings) { - Ok(identity) => { - // No application state was built, so the identity runtime - // comes from the reloaded settings. - ec_state - .ec_context - .set_identity_runtime(Arc::clone(&identity)); - let graph = || maybe_identity_graph(&settings); - finalize_send_and_flush( - PostRouteInput { - runtime: &identity, - settings: &settings, - ec_state: &mut ec_state, - graph: &graph, - rate_limiter: &limiter, - }, - response, - &mut send, - ); - return; - } - Err(e) => { - log::warn!( - "EdgeZero EC finalize skipped: identity runtime unavailable after settings reload: {}", - e.current_context() - ); - } - }, + Ok(settings) => { + // No application state was built, so the identity runtime + // comes from the reloaded settings. Without one, finalize and + // the withdrawal retry still run; only phase 4 is skipped. + let identity = match build_identity_runtime(&settings) { + Ok(identity) => { + ec_state + .ec_context + .set_identity_runtime(Arc::clone(&identity)); + Some(identity) + } + Err(e) => { + log::warn!( + "identity flush skipped: identity runtime unavailable after settings reload: {}", + e.current_context() + ); + None + } + }; + let graph = || { + require_identity_graph(&settings) + .map_err(|e| { + log::warn!("identity graph unavailable: {}", e.current_context()); + }) + .ok() + }; + finalize_send_and_flush( + PostRouteInput { + runtime: identity.as_deref(), + settings: &settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &limiter, + }, + response, + &mut send, + ); + return; + } Err(e) => { log::warn!("EdgeZero EC finalize skipped: failed to reload settings: {e:?}"); } @@ -603,7 +619,9 @@ fn apply_entry_point_finalize_headers( /// What the post-route work reads. It takes the runtime and settings rather /// than [`AppState`], because the settings-reload branch has none. struct PostRouteInput<'a> { - runtime: &'a IdentityRuntime, + /// `None` when the runtime could not be built: finalize and the + /// withdrawal retry still run, and phase 4 is skipped. + runtime: Option<&'a IdentityRuntime>, settings: &'a Settings, ec_state: &'a mut EcFinalizeState, /// Builds the identity graph. Called only when finalize or the after-send @@ -663,6 +681,9 @@ fn finalize_send_and_flush( None => log::warn!("withdrawal retry skipped: identity graph unavailable"), } } + let Some(runtime) = runtime else { + return Some(FlushOutcome::Skipped); + }; if !ec_state.is_real_browser || !may_flush(&ec_state.ec_context) { return Some(FlushOutcome::Skipped); } @@ -929,6 +950,12 @@ mod tests { Some("1:65536;2:0;4:6291456;6:262144"), )); request + .extensions_mut() + .insert(trusted_server_core::platform::ClientInfo { + client_ip: Some("192.0.2.10".parse().expect("should parse the client IP")), + ..trusted_server_core::platform::ClientInfo::default() + }); + request } /// Routes `request`, then finalizes, sends and flushes it as @@ -936,6 +963,17 @@ mod tests { fn drive_request_for_test( state: &Arc, request: HttpRequest, + ) -> (HttpResponse, Option) { + drive_with_runtime(state, request, Some(&state.identity), |_| {}) + } + + /// As [`drive_request_for_test`], with the runtime the post-route work + /// gets: `None` is the settings-reload branch without a runtime. + fn drive_with_runtime( + state: &Arc, + request: HttpRequest, + runtime: Option<&IdentityRuntime>, + prepare: impl FnOnce(&mut EcFinalizeState), ) -> (HttpResponse, Option) { let router = TrustedServerApp::routes_for_state(state); let mut response = @@ -943,11 +981,12 @@ mod tests { let Some(mut ec_state) = response.extensions_mut().remove::() else { return (response, None); }; + prepare(&mut ec_state); let mut sent = None; let graph = || state.identity_graph.clone(); let outcome = finalize_send_and_flush( PostRouteInput { - runtime: &state.identity, + runtime, settings: &state.settings, ec_state: &mut ec_state, graph: &graph, @@ -1013,7 +1052,7 @@ mod tests { let outcome = finalize_send_and_flush( PostRouteInput { - runtime: &state.identity, + runtime: Some(&state.identity), settings: &state.settings, ec_state: &mut ec_state, graph: &graph, @@ -1057,7 +1096,7 @@ mod tests { let outcome = finalize_send_and_flush( PostRouteInput { - runtime: &state.identity, + runtime: Some(&state.identity), settings: &state.settings, ec_state: &mut ec_state, graph: &graph, @@ -1092,6 +1131,73 @@ mod tests { 1, "should read the row whatever the old marker cookie says" ); + + let (withdrawn, _) = drive_with_runtime( + &state, + navigation(Some(&cookie)), + Some(&state.identity), + withdraw, + ); + let cookies = set_cookies(&withdrawn); + assert!( + cookies + .iter() + .any(|value| value.starts_with("ts-ec=;") && value.contains("Max-Age=0")), + "control: finalize should still expire ts-ec on a withdrawal: {cookies:?}" + ); + assert!( + cookies + .iter() + .all(|value| !value.starts_with("ts-ec-pull-complete")), + "should not expire a pull-complete cookie either: {cookies:?}" + ); + } + + /// Turns the request's consent into an explicit withdrawal (GPC in a US + /// state). Viceroy has no geo, so routing alone cannot produce one. + fn withdraw(ec_state: &mut EcFinalizeState) { + let consent = ec_state.ec_context.consent_mut(); + consent.jurisdiction = + trusted_server_core::consent::jurisdiction::Jurisdiction::UsState("CA".to_owned()); + consent.gpc = true; + } + + fn set_cookies(response: &HttpResponse) -> Vec { + response + .headers() + .get_all("set-cookie") + .iter() + .filter_map(|value| value.to_str().ok().map(str::to_owned)) + .collect() + } + + #[test] + fn reload_without_identity_runtime_still_finalizes() { + let (state, _) = seeded_state(); + + let (response, outcome) = drive_with_runtime( + &state, + navigation(Some(&format!("ts-ec={EC_ID}"))), + None, + withdraw, + ); + + let cookies = set_cookies(&response); + assert!( + cookies + .iter() + .any(|value| value.starts_with("ts-ec=;") && value.contains("Max-Age=0")), + "should still finalize and expire ts-ec without a runtime: {cookies:?}" + ); + let (row, _) = state + .identity_graph + .as_ref() + .expect("should hold the graph") + .get(EC_ID) + .expect("should read the row") + .expect("should keep the tombstone"); + assert!(!row.consent.ok, "should still tombstone the row"); + assert_eq!(outcome, Some(FlushOutcome::Skipped), "should skip phase 4"); } #[test] diff --git a/crates/trusted-server-core/src/ec/identity/mutation.rs b/crates/trusted-server-core/src/ec/identity/mutation.rs index 2fdc9444ae..c4de568201 100644 --- a/crates/trusted-server-core/src/ec/identity/mutation.rs +++ b/crates/trusted-server-core/src/ec/identity/mutation.rs @@ -291,12 +291,21 @@ impl IdentityMutation { /// Cuts the mutation down for a navigation whose origin status was not /// 2xx or 304 (spec 5.1). /// - /// Keeps every removal, opt-out and clear, the stand-alone state changes - /// (the `n` and `f` of every outcome other than `found`) and the cluster - /// size. Drops browser ingest and every other positive change, and each - /// provider result whole, keeping only its opt-out and clear. + /// Keeps every removal, opt-out and clear, the `n` and `f` of each + /// stand-alone state change, and the cluster size. Drops browser ingest, + /// every other positive change, any stand-alone `st`, and each provider + /// result whole, keeping only its opt-out and clear. + /// + /// A stand-alone state change carries the `n` and `f` of an outcome other + /// than a stateful `found`, whose scheduling rides in its result. A + /// stateless provider's `found` also lands here, so its `f = 0` is kept; + /// Part 1 dispatches no provider before the status is known, so no such + /// change reaches this cut yet. pub(crate) fn retain_for_error_status(&mut self) { self.partner_ids.retain(PartnerIdChange::is_removal); + for state in self.provider_state.values_mut() { + state.st = StateUpdate::Keep; + } for result in core::mem::take(&mut self.results) { if result.state.has_removal() { self.merge_state(result.provider, result.state.removals()); From a67dca7a408d8254f417e314f1a4dd8384546901 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 00:32:07 +0530 Subject: [PATCH 42/89] Test the one-write-per-page-load invariant end to end Drives each scenario through the router, finalize and phase 4 against a scripted 2xx origin and a counting store. Also pins the paths complete provider EIDs must not change, the ortb2-only EID reaching KV, entry sets read and served whole, version 1 rows upgraded on write, and a server that never reads the ts-eids-s mirror. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 65 +- .../trusted-server-adapter-fastly/src/main.rs | 2420 ++++++++++++++++- .../src/ec/identity/runtime.rs | 23 + .../lib/test/fixtures/ts-eids-golden.txt | 1 + .../lib/test/fixtures/ts-eids-ortb2-only.txt | 1 + .../ts-eids-registered-first-trim.txt | 1 + .../test/integrations/prebid/index.test.ts | 88 + 7 files changed, 2577 insertions(+), 22 deletions(-) create mode 100644 crates/trusted-server-js/lib/test/fixtures/ts-eids-golden.txt create mode 100644 crates/trusted-server-js/lib/test/fixtures/ts-eids-ortb2-only.txt create mode 100644 crates/trusted-server-js/lib/test/fixtures/ts-eids-registered-first-trim.txt diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 7732dac68f..cc103cd5fe 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -125,7 +125,8 @@ use trusted_server_core::integrations::{ RequestFilterRegistryOutcome, }; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformGeo, PlatformHttpClient, PlatformKvStore, RuntimeServices, + StoreName, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, @@ -196,6 +197,13 @@ pub(crate) struct AppState { /// The identity graph over `ec.ec_store`, built once. `None` when no /// store is configured. pub(crate) identity_graph: Option, + /// Replaces the Fastly HTTP client in tests, so a request reaches a + /// scripted origin and bidder instead of the network. + #[cfg(test)] + pub(crate) http_client_for_test: Option>, + /// Replaces the Fastly geolocation in tests, which Viceroy does not serve. + #[cfg(test)] + pub(crate) geo_for_test: Option>, } /// Build the application state, loading settings and constructing all per-application components. @@ -256,6 +264,10 @@ pub(crate) fn build_state_from_settings( identity_graph, auction_telemetry_sink, identity, + #[cfg(test)] + http_client_for_test: None, + #[cfg(test)] + geo_for_test: None, })) } @@ -273,6 +285,36 @@ pub(crate) fn app_state_with_graph_for_test( Arc::new(state) } +/// `state` with `identity` as its identity runtime, for a request that +/// reaches a provider no configuration can name. +#[cfg(test)] +pub(crate) fn app_state_with_identity_for_test( + state: Arc, + identity: IdentityRuntime, +) -> Arc { + let mut state = Arc::try_unwrap(state) + .unwrap_or_else(|_| unreachable!("a freshly built state should have one owner")); + state.identity = Arc::new(identity); + Arc::new(state) +} + +/// As [`app_state_with_graph_for_test`], with `client` answering every +/// outbound request the routes make and `geo` locating every client. +#[cfg(test)] +pub(crate) fn app_state_with_network_for_test( + settings: Settings, + graph: KvIdentityGraph, + client: Arc, + geo: Arc, +) -> Arc { + let state = app_state_with_graph_for_test(settings, graph); + let mut state = Arc::try_unwrap(state) + .unwrap_or_else(|_| unreachable!("a freshly built state should have one owner")); + state.http_client_for_test = Some(client); + state.geo_for_test = Some(geo); + Arc::new(state) +} + fn warn_if_certificate_check_disabled(settings: &Settings) { if !settings.proxy.certificate_check { log::warn!( @@ -297,6 +339,13 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { /// continues to rely on the trusted `fastly-ssl` header injected by /// `edgezero_main` after sanitization. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { + let http_client: Arc = Arc::new(FastlyPlatformHttpClient); + #[cfg(test)] + let http_client = state.http_client_for_test.clone().unwrap_or(http_client); + let geo: Arc = Arc::new(FastlyPlatformGeo); + #[cfg(test)] + let geo = state.geo_for_test.clone().unwrap_or(geo); + let client_info = ctx .request() .extensions() @@ -317,8 +366,8 @@ fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> Runtime .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) .backend(Arc::new(FastlyPlatformBackend)) - .http_client(Arc::new(FastlyPlatformHttpClient)) - .geo(Arc::new(FastlyPlatformGeo)) + .http_client(http_client) + .geo(geo) .auction_telemetry_sink(Arc::clone(&state.auction_telemetry_sink)) .client_info(client_info) .build() @@ -723,7 +772,13 @@ async fn run_named_route( if req.method() == Method::OPTIONS { cors_preflight_identify(&state.settings, &req) } else { - let kv = crate::require_identity_graph(&state.settings)?; + // The state's graph is the one over `ec.ec_store` that + // `require_identity_graph` would build, so the answer reads the + // same store the rest of the request counts against. + let kv = match state.identity_graph.clone() { + Some(graph) => graph, + None => crate::require_identity_graph(&state.settings)?, + }; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; handle_identify( &state.settings, @@ -1660,6 +1715,8 @@ mod tests { default_kv_store, identity, identity_graph: None, + http_client_for_test: None, + geo_for_test: None, }); TrustedServerApp::routes_for_state(&state) } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 98df4bf87c..53dd660974 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -894,9 +894,7 @@ mod tests { use fastly::mime; use trusted_server_core::integrations::HeaderMutation; - fn test_settings() -> Settings { - Settings::from_toml( - r#" + const TEST_SETTINGS_TOML: &str = r#" [[handlers]] path = "^/_ts/admin" username = "admin" @@ -915,9 +913,10 @@ mod tests { enabled = false config_store_id = "test-config-store-id" secret_store_id = "test-secret-store-id" - "#, - ) - .expect("should parse test settings") + "#; + + fn test_settings() -> Settings { + Settings::from_toml(TEST_SETTINGS_TOML).expect("should parse test settings") } const EC_ID: &str = "0000000000000000000000000000000000000000000000000000000000000000.ABC123"; @@ -931,18 +930,8 @@ mod tests { } } - /// A browser document navigation carrying `cookie`. - fn navigation(cookie: Option<&str>) -> HttpRequest { - let mut builder = edgezero_core::http::request_builder() - .method(edgezero_core::http::Method::GET) - .uri("https://test-publisher.com/some-page") - .header("sec-fetch-dest", "document"); - if let Some(cookie) = cookie { - builder = builder.header("cookie", cookie); - } - let mut request = builder - .body(EdgeBody::empty()) - .expect("should build request"); + /// Marks `request` as sent by a real browser at a client address. + fn as_browser(request: &mut HttpRequest) { request.extensions_mut().insert(DeviceSignals::derive( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", @@ -955,6 +944,21 @@ mod tests { client_ip: Some("192.0.2.10".parse().expect("should parse the client IP")), ..trusted_server_core::platform::ClientInfo::default() }); + } + + /// A browser document navigation carrying `cookie`. + fn navigation(cookie: Option<&str>) -> HttpRequest { + let mut builder = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::GET) + .uri("https://test-publisher.com/some-page") + .header("sec-fetch-dest", "document"); + if let Some(cookie) = cookie { + builder = builder.header("cookie", cookie); + } + let mut request = builder + .body(EdgeBody::empty()) + .expect("should build request"); + as_browser(&mut request); request } @@ -1587,4 +1591,2384 @@ mod tests { "should include sec-ch-ua-platform fallback" ); } + + /// End-to-end proof of the one-write invariant, the reads and G10 and + /// G13 (spec 13). Every test routes a request through the real router, + /// finalizes and sends it, and runs phase 4, against one counting store. + mod identity_writes { + use super::*; + use std::collections::BTreeMap; + use std::sync::Mutex; + + use trusted_server_core::ec::identity::test_support::{ + WriteCountingEcKv, allowing_consent, + }; + use trusted_server_core::ec::kv_backend::EcKvStore as _; + use trusted_server_core::ec::kv_types::{KvEntry, KvNetwork, KvPartnerId}; + use trusted_server_core::platform::{ + GeoInfo, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, + PlatformPendingRequest, PlatformResponse, PlatformSelectResult, + }; + + /// The host the publisher origin answers on. + const ORIGIN_HOST: &str = "origin.test-publisher.com"; + + /// A cookie value no test treats as a real ID. + const SHARED_ID: &str = "shared-id-1"; + + /// One answer of the scripted network. + #[derive(Clone)] + struct Reply { + status: u16, + headers: Vec<(String, String)>, + body: Vec, + } + + /// One request the scripted network received. + #[derive(Clone)] + struct Seen { + host: String, + headers: edgezero_core::http::HeaderMap, + body: Vec, + } + + struct Pending { + reply: Reply, + backend_name: String, + } + + /// A network that answers per host and records what it was sent. It + /// stands in for the publisher origin, the bidders and the pull + /// partners. + struct ScriptedNetwork { + replies: Mutex>, + seen: Mutex>, + } + + impl ScriptedNetwork { + fn new() -> Self { + Self { + replies: Mutex::new(BTreeMap::new()), + seen: Mutex::new(Vec::new()), + } + } + + fn respond(&self, host: &str, status: u16, content_type: &str, body: &str) { + self.respond_with_headers(host, status, &[("content-type", content_type)], body); + } + + fn respond_with_headers( + &self, + host: &str, + status: u16, + headers: &[(&str, &str)], + body: &str, + ) { + self.replies + .lock() + .expect("should lock the replies") + .insert( + host.to_owned(), + Reply { + status, + headers: headers + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(), + body: body.as_bytes().to_vec(), + }, + ); + } + + /// Every request that went to `host`, in order. + fn requests_to(&self, host: &str) -> Vec { + self.seen + .lock() + .expect("should lock the requests") + .iter() + .filter(|seen| seen.host == host) + .cloned() + .collect() + } + + /// Records `request` and returns its answer with the backend it named. + fn record( + &self, + request: PlatformHttpRequest, + ) -> Result<(Reply, String), Report> { + let backend_name = request.backend_name; + let (parts, body) = request.request.into_parts(); + let host = parts.uri.host().unwrap_or_default().to_owned(); + self.seen + .lock() + .expect("should lock the requests") + .push(Seen { + host: host.clone(), + headers: parts.headers, + body: body + .into_bytes() + .map(|bytes| bytes.to_vec()) + .unwrap_or_default(), + }); + self.replies + .lock() + .expect("should lock the replies") + .get(&host) + .cloned() + .map(|reply| (reply, backend_name)) + .ok_or_else(|| Report::new(PlatformError::HttpClient)) + } + } + + fn platform_response( + reply: Reply, + backend_name: String, + ) -> Result> { + let mut builder = edgezero_core::http::response_builder().status(reply.status); + for (name, value) in reply.headers { + builder = builder.header(name, value); + } + builder + .body(EdgeBody::from(reply.body)) + .map(|response| PlatformResponse::new(response).with_backend_name(backend_name)) + .map_err(|_| Report::new(PlatformError::HttpClient)) + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for ScriptedNetwork { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let (reply, backend_name) = self.record(request)?; + platform_response(reply, backend_name) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let (reply, backend_name) = self.record(request)?; + Ok(PlatformPendingRequest::new(Pending { + reply, + backend_name: backend_name.clone(), + }) + .with_backend_name(backend_name)) + } + + async fn select( + &self, + mut pending_requests: Vec, + ) -> Result> { + if pending_requests.is_empty() { + return Err(Report::new(PlatformError::HttpClient)); + } + let first = pending_requests.remove(0); + let ready = match first.downcast::() { + Ok(pending) => platform_response(pending.reply, pending.backend_name), + Err(_) => Err(Report::new(PlatformError::HttpClient)), + }; + Ok(PlatformSelectResult { + ready, + remaining: pending_requests, + failed_backend_name: None, + }) + } + } + + /// Locates every client in a country no privacy law of the default + /// consent config covers, so EC and EID consent is granted. + struct NonRegulatedGeo; + + impl PlatformGeo for NonRegulatedGeo { + fn lookup( + &self, + _client_ip: Option, + ) -> Result, Report> { + Ok(Some(GeoInfo { + city: "Example City".to_owned(), + country: "CA".to_owned(), + continent: "NA".to_owned(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: None, + })) + } + } + + /// What every scenario shares: the router state, the counting store + /// behind its identity graph and the scripted network. + struct Scenario { + state: Arc, + store: Arc, + network: Arc, + } + + /// The partner of `source`, registered for the bidstream with no pull. + fn partner_toml(source: &str) -> String { + format!( + r#" + [[ec.partners]] + name = "Partner {source}" + source_domain = "{source}" + bidstream_enabled = true + api_token = "token-for-{source}-0123456789abcdef" + "# + ) + } + + impl Scenario { + /// A scenario whose settings are the test settings plus `extra` + /// TOML, with `sharedid.org` and `id5-sync.com` registered. + fn with_settings(extra: &str) -> Self { + let toml = format!( + "{TEST_SETTINGS_TOML}{}{}{extra}", + partner_toml("sharedid.org"), + partner_toml("id5-sync.com"), + ); + Self::from_toml(&toml) + } + + fn new() -> Self { + Self::with_settings("") + } + + fn from_toml(toml: &str) -> Self { + let (graph, store) = KvIdentityGraph::write_counting("identity-writes-store"); + Self::from_parts(toml, graph, store) + } + + /// A scenario over `graph`, which must count into `store`. + fn from_parts( + toml: &str, + graph: KvIdentityGraph, + store: Arc, + ) -> Self { + let settings = Settings::from_toml(toml).expect("should parse the settings"); + let network = Arc::new(ScriptedNetwork::new()); + network.respond( + ORIGIN_HOST, + 200, + "text/html", + "page", + ); + let state = crate::app::app_state_with_network_for_test( + settings, + graph, + Arc::clone(&network) as Arc, + Arc::new(NonRegulatedGeo), + ); + Self { + state, + store, + network, + } + } + + fn graph(&self) -> &KvIdentityGraph { + self.state + .identity_graph + .as_ref() + .expect("should hold the graph") + } + + /// Stores `entry` under [`EC_ID`] and zeroes the counts. + fn seed(&self, entry: &KvEntry) { + self.graph() + .create(EC_ID, entry) + .expect("should seed the row"); + self.store.reset_counts(); + } + + /// Seeds a live row that already holds its cluster size, so the + /// flush has no change of its own to write (C2). + fn seed_live(&self) -> KvEntry { + let entry = live_entry(); + self.seed(&entry); + entry + } + + /// Drives `request`, and checks that no response of any scenario + /// sets a server identity cookie (spec 13). + fn drive(&self, request: HttpRequest) -> (HttpResponse, Option) { + let (response, outcome) = drive_request_for_test(&self.state, request); + assert_no_server_identity_cookie(&response); + (response, outcome) + } + + /// The row under `ec_id`. + fn row_of(&self, ec_id: &str) -> KvEntry { + self.graph() + .get(ec_id) + .expect("should read the row") + .expect("should hold the row") + .0 + } + + /// The row under [`EC_ID`]. + fn row(&self) -> KvEntry { + self.row_of(EC_ID) + } + } + + /// A live row for [`EC_ID`] with its cluster size present. + fn live_entry() -> KvEntry { + let mut entry = KvEntry::new(&allowing_consent(), None, 1_000, "test-publisher.com"); + entry.network = Some(KvNetwork { + cluster_size: Some(1), + }); + entry + } + + fn with_header(mut request: HttpRequest, name: &'static str, value: &str) -> HttpRequest { + request.headers_mut().insert( + name, + HeaderValue::from_str(value).expect("should encode the header"), + ); + request + } + + fn without_header(mut request: HttpRequest, name: &'static str) -> HttpRequest { + request.headers_mut().remove(name); + request + } + + /// The cookies every response of every scenario must leave unset: + /// the server sets `ts-ec` and no other identity cookie. + fn is_server_identity_cookie(name: &str) -> bool { + name == "ts-eids" + || name == "__Host-ts-eids-s" + || name.starts_with("ts-eids-r") + || name == "ts-ec-pull-complete" + } + + fn cookie_name(set_cookie: &str) -> &str { + set_cookie.split('=').next().unwrap_or_default().trim() + } + + fn assert_no_server_identity_cookie(response: &HttpResponse) { + for cookie in set_cookies(response) { + assert!( + !is_server_identity_cookie(cookie_name(&cookie)), + "should set no identity cookie but ts-ec: {cookie}" + ); + } + } + + /// The value of the `name` cookie the response sets. + fn set_cookie_value(response: &HttpResponse, name: &str) -> Option { + set_cookies(response).into_iter().find_map(|cookie| { + let pair = cookie.split(';').next()?; + let (found, value) = pair.split_once('=')?; + (found.trim() == name).then(|| value.trim().to_owned()) + }) + } + + /// The cookie of a returning visitor whose browser holds a sharedId. + fn returning_with_shared_id() -> String { + format!("ts-ec={EC_ID}; sharedId={SHARED_ID}") + } + + /// Asserts every write mode count, so no mode hides behind another. + fn assert_writes(store: &WriteCountingEcKv, expected: (usize, usize, usize), what: &str) { + let writes = store.writes(); + assert_eq!( + (writes.add, writes.if_generation_match, writes.overwrite), + expected, + "should make exactly (Add, IfGenerationMatch, Overwrite) writes: {what}" + ); + } + + fn assert_no_writes(store: &WriteCountingEcKv, what: &str) { + assert_writes(store, (0, 0, 0), what); + } + + /// The UID the row holds for `source`. + fn stored_uid(row: &KvEntry, source: &str) -> Option { + row.ids + .get(source) + .and_then(KvPartnerId::first_uid) + .map(str::to_owned) + } + + #[test] + fn returning_visitor_with_changes_writes_once() { + let scenario = Scenario::new(); + scenario.seed_live(); + + let (response, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!(response.status(), 200, "should serve the origin page"); + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush"); + assert_writes(&scenario.store, (0, 1, 0), "one CAS write"); + assert_eq!( + stored_uid(&scenario.row(), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should hold the browser's sharedId in KV after the one write" + ); + } + + /// A store that refuses its first `collisions` `Add` writes without + /// storing them, as a minted ID that is already taken does. + struct CollidingStore { + inner: Arc, + collisions: Mutex, + refused: Arc, + } + + impl trusted_server_core::ec::kv_backend::EcKvStore for CollidingStore { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup( + &self, + key: &str, + ) -> Result< + Option, + Report, + > { + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: trusted_server_core::ec::kv_backend::EcKvWrite<'_>, + ) -> Result< + trusted_server_core::ec::kv_backend::EcKvWriteOutcome, + Report, + > { + if matches!( + write.mode, + trusted_server_core::ec::kv_backend::EcKvWriteMode::Add + ) { + let mut remaining = self.collisions.lock().expect("should lock the count"); + if *remaining > 0 { + *remaining -= 1; + self.refused + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + return Ok( + trusted_server_core::ec::kv_backend::EcKvWriteOutcome::PreconditionFailed, + ); + } + } + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + + /// The same navigation with one header replaced. + fn navigation_with_header( + cookie: Option<&str>, + name: &'static str, + value: &str, + ) -> HttpRequest { + with_header(navigation(cookie), name, value) + } + + #[test] + fn new_visitor_navigation_writes_once() { + let scenario = Scenario::new(); + let cookie = format!("sharedId={SHARED_ID}"); + + let (response, outcome) = scenario.drive(navigation(Some(&cookie))); + + assert_eq!(response.status(), 200, "should serve the origin page"); + let minted = set_cookie_value(&response, "ts-ec").expect("should mint an EC"); + assert_eq!( + outcome, + Some(FlushOutcome::Skipped), + "should leave a new visitor nothing to flush" + ); + assert_writes(&scenario.store, (1, 0, 0), "one Add and no flush"); + assert_eq!( + stored_uid(&scenario.row_of(&minted), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should carry the browser ingest in the create" + ); + } + + #[test] + fn new_visitor_collision_remints_and_writes_once() { + let (_, counting) = KvIdentityGraph::write_counting("collision-store"); + let refused = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::new(CollidingStore { + inner: Arc::clone(&counting), + collisions: Mutex::new(1), + refused: Arc::clone(&refused), + }); + let toml = format!("{TEST_SETTINGS_TOML}{}", partner_toml("sharedid.org")); + let scenario = Scenario::from_parts(&toml, graph, counting); + let cookie = format!("sharedId={SHARED_ID}"); + + let (response, _) = scenario.drive(navigation(Some(&cookie))); + + assert_eq!( + refused.load(std::sync::atomic::Ordering::SeqCst), + 1, + "control: should have met one taken ID first" + ); + assert_writes( + &scenario.store, + (1, 0, 0), + "one Add lands after the collision", + ); + let minted: Vec = set_cookies(&response) + .into_iter() + .filter(|cookie| cookie_name(cookie) == "ts-ec") + .collect(); + assert_eq!( + minted.len(), + 1, + "should set one ts-ec, for the ID that landed" + ); + let id = set_cookie_value(&response, "ts-ec").expect("should mint an EC"); + assert_eq!( + stored_uid(&scenario.row_of(&id), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should hold the ingest under the re-minted ID" + ); + } + + #[test] + fn returning_visitor_without_changes_writes_nothing() { + let scenario = Scenario::new(); + let mut entry = live_entry(); + entry.ids.insert( + "sharedid.org".to_owned(), + KvPartnerId::browser(SHARED_ID, Some(1_500)), + ); + scenario.seed(&entry); + + let (response, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!(response.status(), 200, "should serve the origin page"); + assert_eq!( + outcome, + Some(FlushOutcome::NoChange), + "should find the row already holds the ID" + ); + assert_no_writes(&scenario.store, "the browser's ID is already stored"); + + let (_, nothing) = scenario.drive(navigation(Some(&format!("ts-ec={EC_ID}")))); + assert_eq!( + nothing, + Some(FlushOutcome::Skipped), + "should skip a navigation that carries nothing" + ); + assert_no_writes(&scenario.store, "nothing to ingest"); + + let control = Scenario::new(); + control.seed_live(); + control.drive(navigation(Some(&returning_with_shared_id()))); + assert_writes( + &control.store, + (0, 1, 0), + "control: the same request writes when the row lacks the ID", + ); + } + + #[test] + fn revalidation_304_with_changes_writes_once() { + let scenario = Scenario::new(); + scenario.seed_live(); + scenario + .network + .respond_with_headers(ORIGIN_HOST, 304, &[("etag", "\"page-1\"")], ""); + let request = navigation_with_header( + Some(&returning_with_shared_id()), + "if-none-match", + "\"page-1\"", + ); + + let (response, outcome) = scenario.drive(request); + + assert_eq!(response.status(), 304, "should answer the revalidation"); + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush"); + assert_writes(&scenario.store, (0, 1, 0), "one CAS write"); + assert_eq!( + stored_uid(&scenario.row(), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should hold the sharedId" + ); + } + + #[test] + fn returning_navigation_answered_3xx_4xx_5xx_writes_nothing() { + let control = Scenario::new(); + control.seed_live(); + control.drive(navigation(Some(&returning_with_shared_id()))); + assert_writes(&control.store, (0, 1, 0), "control: a 200 writes once"); + + for status in [301_u16, 302, 400, 404, 410, 500, 502, 503] { + let scenario = Scenario::new(); + scenario.seed_live(); + scenario.network.respond_with_headers( + ORIGIN_HOST, + status, + &[ + ("content-type", "text/html"), + ("location", "https://origin.test-publisher.com/next"), + ], + "answer", + ); + + let (response, outcome) = + scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + response.status().as_u16(), + status, + "should pass the origin status through: {status}" + ); + assert_eq!( + outcome, + Some(FlushOutcome::Skipped), + "should have nothing to flush after {status}" + ); + assert_no_writes(&scenario.store, &format!("origin answered {status}")); + assert_eq!( + stored_uid(&scenario.row(), "sharedid.org"), + None, + "should not store the ID the browser sent, after {status}" + ); + } + } + + #[test] + fn new_visitor_prefetch_writes_once() { + let scenario = Scenario::new(); + + let (response, _) = scenario.drive(navigation_with_header( + Some(&format!("sharedId={SHARED_ID}")), + "sec-purpose", + "prefetch", + )); + + assert!( + set_cookie_value(&response, "ts-ec").is_some(), + "should mint an EC for a prefetched navigation" + ); + assert_writes(&scenario.store, (1, 0, 0), "one Add"); + } + + #[test] + fn returning_prefetch_writes_nothing() { + for header in ["sec-purpose", "purpose"] { + let scenario = Scenario::new(); + scenario.seed_live(); + + let (_, outcome) = scenario.drive(navigation_with_header( + Some(&returning_with_shared_id()), + header, + "prefetch", + )); + + assert_eq!( + outcome, + Some(FlushOutcome::Skipped), + "should not be the writer: {header}" + ); + assert_no_writes(&scenario.store, header); + } + + let control = Scenario::new(); + control.seed_live(); + control.drive(navigation(Some(&returning_with_shared_id()))); + assert_writes( + &control.store, + (0, 1, 0), + "control: the same navigation without prefetch writes once", + ); + } + + #[test] + fn new_visitor_without_sec_fetch_dest_writes_once() { + let scenario = Scenario::new(); + let request = with_header( + without_header( + navigation(Some(&format!("sharedId={SHARED_ID}"))), + "sec-fetch-dest", + ), + "accept", + "text/html,application/xhtml+xml", + ); + + let (response, _) = scenario.drive(request); + + assert!( + set_cookie_value(&response, "ts-ec").is_some(), + "should mint an EC" + ); + assert_writes(&scenario.store, (1, 0, 0), "one Add"); + } + + #[test] + fn returning_iframe_or_no_sec_fetch_dest_writes_nothing() { + let requests = [ + ( + "iframe", + navigation_with_header( + Some(&returning_with_shared_id()), + "sec-fetch-dest", + "iframe", + ), + ), + ( + "no Sec-Fetch-Dest", + with_header( + without_header( + navigation(Some(&returning_with_shared_id())), + "sec-fetch-dest", + ), + "accept", + "text/html", + ), + ), + ]; + for (name, request) in requests { + let scenario = Scenario::new(); + scenario.seed_live(); + + let (_, outcome) = scenario.drive(request); + + assert_eq!( + outcome, + Some(FlushOutcome::Skipped), + "should not be the writer: {name}" + ); + assert_no_writes(&scenario.store, name); + } + + let control = Scenario::new(); + control.seed_live(); + control.drive(navigation(Some(&returning_with_shared_id()))); + assert_writes( + &control.store, + (0, 1, 0), + "control: a document navigation writes once", + ); + } + + /// The settings of an enabled Didomi proxy, whose upstream answers on + /// `sdk.example.com`. + const DIDOMI_TOML: &str = r#" + [integrations.didomi] + enabled = true + sdk_origin = "https://sdk.example.com" + api_origin = "https://api.example.com" + "#; + + fn didomi_scenario() -> Scenario { + let scenario = Scenario::with_settings(DIDOMI_TOML); + scenario + .network + .respond("sdk.example.com", 200, "application/javascript", "loader()"); + scenario + } + + fn didomi_navigation(cookie: Option<&str>) -> HttpRequest { + let mut request = navigation(cookie); + *request.uri_mut() = "https://test-publisher.com/integrations/didomi/consent/loader.js" + .parse() + .expect("should parse the proxy URI"); + request + } + + #[test] + fn integration_proxy_navigation_minting_ec_writes_once() { + let scenario = didomi_scenario(); + + let (response, _) = + scenario.drive(didomi_navigation(Some(&format!("sharedId={SHARED_ID}")))); + + assert_eq!(response.status(), 200, "should serve the proxied answer"); + assert_eq!( + scenario.network.requests_to("sdk.example.com").len(), + 1, + "control: should have reached the integration, not the publisher" + ); + assert!( + scenario.network.requests_to(ORIGIN_HOST).is_empty(), + "should not reach the publisher origin" + ); + let minted = set_cookie_value(&response, "ts-ec").expect("should mint an EC"); + assert_writes(&scenario.store, (1, 0, 0), "one Add"); + assert_eq!( + stored_uid(&scenario.row_of(&minted), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should carry the ingest in the create" + ); + } + + /// An enabled auction with one bidder, and one slot for page bids. + const AUCTION_TOML: &str = r#" + [auction] + enabled = true + timeout_ms = 2000 + + [auction.providers.bidder-main] + protocol = "openrtb-2.6" + profile = "standard" + endpoint = "https://bidder.example.com/auction" + routing = "all_eligible" + + [creative_opportunities] + enabled = true + gam_network_id = "123456789" + + [[creative_opportunities.slot]] + id = "leaderboard" + gam_unit_path = "/123456789/leaderboard" + div_id = "div-gpt-ad-leaderboard" + page_patterns = ["/"] + formats = [{ width = 728, height = 90 }] + "#; + + /// A first-party asset route to a scripted origin. + const ASSET_TOML: &str = r#" + [[proxy.asset_routes]] + prefix = "/assets/" + origin_url = "https://assets.example.com" + "#; + + /// A pull partner whose answers the scripted network serves. + const PULL_PARTNER_TOML: &str = r#" + [[ec.partners]] + name = "Pull SSP" + source_domain = "ssp.example.com" + bidstream_enabled = true + pull_sync_enabled = true + pull_sync_url = "https://sync.ssp.example.com/pull" + pull_sync_allowed_domains = ["sync.ssp.example.com"] + ts_pull_token = "example-token-value-0123456789abcdef" + "#; + + /// The provider the pull partner becomes. + const PULL_PROVIDER: &str = "pull-ssp-example-com"; + + const BIDDER_HOST: &str = "bidder.example.com"; + const PULL_HOST: &str = "sync.ssp.example.com"; + + /// A scenario with the auction, the Didomi proxy, an asset route and + /// the pull partner, and a network that answers all of them. + fn full_scenario() -> Scenario { + let scenario = Scenario::with_settings(&format!( + "{AUCTION_TOML}{DIDOMI_TOML}{ASSET_TOML}{PULL_PARTNER_TOML}" + )); + scenario.network.respond( + BIDDER_HOST, + 200, + "application/json", + r#"{"id":"response","seatbid":[]}"#, + ); + scenario + .network + .respond("sdk.example.com", 200, "application/javascript", "loader()"); + scenario.network.respond( + "assets.example.com", + 200, + "application/javascript", + "asset()", + ); + scenario.network.respond( + PULL_HOST, + 200, + "application/json", + r#"{"uid":"pull-uid-1"}"#, + ); + scenario + } + + /// A scenario with the auction and no pull partner. + fn auction_scenario() -> Scenario { + let scenario = Scenario::with_settings(AUCTION_TOML); + scenario.network.respond( + BIDDER_HOST, + 200, + "application/json", + r#"{"id":"response","seatbid":[]}"#, + ); + scenario + } + + fn auction_body(eids: Option<&serde_json::Value>) -> serde_json::Value { + let mut body = serde_json::json!({ + "adUnits": [{ + "code": "div-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + if let Some(eids) = eids { + body["eids"] = eids.clone(); + } + body + } + + /// A `/auction` request from a real browser. + fn auction_request(cookie: Option<&str>, body: &serde_json::Value) -> HttpRequest { + let mut builder = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::POST) + .uri("https://test-publisher.com/auction") + .header("content-type", "application/json"); + if let Some(cookie) = cookie { + builder = builder.header("cookie", cookie); + } + let mut request = builder + .body(EdgeBody::from( + serde_json::to_vec(body).expect("should serialize the body"), + )) + .expect("should build request"); + as_browser(&mut request); + request + } + + /// A browser's `/_ts/page-bids` call for the home page. + fn page_bids_request(cookie: &str) -> HttpRequest { + let mut request = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::GET) + .uri("https://test-publisher.com/_ts/page-bids?path=%2F") + .header("x-tsjs-page-bids", "1") + .header("cookie", cookie) + .body(EdgeBody::empty()) + .expect("should build request"); + as_browser(&mut request); + request + } + + /// The partner `source` asking who the visitor is. + fn identify_request(cookie: &str, source: &str) -> HttpRequest { + let mut request = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::GET) + .uri("https://test-publisher.com/_ts/api/v1/identify") + .header( + "authorization", + format!("Bearer token-for-{source}-0123456789abcdef"), + ) + .header("cookie", cookie) + .body(EdgeBody::empty()) + .expect("should build request"); + as_browser(&mut request); + request + } + + fn asset_request(cookie: &str) -> HttpRequest { + let mut request = edgezero_core::http::request_builder() + .method(edgezero_core::http::Method::GET) + .uri("https://test-publisher.com/assets/app.js") + .header("sec-fetch-dest", "script") + .header("cookie", cookie) + .body(EdgeBody::empty()) + .expect("should build request"); + as_browser(&mut request); + request + } + + fn json_of(response: HttpResponse) -> serde_json::Value { + let bytes = response + .into_body() + .into_bytes() + .expect("should buffer the body"); + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null) + } + + /// The last request body the bidder received. + fn bidder_body(scenario: &Scenario) -> serde_json::Value { + let seen = scenario + .network + .requests_to(BIDDER_HOST) + .pop() + .expect("should have called the bidder"); + serde_json::from_slice(&seen.body).expect("should parse the bidder request") + } + + /// The EIDs of a bidder request, from `user.ext.eids`. + fn bidder_eids(body: &serde_json::Value) -> serde_json::Value { + body["user"]["ext"]["eids"].clone() + } + + /// What `x-ts-eids` carries. + fn header_eids(response: &HttpResponse) -> serde_json::Value { + use base64::Engine as _; + let value = response + .headers() + .get("x-ts-eids") + .and_then(|value| value.to_str().ok()) + .expect("should carry x-ts-eids"); + let bytes = base64::engine::general_purpose::STANDARD + .decode(value) + .expect("should decode x-ts-eids"); + serde_json::from_slice(&bytes).expect("should parse x-ts-eids") + } + + /// The decoded `ts-eids-s` payload of an `/auction` body, or its + /// `clear` marker. + fn mirror_of(body: &serde_json::Value) -> serde_json::Value { + use base64::Engine as _; + let member = &body["ext"]["ts_eids_s"]; + match member["value"].as_str() { + Some(value) => { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(value) + .expect("should decode the mirror"); + serde_json::from_slice(&bytes).expect("should parse the mirror") + } + None => member.clone(), + } + } + + #[test] + fn auction_page_bids_identify_proxy_and_assets_write_nothing() { + let scenario = full_scenario(); + scenario.seed_live(); + let cookie = returning_with_shared_id(); + let body = auction_body(None); + + let requests = [ + ("auction", auction_request(Some(&cookie), &body)), + ("page bids", page_bids_request(&cookie)), + ("identify", identify_request(&cookie, "sharedid.org")), + ("integration proxy subresource", { + let mut request = didomi_navigation(Some(&cookie)); + request + .headers_mut() + .insert("sec-fetch-dest", HeaderValue::from_static("script")); + request + }), + ("asset", asset_request(&cookie)), + ]; + for (name, request) in requests { + let (response, _) = scenario.drive(request); + + assert!( + response.status().is_success(), + "should answer {name} with a success, got {}", + response.status() + ); + assert_no_writes(&scenario.store, name); + } + assert_eq!( + scenario.network.requests_to(BIDDER_HOST).len(), + 2, + "control: the auction and the page bids should both have reached the bidder" + ); + for host in ["sdk.example.com", "assets.example.com"] { + assert_eq!( + scenario.network.requests_to(host).len(), + 1, + "control: the proxy and the asset route should each have reached {host}" + ); + } + + let control = full_scenario(); + control.seed_live(); + control.drive(navigation(Some(&cookie))); + assert_writes( + &control.store, + (0, 1, 0), + "control: a navigation with the same cookies writes once", + ); + } + + #[test] + fn returning_writer_navigation_reads_once() { + let changed = Scenario::new(); + changed.seed_live(); + + changed.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + changed.store.writes().lookups, + 1, + "should read once, with a write" + ); + assert_writes(&changed.store, (0, 1, 0), "one CAS write"); + + let unchanged = Scenario::new(); + let mut entry = live_entry(); + entry.ids.insert( + "sharedid.org".to_owned(), + KvPartnerId::browser(SHARED_ID, Some(1_500)), + ); + unchanged.seed(&entry); + + unchanged.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + unchanged.store.writes().lookups, + 1, + "should read once, with nothing to write" + ); + assert_no_writes(&unchanged.store, "the row already holds the ID"); + + unchanged.drive(navigation(Some(&returning_with_shared_id()))); + assert_eq!( + unchanged.store.writes().lookups, + 2, + "control: the counter should count each navigation's read" + ); + } + + #[test] + fn due_after_send_provider_adds_one_reread() { + let scenario = full_scenario(); + scenario.seed_live(); + + let (_, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush"); + assert_eq!( + scenario.network.requests_to(PULL_HOST).len(), + 1, + "should call the due pull provider after send" + ); + assert_eq!( + scenario.store.writes().lookups, + 2, + "should read in phase 1 and once more before dispatching" + ); + assert_writes( + &scenario.store, + (0, 1, 0), + "the browser ingest and the pull in one write", + ); + let row = scenario.row(); + assert_eq!( + stored_uid(&row, "sharedid.org").as_deref(), + Some(SHARED_ID), + "should hold the browser ID" + ); + assert_eq!( + stored_uid(&row, "ssp.example.com").as_deref(), + Some("pull-uid-1"), + "should hold the pulled ID" + ); + + let idle = full_scenario(); + let mut entry = live_entry(); + entry.ps.insert( + PULL_PROVIDER.to_owned(), + trusted_server_core::ec::kv_types::KvProviderState { + n: Some(4_000_000_000), + ..Default::default() + }, + ); + idle.seed(&entry); + + idle.drive(navigation(Some(&returning_with_shared_id()))); + + assert!( + idle.network.requests_to(PULL_HOST).is_empty(), + "control: should not call a provider that is not due" + ); + assert_eq!( + idle.store.writes().lookups, + 1, + "control: should read once when no provider is due" + ); + } + + #[test] + fn auction_and_page_bids_keep_one_read() { + let cookie = returning_with_shared_id(); + let auction = full_scenario(); + auction.seed_live(); + + auction.drive(auction_request(Some(&cookie), &auction_body(None))); + + assert_eq!( + auction.store.writes().lookups, + 1, + "/auction should read once" + ); + assert_no_writes(&auction.store, "/auction"); + assert_eq!( + auction.network.requests_to(BIDDER_HOST).len(), + 1, + "control: the auction should have run" + ); + + let page_bids = full_scenario(); + page_bids.seed_live(); + + page_bids.drive(page_bids_request(&cookie)); + + assert_eq!( + page_bids.store.writes().lookups, + 1, + "/_ts/page-bids should read once" + ); + assert_no_writes(&page_bids.store, "/_ts/page-bids"); + assert_eq!( + page_bids.network.requests_to(BIDDER_HOST).len(), + 1, + "control: the page-bids auction should have run" + ); + } + + #[test] + fn no_response_sets_an_identity_cookie_other_than_ts_ec() { + assert!( + is_server_identity_cookie("ts-eids") + && is_server_identity_cookie("__Host-ts-eids-s") + && is_server_identity_cookie("ts-eids-r1") + && is_server_identity_cookie("ts-ec-pull-complete") + && !is_server_identity_cookie("ts-ec"), + "control: the check should name exactly the cookies the server never sets" + ); + let browser_cookies = format!( + "{}; ts-eids=ignored; __Host-ts-eids-s=ignored; ts-eids-r1=ignored; \ + ts-ec-pull-complete=v1.example", + returning_with_shared_id() + ); + + let new_visitor = full_scenario(); + let (minted, _) = new_visitor.drive(navigation(Some("sharedId=shared-id-1"))); + assert!( + set_cookie_value(&minted, "ts-ec").is_some(), + "control: a new visitor's response should set ts-ec" + ); + + let returning = full_scenario(); + returning.seed_live(); + let body = auction_body(None); + for request in [ + navigation(Some(&browser_cookies)), + auction_request(Some(&browser_cookies), &body), + page_bids_request(&browser_cookies), + identify_request(&browser_cookies, "sharedid.org"), + ] { + let (response, _) = returning.drive(request); + let names: Vec = set_cookies(&response) + .iter() + .map(|cookie| cookie_name(cookie).to_owned()) + .collect(); + assert!( + names.iter().all(|name| name == "ts-ec"), + "should set at most ts-ec: {names:?}" + ); + } + } + + /// A source `id5-sync.com` entry set with two entries, three UIDs and + /// every UID-level and EID-level field. + fn full_entry_set() -> Vec { + use trusted_server_core::ec::kv_types::{KvEid, KvUid}; + vec![ + KvEid { + uids: vec![ + KvUid { + id: "set-uid-a".to_owned(), + atype: Some(1), + ext: Some(serde_json::json!({ "rtiPartner": "A" })), + }, + KvUid { + id: "set-uid-b".to_owned(), + atype: Some(3), + ext: Some(serde_json::json!({ "stype": "ppuid" })), + }, + ], + inserter: Some("inserter-a.example.com".to_owned()), + matcher: Some("matcher-a.example.com".to_owned()), + mm: Some(3), + ext: Some(serde_json::json!({ "segments": ["s1", "s2"] })), + }, + KvEid { + uids: vec![KvUid { + id: "set-uid-c".to_owned(), + atype: Some(1), + ext: Some(serde_json::json!({ "label": "c" })), + }], + inserter: Some("inserter-b.example.com".to_owned()), + matcher: Some("matcher-b.example.com".to_owned()), + mm: Some(1), + ext: Some(serde_json::json!({ "tier": "gold" })), + }, + ] + } + + /// What bidders receive for [`full_entry_set`]. + fn full_entry_set_eids() -> serde_json::Value { + serde_json::json!([ + { + "source": "id5-sync.com", + "inserter": "inserter-a.example.com", + "matcher": "matcher-a.example.com", + "mm": 3, + "ext": { "segments": ["s1", "s2"] }, + "uids": [ + { "id": "set-uid-a", "atype": 1, "ext": { "rtiPartner": "A" } }, + { "id": "set-uid-b", "atype": 3, "ext": { "stype": "ppuid" } } + ] + }, + { + "source": "id5-sync.com", + "inserter": "inserter-b.example.com", + "matcher": "matcher-b.example.com", + "mm": 1, + "ext": { "tier": "gold" }, + "uids": [{ "id": "set-uid-c", "atype": 1, "ext": { "label": "c" } }] + } + ]) + } + + /// A live row holding `eids` for `id5-sync.com`, with no writer. + fn row_with_set(exp: Option) -> KvEntry { + let mut entry = live_entry(); + entry.ids.insert( + "id5-sync.com".to_owned(), + KvPartnerId { + at: Some(1_500), + exp, + eids: full_entry_set(), + ..KvPartnerId::default() + }, + ); + entry + } + + /// A browser entry of a source no partner registers. + fn browser_entry() -> serde_json::Value { + serde_json::json!({ + "source": "browser.example.com", + "uids": [{ "id": "browser-uid", "atype": 1 }] + }) + } + + const FAR_FUTURE: u64 = 4_000_000_000; + + fn ec_cookie() -> String { + format!("ts-ec={EC_ID}") + } + + #[test] + fn entry_set_round_trips_from_kv_to_bidders() { + let scenario = auction_scenario(); + scenario.seed(&row_with_set(Some(FAR_FUTURE))); + let body = auction_body(Some(&serde_json::json!([browser_entry()]))); + + let (response, _) = scenario.drive(auction_request(Some(&ec_cookie()), &body)); + + let mut expected = vec![browser_entry()]; + expected.extend( + full_entry_set_eids() + .as_array() + .expect("should be a list") + .clone(), + ); + let expected = serde_json::Value::Array(expected); + assert_eq!( + bidder_eids(&bidder_body(&scenario)), + expected, + "should send the browser entry, then both stored entries in order, every field intact" + ); + assert_eq!( + header_eids(&response), + expected, + "x-ts-eids should carry what was sent" + ); + let mirror = mirror_of(&json_of(response)); + assert_eq!( + mirror, + serde_json::json!({ "v": 1, "eids": full_entry_set_eids() }), + "the mirror should hold the same stored entries and no browser entry" + ); + assert_no_writes(&scenario.store, "/auction"); + } + + #[test] + fn entry_set_survives_an_unrelated_flush() { + let scenario = Scenario::new(); + let seeded = row_with_set(Some(FAR_FUTURE)); + scenario.seed(&seeded); + + let (_, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + outcome, + Some(FlushOutcome::Written), + "should flush the sharedId" + ); + assert_writes(&scenario.store, (0, 1, 0), "one CAS write"); + let row = scenario.row(); + assert_eq!( + stored_uid(&row, "sharedid.org").as_deref(), + Some(SHARED_ID), + "control: the flush should have written the sharedId fill" + ); + assert_eq!( + row.ids.get("id5-sync.com"), + seeded.ids.get("id5-sync.com"), + "should leave the stored entry set exactly as it was" + ); + } + + #[test] + fn expired_entry_set_is_never_served_or_mirrored() { + let live = auction_scenario(); + live.seed(&row_with_set(Some(FAR_FUTURE))); + let (live_response, _) = + live.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + assert_eq!( + bidder_eids(&bidder_body(&live)), + full_entry_set_eids(), + "control: an unexpired set should be served" + ); + assert_ne!( + mirror_of(&json_of(live_response)), + serde_json::json!({ "clear": true }), + "control: an unexpired set should be mirrored" + ); + + let expired = auction_scenario(); + expired.seed(&row_with_set(Some(1_500))); + let (response, _) = + expired.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + + assert_eq!( + bidder_eids(&bidder_body(&expired)), + serde_json::Value::Null, + "should send no EIDs from an expired set" + ); + assert_eq!( + mirror_of(&json_of(response)), + serde_json::json!({ "clear": true }), + "should mirror nothing for an expired set" + ); + } + + /// A version 1 row: `uid` records and no entry sets. + fn seed_version_1_row(scenario: &Scenario) { + scenario.seed(&live_entry()); + let lookup = scenario + .store + .lookup(EC_ID) + .expect("should read the row") + .expect("should hold the row"); + let body = serde_json::json!({ + "v": 1, + "created": 1_000, + "consent": { "ok": true, "updated": 1_000 }, + "geo": { "country": "CA" }, + "network": { "cluster_size": 1 }, + "ids": { "id5-sync.com": { "uid": "version-1-uid", "at": 1_500 } } + }) + .to_string(); + let metadata = lookup + .metadata + .map(|bytes| String::from_utf8(bytes).expect("should be UTF-8")) + .unwrap_or_else(|| "{}".to_owned()); + scenario + .store + .insert( + EC_ID, + trusted_server_core::ec::kv_backend::EcKvWrite { + body: &body, + metadata: &metadata, + ttl: std::time::Duration::from_secs(3_600), + mode: trusted_server_core::ec::kv_backend::EcKvWriteMode::Overwrite, + }, + ) + .expect("should store the version 1 row"); + scenario.store.reset_counts(); + } + + fn stored_body(scenario: &Scenario) -> serde_json::Value { + let lookup = scenario + .store + .lookup(EC_ID) + .expect("should read the row") + .expect("should hold the row"); + serde_json::from_slice(&lookup.body).expect("should parse the stored row") + } + + #[test] + fn version_1_row_is_served_and_upgraded_by_the_next_write() { + let scenario = auction_scenario(); + seed_version_1_row(&scenario); + + scenario.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + + assert_eq!( + bidder_eids(&bidder_body(&scenario)), + serde_json::json!([{ + "source": "id5-sync.com", + "uids": [{ + "id": "version-1-uid", + "atype": trusted_server_core::settings::EcPartner::default_openrtb_atype() + }] + }]), + "should serve the version 1 record as one entry with one UID" + ); + assert_no_writes(&scenario.store, "/auction"); + assert_eq!( + stored_body(&scenario)["v"], + 1, + "control: the auction should leave the row at version 1" + ); + + let (_, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + outcome, + Some(FlushOutcome::Written), + "should flush the sharedId" + ); + assert_writes(&scenario.store, (0, 1, 0), "one CAS write"); + let written = stored_body(&scenario); + assert_eq!(written["v"], 2, "should write version 2"); + let record = &written["ids"]["id5-sync.com"]; + assert!( + record.get("uid").is_none(), + "should drop the version 1 uid field: {record}" + ); + assert_eq!( + record["eids"][0]["uids"][0]["id"], "version-1-uid", + "should keep the UID as an entry" + ); + } + + #[test] + fn generic_pull_uid_written_after_send_is_served_on_the_next_auction() { + let scenario = full_scenario(); + scenario.seed_live(); + + let (_, outcome) = scenario.drive(navigation(Some(&ec_cookie()))); + + assert_eq!( + outcome, + Some(FlushOutcome::Written), + "should flush the pull" + ); + let record = scenario.row().ids.get("ssp.example.com").cloned(); + assert_eq!( + record.as_ref().and_then(|record| record.w.as_deref()), + Some(PULL_PROVIDER), + "should record the pull provider as the writer" + ); + + let (response, _) = + scenario.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + + let atype = trusted_server_core::settings::EcPartner::default_openrtb_atype(); + let served = serde_json::json!([{ + "source": "ssp.example.com", + "uids": [{ "id": "pull-uid-1", "atype": atype }] + }]); + assert_eq!( + bidder_eids(&bidder_body(&scenario)), + served, + "should serve the pulled UID to bidders" + ); + assert_eq!( + mirror_of(&json_of(response)), + serde_json::json!({ "v": 1, "eids": served }), + "should mirror the pulled UID" + ); + } + + /// As [`drive_with_runtime`], running `after_send` between the send + /// and phase 4, where another request can land. + fn drive_with_send_hook( + state: &Arc, + request: HttpRequest, + after_send: &mut dyn FnMut(), + ) -> Option { + let router = TrustedServerApp::routes_for_state(state); + let mut response = futures::executor::block_on(router.oneshot(request)) + .expect("should route the request"); + let mut ec_state = response + .extensions_mut() + .remove::() + .expect("should carry EC state"); + let graph = || state.identity_graph.clone(); + finalize_send_and_flush( + PostRouteInput { + runtime: Some(&state.identity), + settings: &state.settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &AllowAll, + }, + response, + &mut |_| after_send(), + ) + } + + /// Withdraws consent on an `/auction` call, as a second tab can. + fn withdraw_on_auction(state: &Arc) { + let (response, _) = drive_with_runtime( + state, + auction_request(Some(&ec_cookie()), &auction_body(None)), + Some(&state.identity), + withdraw, + ); + assert_no_server_identity_cookie(&response); + assert!( + set_cookies(&response) + .iter() + .any(|cookie| cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0")), + "control: the withdrawal should expire ts-ec: {:?}", + set_cookies(&response) + ); + } + + #[test] + fn withdrawal_on_auction_after_navigation_send_makes_phase_four_drop_and_call_nothing() { + let control = full_scenario(); + control.seed_live(); + let (_, outcome) = control.drive(navigation(Some(&returning_with_shared_id()))); + assert_eq!( + outcome, + Some(FlushOutcome::Written), + "control: should flush" + ); + assert_eq!( + control.network.requests_to(PULL_HOST).len(), + 1, + "control: should call the due provider" + ); + + let scenario = full_scenario(); + scenario.seed_live(); + let state = Arc::clone(&scenario.state); + + let outcome = drive_with_send_hook( + &state, + navigation(Some(&returning_with_shared_id())), + &mut || withdraw_on_auction(&state), + ); + + assert_eq!( + outcome, + Some(FlushOutcome::Dropped("tombstone")), + "should drop the navigation's mutation" + ); + assert!( + scenario.network.requests_to(PULL_HOST).is_empty(), + "should call no provider after the tombstone" + ); + assert_writes( + &scenario.store, + (1, 1, 0), + "the withdrawal's marker and tombstone only, nothing from phase 4", + ); + let row = scenario.row(); + assert!(!row.consent.ok, "should leave the tombstone"); + assert!(row.ids.is_empty(), "should hold no ID after the tombstone"); + } + + #[test] + fn withdrawal_returns_clear_for_the_mirror() { + let scenario = auction_scenario(); + scenario.seed(&row_with_set(Some(FAR_FUTURE))); + let (before, _) = + scenario.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + assert_eq!( + mirror_of(&json_of(before)), + serde_json::json!({ "v": 1, "eids": full_entry_set_eids() }), + "control: a live row should be mirrored" + ); + + withdraw_on_auction(&scenario.state); + let (after, _) = + scenario.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + + assert_eq!( + mirror_of(&json_of(after)), + serde_json::json!({ "clear": true }), + "should tell the shim to expire the mirror" + ); + assert_eq!( + bidder_eids(&bidder_body(&scenario)), + serde_json::Value::Null, + "should send bidders no ID from a tombstoned row" + ); + } + + #[test] + fn consent_withdrawal_writes_tombstone_and_marker() { + // (refused conditional writes, tombstone CAS attempts, marker + // writes, whether the row ends tombstoned) + let cases = [ + (0_usize, 1_usize, 1_usize, true), + (5, 6, 1, true), + (10, 10, 0, false), + ]; + for (refused, tombstone_attempts, markers, tombstoned) in cases { + let scenario = Scenario::new(); + scenario.seed_live(); + if refused > 0 { + scenario.store.refuse_conditional_writes(EC_ID, refused); + } + + let (response, outcome) = drive_with_runtime( + &scenario.state, + navigation(Some(&ec_cookie())), + Some(&scenario.state.identity), + withdraw, + ); + + assert_no_server_identity_cookie(&response); + assert_eq!( + outcome, + Some(FlushOutcome::Skipped), + "should flush nothing after a withdrawal: {refused} refused" + ); + assert_writes( + &scenario.store, + (markers, tombstone_attempts, 0), + &format!("{refused} refused writes: the marker, then every tombstone attempt"), + ); + assert_eq!( + !scenario.row().consent.ok, + tombstoned, + "should end tombstoned only when an attempt landed: {refused} refused" + ); + } + } + + #[test] + fn cas_conflict_on_flush_is_one_attempt() { + let scenario = Scenario::new(); + scenario.seed_live(); + scenario.store.refuse_conditional_writes(EC_ID, 5); + + let (_, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + outcome, + Some(FlushOutcome::WriteFailed { + removals: trusted_server_core::ec::identity::flush::RemovalWriteOutcome::NothingToWrite + }), + "should fail the flush with no removal to write" + ); + assert_writes( + &scenario.store, + (0, 1, 0), + "one refused attempt, and no retry although four more would be refused", + ); + + let control = Scenario::new(); + control.seed_live(); + control.drive(navigation(Some(&returning_with_shared_id()))); + assert_writes(&control.store, (0, 1, 0), "control: one attempt that lands"); + } + + #[test] + fn refused_flush_is_rederived_by_the_next_navigation() { + let scenario = Scenario::new(); + scenario.seed_live(); + scenario.store.refuse_conditional_writes(EC_ID, 1); + + scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + stored_uid(&scenario.row(), "sharedid.org"), + None, + "should not hold the ID after the refused write" + ); + assert_writes(&scenario.store, (0, 1, 0), "the refused attempt"); + + let (_, outcome) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + outcome, + Some(FlushOutcome::Written), + "should write the second time" + ); + assert_writes(&scenario.store, (0, 2, 0), "one attempt per navigation"); + assert_eq!( + stored_uid(&scenario.row(), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should hold the ingest the second navigation re-derived from the same cookies" + ); + } + + #[test] + fn deleted_row_is_found_by_the_next_writer_and_rotated() { + let scenario = Scenario::new(); + scenario.seed_live(); + scenario + .graph() + .delete(EC_ID) + .expect("should delete the row out of band"); + scenario.store.reset_counts(); + + let (response, _) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + let rotated = set_cookie_value(&response, "ts-ec").expect("should set the new EC"); + assert_ne!(rotated, EC_ID, "should rotate to a new ID"); + assert_writes( + &scenario.store, + (1, 0, 0), + "one Add creates the replacement", + ); + assert_eq!( + stored_uid(&scenario.row_of(&rotated), "sharedid.org").as_deref(), + Some(SHARED_ID), + "should carry the ingest in the replacement row" + ); + assert!( + scenario.graph().get(EC_ID).expect("should read").is_none(), + "should not resurrect the deleted ID" + ); + } + + #[test] + fn stager_that_sees_missing_row_returns_clear_and_next_writer_recovers() { + let live = auction_scenario(); + live.seed(&row_with_set(Some(FAR_FUTURE))); + let (served, _) = live.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + assert_ne!( + mirror_of(&json_of(served)), + serde_json::json!({ "clear": true }), + "control: a live row should be mirrored" + ); + + let scenario = auction_scenario(); + let (response, _) = + scenario.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + + assert_eq!( + mirror_of(&json_of(response)), + serde_json::json!({ "clear": true }), + "should return clear for a missing row" + ); + assert_no_writes(&scenario.store, "the stager stores nothing"); + assert!( + scenario.graph().get(EC_ID).expect("should read").is_none(), + "should not create the row" + ); + + let (navigated, _) = scenario.drive(navigation(Some(&returning_with_shared_id()))); + + let rotated = set_cookie_value(&navigated, "ts-ec").expect("should set the new EC"); + assert_ne!(rotated, EC_ID, "the next writer should recover by rotating"); + assert_writes(&scenario.store, (1, 0, 0), "one Add by the writer"); + } + + const REGISTERED_FIRST_TRIM_COOKIE: &str = include_str!( + "../../trusted-server-js/lib/test/fixtures/ts-eids-registered-first-trim.txt" + ); + const ORTB2_ONLY_COOKIE: &str = + include_str!("../../trusted-server-js/lib/test/fixtures/ts-eids-ortb2-only.txt"); + + /// A provider that answers every call by invalidating the ID of its + /// one source, so the flush carries a removal. + struct InvalidatingProvider { + descriptor: trusted_server_core::ec::identity::ProviderDescriptor, + source: trusted_server_core::ec::identity::SourceDomain, + } + + impl InvalidatingProvider { + fn new(id: &str, source: &str) -> Self { + use trusted_server_core::ec::identity::{ + ConsentPolicy, ProviderDescriptor, Resolves, + }; + let source: trusted_server_core::ec::identity::SourceDomain = + source.parse().expect("should parse the source"); + let descriptor = + ProviderDescriptor::builder(id.parse().expect("should parse the provider ID")) + .with_source(source.clone()) + .with_resolves( + Resolves::new().with_endpoint("https://probe.example.com/resolve"), + ) + .with_consent_policy(ConsentPolicy::strict()) + .build() + .expect("should build the descriptor"); + Self { descriptor, source } + } + } + + impl trusted_server_core::ec::identity::IdentityProvider for InvalidatingProvider { + fn descriptor(&self) -> &trusted_server_core::ec::identity::ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &trusted_server_core::ec::identity::ResolveInput<'_>, + ) -> Result< + Option, + Report, + > { + Ok(Some( + trusted_server_core::ec::identity::ProviderRequest::get(input.endpoint()?), + )) + } + + fn response( + &self, + response: &trusted_server_core::ec::identity::ProviderResponse, + _input: &trusted_server_core::ec::identity::ResolveInput<'_>, + ) -> Result< + trusted_server_core::ec::identity::Resolution, + Report, + > { + response.success_text()?; + Ok(trusted_server_core::ec::identity::Resolution::invalidate( + &self.source, + )) + } + } + + /// A scenario whose one provider invalidates the ID it wrote for + /// `ssp.example.com`, which the seeded row holds and is due to refresh. + fn invalidating_scenario() -> Scenario { + let Scenario { + state, + store, + network, + } = Scenario::new(); + let provider: Arc = Arc::new( + InvalidatingProvider::new("removal-probe", "ssp.example.com"), + ); + let runtime = IdentityRuntime::from_settings_with_providers( + &state.settings, + vec![provider], + BTreeMap::from([( + "ssp.example.com".parse().expect("should parse the source"), + "removal-probe" + .parse() + .expect("should parse the provider ID"), + )]), + ) + .expect("should build the identity runtime"); + let scenario = Scenario { + state: crate::app::app_state_with_identity_for_test(state, runtime), + store, + network, + }; + scenario + .network + .respond("probe.example.com", 200, "text/plain", "ok"); + let mut entry = live_entry(); + entry.ids.insert( + "ssp.example.com".to_owned(), + KvPartnerId { + w: Some("removal-probe".to_owned()), + rv: Some("0123456789abcdef".to_owned()), + at: Some(1_000), + ra: Some(1_500), + ..KvPartnerId::new("probe-uid") + }, + ); + scenario.seed(&entry); + scenario + } + + #[test] + fn refused_flush_writes_its_removals_alone() { + let control = invalidating_scenario(); + control.drive(navigation(Some(&returning_with_shared_id()))); + assert_writes(&control.store, (0, 1, 0), "control: one write holds both"); + let row = control.row(); + assert_eq!( + stored_uid(&row, "ssp.example.com"), + None, + "control: the removal should be in the one write" + ); + assert_eq!( + stored_uid(&row, "sharedid.org").as_deref(), + Some(SHARED_ID), + "control: so should the ingest" + ); + + let refused_once = invalidating_scenario(); + refused_once.store.refuse_conditional_writes(EC_ID, 1); + + let (_, outcome) = refused_once.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + outcome, + Some(FlushOutcome::WriteFailed { + removals: + trusted_server_core::ec::identity::flush::RemovalWriteOutcome::Written + }), + "should write the removals after the flush is refused" + ); + assert_writes( + &refused_once.store, + (0, 2, 0), + "the refused flush, then the removals alone", + ); + let row = refused_once.row(); + assert_eq!( + stored_uid(&row, "ssp.example.com"), + None, + "should have removed the invalidated ID" + ); + assert_eq!( + stored_uid(&row, "sharedid.org"), + None, + "should not write the refused ingest with the removals" + ); + + let refused_always = invalidating_scenario(); + refused_always.store.refuse_conditional_writes(EC_ID, 6); + + let (_, outcome) = refused_always.drive(navigation(Some(&returning_with_shared_id()))); + + assert_eq!( + outcome, + Some(FlushOutcome::WriteFailed { + removals: trusted_server_core::ec::identity::flush::RemovalWriteOutcome::Failed + }), + "should report the removals lost when every attempt is refused" + ); + assert_writes( + &refused_always.store, + (0, 6, 0), + "the flush once, then the removals up to five times", + ); + assert_eq!( + stored_uid(&refused_always.row(), "ssp.example.com").as_deref(), + Some("probe-uid"), + "should leave the row as it was" + ); + } + + #[test] + fn registered_source_beyond_todays_trim_reaches_kv_on_next_navigation() { + use base64::Engine as _; + let cookie_value = REGISTERED_FIRST_TRIM_COOKIE.trim_end(); + let decoded: serde_json::Value = serde_json::from_slice( + &base64::engine::general_purpose::STANDARD + .decode(cookie_value) + .expect("should decode the shim's cookie"), + ) + .expect("should parse the shim's cookie"); + let sources: Vec<&str> = decoded + .as_array() + .expect("should be a list") + .iter() + .filter_map(|entry| entry["source"].as_str()) + .collect(); + assert_eq!( + sources.first().copied(), + Some("registered.example.com"), + "control: the shim's trim should keep the registered source first" + ); + assert!( + !sources.contains(&"u6.example.com"), + "control: the trim should have dropped an unregistered source: {sources:?}" + ); + let scenario = Scenario::with_settings(&partner_toml("registered.example.com")); + scenario.seed_live(); + + let (_, outcome) = scenario.drive(navigation(Some(&format!( + "ts-ec={EC_ID}; ts-eids={cookie_value}" + )))); + + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush"); + assert_writes(&scenario.store, (0, 1, 0), "one write"); + let row = scenario.row(); + assert_eq!( + stored_uid(&row, "registered.example.com"), + Some("x".repeat(400)), + "should hold the registered source the old trim would have dropped" + ); + assert!( + (0..7).all(|index| !row.ids.contains_key(&format!("u{index}.example.com"))), + "should hold no unregistered source: {:?}", + row.ids.keys().collect::>() + ); + } + + #[test] + fn ortb2_only_eid_reaches_kv() { + let scenario = Scenario::new(); + scenario.seed_live(); + assert_eq!( + stored_uid(&scenario.row(), "id5-sync.com"), + None, + "control: id5-sync.com should be absent from KV before the navigation" + ); + let cookie = format!("ts-ec={EC_ID}; ts-eids={}", ORTB2_ONLY_COOKIE.trim_end()); + + let (_, outcome) = scenario.drive(navigation(Some(&cookie))); + + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush"); + assert_writes(&scenario.store, (0, 1, 0), "one write"); + assert_eq!( + stored_uid(&scenario.row(), "id5-sync.com").as_deref(), + Some("fictional-id5"), + "should hold the ID5 UID the shim copied from the ortb2 config" + ); + } + + /// Everything a visitor's navigation, `/auction` and `/identify` + /// produce, for comparing runs. + #[derive(Debug, PartialEq)] + struct Observed { + navigation: (u16, Option), + origin_cookie: Option, + row_uids: BTreeMap>, + bidder_request: serde_json::Value, + auction_header_eids: serde_json::Value, + auction_mirror: serde_json::Value, + identify: (u16, serde_json::Value), + writes: (usize, usize, usize), + } + + fn observe_with_mirror_cookie(mirror: Option<&str>) -> Observed { + let scenario = auction_scenario(); + let mut entry = live_entry(); + entry.ids.insert( + "id5-sync.com".to_owned(), + KvPartnerId::browser("server-id5", Some(1_500)), + ); + scenario.seed(&entry); + let mut cookie = returning_with_shared_id(); + if let Some(mirror) = mirror { + cookie.push_str(&format!("; __Host-ts-eids-s={mirror}")); + } + + let (nav, nav_outcome) = scenario.drive(navigation(Some(&cookie))); + let (auction, _) = scenario.drive(auction_request(Some(&cookie), &auction_body(None))); + let (identify, _) = scenario.drive(identify_request(&cookie, "sharedid.org")); + + let mut bidder_request = bidder_body(&scenario); + bidder_request + .as_object_mut() + .expect("should be an object") + .remove("id"); + let row = scenario.row(); + let writes = scenario.store.writes(); + let origin_cookie = scenario + .network + .requests_to(ORIGIN_HOST) + .first() + .and_then(|seen| seen.headers.get("cookie")) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + Observed { + navigation: (nav.status().as_u16(), nav_outcome), + origin_cookie, + row_uids: row + .ids + .keys() + .map(|source| (source.clone(), stored_uid(&row, source))) + .collect(), + bidder_request, + auction_header_eids: header_eids(&auction), + auction_mirror: mirror_of(&json_of(auction)), + identify: (identify.status().as_u16(), json_of(identify)), + writes: (writes.add, writes.if_generation_match, writes.overwrite), + } + } + + #[test] + fn forged_mirror_cookie_changes_nothing() { + use base64::Engine as _; + let encode = |value: &serde_json::Value| { + base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_vec(value).expect("should serialize the mirror")) + }; + let forged = encode(&serde_json::json!({ + "v": 1, + "eids": [ + { "source": "sharedid.org", "uids": [{ "id": "forged-uid", "atype": 1 }] }, + { "source": "forged.example.com", "uids": [{ "id": "forged-uid", "atype": 1 }] } + ] + })); + let stale = encode(&serde_json::json!({ + "v": 1, + "eids": [{ "source": "id5-sync.com", "uids": [{ "id": "stale-id5", "atype": 1 }] }] + })); + let baseline = observe_with_mirror_cookie(None); + let shown = format!("{baseline:?}"); + assert!( + shown.contains("server-id5") && shown.contains(SHARED_ID), + "control: the baseline should hold the server and browser IDs: {shown}" + ); + assert_eq!( + baseline.writes, + (0, 1, 0), + "control: the navigation should write once" + ); + + for (name, value) in [ + ("forged", forged.as_str()), + ("stale", stale.as_str()), + ("malformed", "%%%not-base64"), + ("empty", ""), + ] { + let observed = observe_with_mirror_cookie(Some(value)); + + assert_eq!( + observed, baseline, + "should change no bidder request, answer or write: {name}" + ); + let shown = format!("{observed:?}"); + assert!( + !shown.contains("forged-uid") && !shown.contains("stale-id5"), + "should carry nothing from the {name} mirror: {shown}" + ); + } + } + + /// What the code before the identity work, run on this input, + /// stores: the first UID of the last entry of each registered + /// source, with the sharedId cookie last, and nothing for an + /// unregistered source. + const GOLDEN_INGEST_COOKIE: &str = "W3siZXh0Ijp7InNlZ21lbnRzIjpbInMxIl19LCJpbnNlcnRlciI6Imluc2VydGVyLmV4YW1wbGUuY29tIiwibW0iOjIsInNvdXJjZSI6ImlkNS1zeW5jLmNvbSIsInVpZHMiOlt7ImF0eXBlIjoxLCJpZCI6ImlkNS1maXJzdCJ9LHsiYXR5cGUiOjEsImlkIjoiaWQ1LXNlY29uZCJ9XX0seyJzb3VyY2UiOiJ1bnJlZ2lzdGVyZWQuZXhhbXBsZS5jb20iLCJ1aWRzIjpbeyJhdHlwZSI6MSwiaWQiOiJ1bnJlZy0xIn1dfSx7InNvdXJjZSI6InNoYXJlZGlkLm9yZyIsInVpZHMiOlt7ImF0eXBlIjoxLCJpZCI6InByZWJpZC1zaGFyZWQifV19LHsiaW5zZXJ0ZXIiOiJsYXRlci5leGFtcGxlLmNvbSIsIm1tIjoxLCJzb3VyY2UiOiJpZDUtc3luYy5jb20iLCJ1aWRzIjpbeyJhdHlwZSI6MSwiaWQiOiJpZDUtbGFzdC1maXJzdCJ9LHsiYXR5cGUiOjEsImlkIjoiaWQ1LWxhc3Qtc2Vjb25kIn1dfV0="; + + /// The `x-ts-eids` value the code before the identity work builds + /// for [`GOLDEN_ROW_EIDS`]. + const GOLDEN_ROW_HEADER: &str = "W3sic291cmNlIjoiaWQ1LXN5bmMuY29tIiwidWlkcyI6W3siaWQiOiJpZDUtdWlkIiwiYXR5cGUiOjN9XX0seyJzb3VyY2UiOiJzaGFyZWRpZC5vcmciLCJ1aWRzIjpbeyJpZCI6InNoYXJlZC11aWQiLCJhdHlwZSI6M31dfV0="; + + /// A body with repeated sources and EID-level fields. + const GOLDEN_BODY_EIDS: &str = r#"[ + {"inserter":"adserver.org","matcher":"adserver.org","mm":4,"source":"adserver.org","uids":[{"atype":1,"ext":{"rtiPartner":"TDID"},"id":"body-tdid"}]}, + {"ext":{"segments":["s1"]},"inserter":"other.example.com","mm":1,"source":"adserver.org","uids":[{"atype":1,"id":"body-tdid-2"}]}, + {"source":"ids.example.com","uids":[{"atype":3,"id":"ids-uid"}]} + ]"#; + + /// What the code before the identity work forwards for + /// [`GOLDEN_BODY_EIDS`] and an empty row. + const GOLDEN_BODY_FORWARDED: &str = r#"[ + {"source":"adserver.org","uids":[{"id":"body-tdid","atype":1,"ext":{"rtiPartner":"TDID"}}],"inserter":"adserver.org","matcher":"adserver.org","mm":4}, + {"source":"adserver.org","uids":[{"id":"body-tdid-2","atype":1}],"inserter":"other.example.com","mm":1,"ext":{"segments":["s1"]}}, + {"source":"ids.example.com","uids":[{"id":"ids-uid","atype":3}]} + ]"#; + + fn golden_expected_ingest() -> BTreeMap { + BTreeMap::from([ + ("id5-sync.com".to_owned(), "id5-last-first".to_owned()), + ("sharedid.org".to_owned(), "shared-1".to_owned()), + ]) + } + + /// Each source's first UID, with a check that the record is one + /// entry holding one UID and nothing else. + fn assert_one_uid_records(row: &KvEntry) -> BTreeMap { + row.ids + .iter() + .map(|(source, record)| { + assert_eq!(record.w, None, "should write no provider: {source}"); + assert_eq!(record.eids.len(), 1, "should hold one entry: {source}"); + let entry = &record.eids[0]; + assert_eq!(entry.uids.len(), 1, "should hold one UID: {source}"); + assert_eq!( + (&entry.inserter, &entry.matcher, entry.mm, &entry.ext), + (&None, &None, None, &None), + "should hold no EID-level field: {source}" + ); + assert_eq!( + (entry.uids[0].atype, &entry.uids[0].ext), + (None, &None), + "should hold no UID-level field: {source}" + ); + (source.clone(), entry.uids[0].id.clone()) + }) + .collect() + } + + #[test] + fn golden_browser_ingest_writes_the_first_uid() { + let cookie = format!("ts-eids={GOLDEN_INGEST_COOKIE}; sharedId=shared-1"); + + let returning = Scenario::new(); + returning.seed_live(); + let (_, outcome) = + returning.drive(navigation(Some(&format!("ts-ec={EC_ID}; {cookie}")))); + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush"); + assert_writes(&returning.store, (0, 1, 0), "one flush"); + assert_eq!( + assert_one_uid_records(&returning.row()), + golden_expected_ingest(), + "the flushed row should hold the UIDs the code before this spec stored" + ); + + let new_visitor = Scenario::new(); + let (response, _) = new_visitor.drive(navigation(Some(&cookie))); + let minted = set_cookie_value(&response, "ts-ec").expect("should mint an EC"); + assert_writes(&new_visitor.store, (1, 0, 0), "one create"); + assert_eq!( + assert_one_uid_records(&new_visitor.row_of(&minted)), + golden_expected_ingest(), + "the created row should hold the same UIDs" + ); + } + + #[test] + fn golden_one_uid_row_goes_out_unchanged() { + let scenario = auction_scenario(); + let mut entry = live_entry(); + for (source, uid) in [("id5-sync.com", "id5-uid"), ("sharedid.org", "shared-uid")] { + entry + .ids + .insert(source.to_owned(), KvPartnerId::browser(uid, Some(1_500))); + } + scenario.seed(&entry); + + let (response, _) = + scenario.drive(auction_request(Some(&ec_cookie()), &auction_body(None))); + + let expected: serde_json::Value = serde_json::json!([ + { "source": "id5-sync.com", "uids": [{ "id": "id5-uid", "atype": 3 }] }, + { "source": "sharedid.org", "uids": [{ "id": "shared-uid", "atype": 3 }] } + ]); + assert_eq!( + bidder_eids(&bidder_body(&scenario)), + expected, + "user.ext.eids should equal what the code before this spec sent" + ); + assert_eq!( + response + .headers() + .get("x-ts-eids") + .and_then(|value| value.to_str().ok()), + Some(GOLDEN_ROW_HEADER), + "x-ts-eids should equal what the code before this spec built, byte for byte" + ); + } + + #[test] + fn golden_auction_body_forwarding_is_unchanged() { + let scenario = auction_scenario(); + scenario.seed_live(); + let eids: serde_json::Value = + serde_json::from_str(GOLDEN_BODY_EIDS).expect("should parse the body EIDs"); + let forwarded: serde_json::Value = serde_json::from_str(GOLDEN_BODY_FORWARDED) + .expect("should parse the forwarded EIDs"); + assert!( + forwarded.to_string().contains("rtiPartner") + && forwarded.to_string().contains("other.example.com"), + "control: the golden should hold UID-level and EID-level fields" + ); + + scenario.drive(auction_request( + Some(&ec_cookie()), + &auction_body(Some(&eids)), + )); + + assert_eq!( + bidder_eids(&bidder_body(&scenario)), + forwarded, + "user.ext.eids should equal what the code before this spec forwarded" + ); + assert_no_writes(&scenario.store, "/auction"); + } + } } diff --git a/crates/trusted-server-core/src/ec/identity/runtime.rs b/crates/trusted-server-core/src/ec/identity/runtime.rs index 19d0aa044d..5ae2a52f25 100644 --- a/crates/trusted-server-core/src/ec/identity/runtime.rs +++ b/crates/trusted-server-core/src/ec/identity/runtime.rs @@ -55,6 +55,29 @@ impl IdentityRuntime { }) } + /// Builds the runtime with `providers` and source `owners` in place of + /// the generic pull providers `settings` describes, for tests that drive + /// a request through a provider no configuration can name. + /// + /// # Errors + /// + /// Never fails today; it returns a `Result` to match + /// [`from_settings`](Self::from_settings). + #[cfg(any(test, feature = "test-utils"))] + pub fn from_settings_with_providers( + settings: &Settings, + providers: Vec>, + owners: BTreeMap, + ) -> Result> { + let catalog = ProviderCatalog::new(providers, owners) + .with_narrowing(narrowing(&settings.ec.identity)); + Ok(Self { + catalog, + keys: IdentityKeys::from_settings(settings), + identity: settings.ec.identity.clone(), + }) + } + /// The enabled providers. #[must_use] pub(crate) fn catalog(&self) -> &ProviderCatalog { diff --git a/crates/trusted-server-js/lib/test/fixtures/ts-eids-golden.txt b/crates/trusted-server-js/lib/test/fixtures/ts-eids-golden.txt new file mode 100644 index 0000000000..24991f0ade --- /dev/null +++ b/crates/trusted-server-js/lib/test/fixtures/ts-eids-golden.txt @@ -0,0 +1 @@ +W3sic291cmNlIjoiaWQ1LXN5bmMuY29tIiwiaW5zZXJ0ZXIiOiJpbnNlcnRlci5leGFtcGxlLmNvbSIsIm1hdGNoZXIiOiJpZDUtc3luYy5jb20iLCJtbSI6MiwidWlkcyI6W3siaWQiOiJnb2xkZW4taWQ1LWEiLCJhdHlwZSI6MSwiZXh0Ijp7ImxpbmtUeXBlIjoyfX0seyJpZCI6ImdvbGRlbi1pZDUtYiIsImF0eXBlIjoxfV19LHsic291cmNlIjoic2hhcmVkaWQub3JnIiwidWlkcyI6W3siaWQiOiJnb2xkZW4tc2hhcmVkIiwiYXR5cGUiOjF9XX1d diff --git a/crates/trusted-server-js/lib/test/fixtures/ts-eids-ortb2-only.txt b/crates/trusted-server-js/lib/test/fixtures/ts-eids-ortb2-only.txt new file mode 100644 index 0000000000..e9e9113453 --- /dev/null +++ b/crates/trusted-server-js/lib/test/fixtures/ts-eids-ortb2-only.txt @@ -0,0 +1 @@ +W3sic291cmNlIjoiaWQ1LXN5bmMuY29tIiwiaW5zZXJ0ZXIiOiJpbnNlcnRlci5leGFtcGxlLmNvbSIsIm1tIjoyLCJ1aWRzIjpbeyJpZCI6ImZpY3Rpb25hbC1pZDUiLCJhdHlwZSI6MX1dfV0= diff --git a/crates/trusted-server-js/lib/test/fixtures/ts-eids-registered-first-trim.txt b/crates/trusted-server-js/lib/test/fixtures/ts-eids-registered-first-trim.txt new file mode 100644 index 0000000000..5d3885eaac --- /dev/null +++ b/crates/trusted-server-js/lib/test/fixtures/ts-eids-registered-first-trim.txt @@ -0,0 +1 @@ +W3sic291cmNlIjoicmVnaXN0ZXJlZC5leGFtcGxlLmNvbSIsInVpZHMiOlt7ImlkIjoieHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eCIsImF0eXBlIjoxfV19LHsic291cmNlIjoidTAuZXhhbXBsZS5jb20iLCJ1aWRzIjpbeyJpZCI6Inh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHgiLCJhdHlwZSI6MX1dfSx7InNvdXJjZSI6InUxLmV4YW1wbGUuY29tIiwidWlkcyI6W3siaWQiOiJ4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4IiwiYXR5cGUiOjF9XX0seyJzb3VyY2UiOiJ1Mi5leGFtcGxlLmNvbSIsInVpZHMiOlt7ImlkIjoieHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eCIsImF0eXBlIjoxfV19LHsic291cmNlIjoidTMuZXhhbXBsZS5jb20iLCJ1aWRzIjpbeyJpZCI6Inh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHgiLCJhdHlwZSI6MX1dfV0= diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 23cf90d48a..07921c2368 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -2803,6 +2803,94 @@ describe('prebid/installPrebidNpm', () => { } }); + it('ortb2_config_only_eid_reaches_ts_eids_and_the_auction_body', () => { + // Lockr writes its ID5 into Prebid's ortb2 config only, so + // getUserIdsAsEids() (the canary) never returns it. The Rust identity + // tests carry the same cookie value as a fixture. + const spec = getAdapterSpec(); + const lockrId5 = { + source: 'id5-sync.com', + inserter: 'inserter.example.com', + mm: 2, + uids: [{ id: 'fictional-id5', atype: 1 }], + }; + expect( + mockGetUserIdsAsEids().map((eid: { source: string }) => eid.source), + 'control: getUserIdsAsEids should not hold the ID5 entry' + ).not.toContain('id5-sync.com'); + + const result = spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [lockrId5] } }) + ); + + expect(JSON.parse(String(result.data)).eids, 'the /auction body carries the entry').toEqual([ + lockrId5, + ]); + expect(readTsEidsCookie(), 'ts-eids carries the entry, as this exact value').toBe( + readFixture('ts-eids-ortb2-only.txt') + ); + }); + + it('ts_eids_cookie_golden', () => { + // A set with no unregistered source, so the registered-first trim + // leaves it exactly as before. EID-level ext never reaches the cookie. + testWindow.__tsjs_prebid = { registeredSources: ['id5-sync.com', 'sharedid.org'] }; + try { + const spec = getAdapterSpec(); + + spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ + ext: { + eids: [ + { + source: 'id5-sync.com', + inserter: 'inserter.example.com', + matcher: 'id5-sync.com', + mm: 2, + ext: { segments: ['s1'] }, + uids: [ + { id: 'golden-id5-a', atype: 1, ext: { linkType: 2 } }, + { id: 'golden-id5-b', atype: 1 }, + ], + }, + { source: 'sharedid.org', uids: [{ id: 'golden-shared', atype: 1 }] }, + ], + }, + }) + ); + + expect(readTsEidsCookie()).toBe(readFixture('ts-eids-golden.txt')); + } finally { + delete testWindow.__tsjs_prebid; + } + }); + + it('writes a registered source beyond the old trim into ts-eids as this exact value', () => { + // The Rust identity tests ingest this value: its registered source is + // the eighth entry, which the pre-G10 trim dropped. + testWindow.__tsjs_prebid = { registeredSources: ['registered.example.com'] }; + try { + const spec = getAdapterSpec(); + const long = 'x'.repeat(400); + const unregistered = [0, 1, 2, 3, 4, 5, 6].map((index) => ({ + source: `u${index}.example.com`, + uids: [{ id: long, atype: 1 }], + })); + const registered = { source: 'registered.example.com', uids: [{ id: long, atype: 1 }] }; + + spec.buildRequests( + trustedServerBidRequests(), + prebidBidderRequest({ ext: { eids: [...unregistered, registered] } }) + ); + + expect(readTsEidsCookie()).toBe(readFixture('ts-eids-registered-first-trim.txt')); + } finally { + delete testWindow.__tsjs_prebid; + } + }); + it('logs EID counts and the cookie decision at debug level, never values', () => { log.setLevel('debug'); const spies = [ From e8a57e23acc8aed4d88613d6fc1758f1ef5cdcb5 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 00:51:47 +0530 Subject: [PATCH 43/89] Document the single identity write and the ts-eids-s mirror Log one identity request measurement line per request after phase 4, with the Cookie header size, KV reads and writes by route kind, the flush outcome, lost removals and the after-send withdrawal run. It holds no identifier. Document the writer-request rule, the mirror payload, the [ec.identity] table, the removed config keys, the rollback limit and the removed pull-complete cookie. Trim the EID sync outcome enum to the one value still logged. Signed-off-by: dhruv8sh --- CHANGELOG.md | 8 + .../trusted-server-adapter-fastly/src/main.rs | 161 +++++++++++++++-- crates/trusted-server-core/src/ec/finalize.rs | 131 ++------------ .../src/ec/identity/measurement.rs | 162 ++++++++++++++++++ .../src/ec/identity/mod.rs | 1 + .../src/ec/identity/request.rs | 12 ++ crates/trusted-server-core/src/ec/kv.rs | 48 ++---- .../trusted-server-core/src/ec/kv_backend.rs | 85 +++++++++ crates/trusted-server-core/src/ec/mod.rs | 49 +++++- docs/guide/api-reference.md | 28 +++ docs/guide/configuration.md | 66 ++++--- docs/guide/ec-setup-guide.md | 9 + docs/guide/edge-cookies.md | 32 +++- trusted-server.example.toml | 22 ++- 14 files changed, 610 insertions(+), 204 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/measurement.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b9026e031..c393e5cfdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Identity KV is written once per page load. Only a full page navigation from a returning visitor (the writer request) writes KV, once, after the response is sent: browser EIDs, pull partner results and the cluster size all go in that one write, with a generation check and no retry of the whole write. Every other route reads KV and stages but writes nothing. A consent-withdrawal tombstone and the removal-only retry of a failed write are the only exceptions. Mid-branch builds drop browser ingest until the whole change ships. +- `POST /auction`, `GET /_ts/page-bids` and `GET /_ts/api/v1/identify` no longer write KV, and `/_ts/api/v1/identify` omits expired IDs. The `/identify` response shape is unchanged: one UID per source. +- Pull partners now run from a visitor's second page load, from the writer request after send, and back off on errors. They are paced by `[ec.identity.providers.pull-]` (`refresh_after_sec`, `max_calls_per_minute`), so the old default of 10 calls per hour becomes an effective 60 per hour. Runtime startup now builds each pull partner's descriptor: a partner whose allowed host fails the descriptor host check stops startup. See [Identity Providers](docs/guide/configuration.md#identity-providers-ec-identity). +- **Breaking:** `ec.pull_sync_concurrency`, and `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]`, are removed, and validation now rejects them with an error that names the replacement. Use `[ec.identity].max_concurrency`, `refresh_after_sec`, and `max_calls_per_minute = ceil(N / 60)` for `pull_sync_rate_limit = N`, the last two in `[ec.identity.providers.]`. +- `/auction` responses carry `ext.ts_eids_s` and `/_ts/page-bids` responses carry `ts_eids_s`: the payload (`value` with `max_age`, `clear`, or absent) for the client-only `__Host-ts-eids-s` mirror cookie, at most 2,048 bytes and one day. The shim's `ts-eids` size trim now keeps registered sources first. +- The `ts-ec-pull-complete` cookie is removed: Trusted Server no longer sets or reads it, and a copy a browser holds expires within one hour. +- **Breaking:** The KV identity row moves to schema version 2. A release before this one cannot read a version 2 row, so rolling back past this release is unsupported once it has written one. See [Rolling back](docs/guide/configuration.md#rolling-back). This release should merge after the EID-fidelity changes, whose behaviour its golden tests pin. +- The `EID sync measurement: ...` log line is no longer logged for returning visitors; only a new EC still logs it. Each request now logs one `identity request measurement: ...` line after the response is sent, with the inbound `Cookie` header size, KV reads and writes by route kind, the flush outcome, lost removals and the after-send withdrawal run. It holds no identifiers. Operator dashboards that read the old line should read the new one. - **Breaking:** The publisher origin no longer receives the `ts-eids` cookie, and Prebid Server no longer receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, and the `__Host-ts-eids-s` mirror that this release's shim writes). The origin still receives `ts-ec`. Origin code that read `ts-eids` should read the `x-ts-eids` response header instead. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. - **Breaking:** `ts prebid bundle` is now `ts prebid client`, alongside the new `ts prebid server` namespace. Update scripts and runbooks to use `ts prebid client` with the same arguments. The old `bundle` spelling is no longer accepted and has no compatibility alias. diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 53dd660974..56e6dfc9b0 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,13 +13,16 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::EcKvSnapshot; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::{ec_finalize_response, retry_failed_withdrawals}; use trusted_server_core::ec::identity::IdentityRuntime; use trusted_server_core::ec::identity::flush::{ AfterSendInput, FlushOutcome, may_flush, run_after_send, }; +use trusted_server_core::ec::identity::measurement::{IdentityMeasurement, MeasurementInput}; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::kv_backend::KvOperationCounts; use trusted_server_core::ec::rate_limiter::RateLimiter; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; @@ -631,15 +634,30 @@ struct PostRouteInput<'a> { } /// Finalizes the response, hands it to `send`, then runs the after-send work: -/// the withdrawal retry first, then phase 4's one flush (spec 5.2). +/// the withdrawal retry first, then phase 4's one flush (spec 5.2), and logs +/// the request's `identity request measurement` line. /// /// `send` is a callback so phase 4 runs only after the response is sent. /// Returns phase 4's outcome, or `None` when finalize could not run. fn finalize_send_and_flush( input: PostRouteInput<'_>, - mut response: HttpResponse, + response: HttpResponse, send: &mut dyn FnMut(HttpResponse), ) -> Option { + let (outcome, measurement) = finalize_send_and_flush_measured(input, response, send); + if let Some(measurement) = measurement { + measurement.log(); + } + outcome +} + +/// As [`finalize_send_and_flush`], returning the measurement instead of +/// logging it. The measurement is `None` when finalize could not run. +fn finalize_send_and_flush_measured( + input: PostRouteInput<'_>, + mut response: HttpResponse, + send: &mut dyn FnMut(HttpResponse), +) -> (Option, Option) { let PostRouteInput { runtime, settings, @@ -652,11 +670,19 @@ fn finalize_send_and_flush( Err(e) => { log::error!("EdgeZero EC finalize skipped: failed to build partner registry: {e:?}"); send(response); - return None; + return (None, None); } }; + // Every KV operation from here on counts into this; phase 1's read is + // known from the snapshot the route left. + let kv_counts = KvOperationCounts::default(); + let phase_one_read = !matches!(ec_state.ec_context.kv_snapshot(), EcKvSnapshot::NotRead); let mut built: Option> = None; - let mut graph_once = || built.get_or_insert_with(graph).clone(); + let mut graph_once = || { + built + .get_or_insert_with(|| graph().map(|graph| graph.with_operation_counts(&kv_counts))) + .clone() + }; let finalize_kv_graph = if ec_state.use_finalize_kv { graph_once() @@ -675,23 +701,56 @@ fn finalize_send_and_flush( let status = response.status(); send(response); - if ec_state.ec_context.has_failed_withdrawals() { + // A retry skipped because the graph is unavailable leaves the ID failed, so + // it counts as a failed run. + let withdrawal_after_send = ec_state.ec_context.has_failed_withdrawals(); + if withdrawal_after_send { match graph_once() { Some(graph) => retry_failed_withdrawals(&mut ec_state.ec_context, &graph), None => log::warn!("withdrawal retry skipped: identity graph unavailable"), } } + let withdrawal_after_send_failed = ec_state.ec_context.has_failed_withdrawals(); + + let outcome = flush_after_send( + runtime, + settings, + ec_state, + &mut graph_once, + status, + rate_limiter, + ); + let measurement = ec_state.ec_context.identity_measurement(&MeasurementInput { + phase_one_read, + kv_reads: kv_counts.reads(), + kv_writes: kv_counts.writes(), + flush: Some(&outcome), + withdrawal_after_send, + withdrawal_after_send_failed, + }); + (Some(outcome), Some(measurement)) +} + +/// Phase 4's one flush, or [`FlushOutcome::Skipped`] when it cannot run. +fn flush_after_send( + runtime: Option<&IdentityRuntime>, + settings: &Settings, + ec_state: &mut EcFinalizeState, + graph_once: &mut dyn FnMut() -> Option, + status: edgezero_core::http::StatusCode, + rate_limiter: &dyn RateLimiter, +) -> FlushOutcome { let Some(runtime) = runtime else { - return Some(FlushOutcome::Skipped); + return FlushOutcome::Skipped; }; if !ec_state.is_real_browser || !may_flush(&ec_state.ec_context) { - return Some(FlushOutcome::Skipped); + return FlushOutcome::Skipped; } let Some(graph) = graph_once() else { log::debug!("identity flush skipped: identity graph unavailable"); - return Some(FlushOutcome::Skipped); + return FlushOutcome::Skipped; }; - Some(run_after_send(AfterSendInput { + run_after_send(AfterSendInput { runtime, settings, services: &ec_state.services, @@ -699,7 +758,7 @@ fn finalize_send_and_flush( context: &mut ec_state.ec_context, status, rate_limiter, - })) + }) } /// Sends a finalized `EdgeZero` response to the client. @@ -892,6 +951,7 @@ mod tests { use edgezero_core::http::HeaderValue; use edgezero_core::http::response_builder; use fastly::mime; + use trusted_server_core::ec::identity::measurement::RouteKind; use trusted_server_core::integrations::HeaderMutation; const TEST_SETTINGS_TOML: &str = r#" @@ -1002,6 +1062,36 @@ mod tests { (sent.expect("should send the response"), outcome) } + /// As [`drive_request_for_test`], keeping the request's measurement. + fn drive_measured_for_test( + state: &Arc, + request: HttpRequest, + ) -> (Option, IdentityMeasurement) { + let router = TrustedServerApp::routes_for_state(state); + let mut response = + futures::executor::block_on(router.oneshot(request)).expect("should route the request"); + let mut ec_state = response + .extensions_mut() + .remove::() + .expect("should leave EC state on the response"); + let graph = || state.identity_graph.clone(); + let (outcome, measurement) = finalize_send_and_flush_measured( + PostRouteInput { + runtime: Some(&state.identity), + settings: &state.settings, + ec_state: &mut ec_state, + graph: &graph, + rate_limiter: &AllowAll, + }, + response, + &mut |_| {}, + ); + ( + outcome, + measurement.expect("should measure a finalized request"), + ) + } + /// A returning visitor's live row, without a cluster size so the flush /// has one change to write. fn seeded_state() -> ( @@ -1022,6 +1112,57 @@ mod tests { (state, store) } + #[test] + fn identity_measurement_records_counts_for_a_writer_navigation() { + let (state, _store) = seeded_state(); + let cookie = format!("ts-ec={EC_ID}"); + + let (outcome, measurement) = drive_measured_for_test(&state, navigation(Some(&cookie))); + + assert_eq!(outcome, Some(FlushOutcome::Written), "should flush once"); + assert_eq!( + measurement.cookie_header_bytes(), + cookie.len(), + "should hold the inbound Cookie header size" + ); + assert_eq!(measurement.kv_reads(), 1, "should count the phase 1 read"); + assert_eq!(measurement.kv_writes(), 1, "should count the one write"); + assert_eq!( + measurement.route(), + RouteKind::Writer, + "should label a returning navigation as the writer" + ); + assert_eq!( + measurement.flush(), + "written", + "should hold the phase 4 outcome" + ); + assert_eq!(measurement.removal_lost(), 0, "should lose no removal"); + assert!( + !measurement.withdrawal_after_send() && !measurement.withdrawal_after_send_failed(), + "should need no after-send withdrawal run" + ); + } + + #[test] + fn identity_measurement_holds_no_identifier() { + let (state, _store) = seeded_state(); + let cookie = format!("ts-ec={EC_ID}; ts-eids=secret-uid-value"); + + let (_, measurement) = drive_measured_for_test(&state, navigation(Some(&cookie))); + let line = measurement.to_string(); + + assert!( + line.contains(&format!("cookie_header_bytes={}", cookie.len())), + "should still carry the cookie size: {line}" + ); + assert!(!line.contains(EC_ID), "should not hold the EC ID: {line}"); + assert!( + !line.contains("secret-uid-value"), + "should not hold a cookie value: {line}" + ); + } + #[test] fn drive_request_for_test_counts_routing_finalize_and_flush_against_one_store() { let (state, store) = seeded_state(); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 21c7f49b45..8106c6383d 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -243,49 +243,13 @@ fn queue_browser_ingest(ec_context: &mut EcContext, ec_id: &str, updates: &[Part } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct EidSyncMeasurement { - source: EidSyncSource, - outcome: EidCookieSyncOutcome, - already_matched: u8, - written: u8, - conflict_duplicate: u8, - deferred: u8, -} - -impl EidSyncMeasurement { - fn new(source: EidSyncSource, outcome: EidCookieSyncOutcome) -> Self { - Self { - source, - outcome, - already_matched: u8::from(matches!(outcome, EidCookieSyncOutcome::AlreadyMatched)), - written: u8::from(matches!( - outcome, - EidCookieSyncOutcome::Written | EidCookieSyncOutcome::WrittenWithDeferredFreshness - )), - conflict_duplicate: u8::from(matches!(outcome, EidCookieSyncOutcome::ConflictMatched)), - deferred: u8::from(matches!( - outcome, - EidCookieSyncOutcome::WrittenWithDeferredFreshness - | EidCookieSyncOutcome::DeferredConflict - | EidCookieSyncOutcome::DeferredFreshness - | EidCookieSyncOutcome::DeferredStaleRead - )), - } - } -} - +/// Logs the one remaining `EID sync measurement` line: a new EC whose +/// create carried the browser ingest. Returning visitors log the +/// `identity request measurement` line instead. fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome) { - let measurement = EidSyncMeasurement::new(source, outcome); log::info!( - "EID sync measurement: source={} outcome={} attempted=1 already_matched={} written={} \ - conflict_duplicate={} deferred={}", - measurement.source, - measurement.outcome, - measurement.already_matched, - measurement.written, - measurement.conflict_duplicate, - measurement.deferred, + "EID sync measurement: source={source} outcome={outcome} attempted=1 already_matched=0 \ + written=1 conflict_duplicate=0 deferred=0", ); } @@ -1427,91 +1391,16 @@ mod tests { EidSyncSource::PageBids, EidSyncSource::NewEc, ]; - let outcomes = [ - EidCookieSyncOutcome::AlreadyMatched, - EidCookieSyncOutcome::Written, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - EidCookieSyncOutcome::ConflictMatched, - EidCookieSyncOutcome::DeferredConflict, - EidCookieSyncOutcome::DeferredFreshness, - EidCookieSyncOutcome::DeferredStaleRead, - EidCookieSyncOutcome::Missing, - EidCookieSyncOutcome::ConsentWithdrawn, - EidCookieSyncOutcome::Failed, - ]; assert_eq!( sources.map(|source| source.to_string()), - ["navigation", "auction", "page_bids", "new_ec"] + ["navigation", "auction", "page_bids", "new_ec"], + "should label each source with a bounded name" ); assert_eq!( - outcomes.map(|outcome| outcome.to_string()), - [ - "already_matched", - "written", - "written_with_deferred_freshness", - "conflict_matched", - "deferred_conflict", - "deferred_freshness", - "deferred_stale_read", - "missing", - "consent_withdrawn", - "failed", - ] - ); - - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Navigation, - EidCookieSyncOutcome::AlreadyMatched, - ), - EidSyncMeasurement { - source: EidSyncSource::Navigation, - outcome: EidCookieSyncOutcome::AlreadyMatched, - already_matched: 1, - written: 0, - conflict_duplicate: 0, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Auction, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - ), - EidSyncMeasurement { - source: EidSyncSource::Auction, - outcome: EidCookieSyncOutcome::WrittenWithDeferredFreshness, - already_matched: 0, - written: 1, - conflict_duplicate: 0, - deferred: 1, - } - ); - assert_eq!( - EidSyncMeasurement::new(EidSyncSource::NewEc, EidCookieSyncOutcome::ConflictMatched,), - EidSyncMeasurement { - source: EidSyncSource::NewEc, - outcome: EidCookieSyncOutcome::ConflictMatched, - already_matched: 0, - written: 0, - conflict_duplicate: 1, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::PageBids, - EidCookieSyncOutcome::DeferredStaleRead, - ), - EidSyncMeasurement { - source: EidSyncSource::PageBids, - outcome: EidCookieSyncOutcome::DeferredStaleRead, - already_matched: 0, - written: 0, - conflict_duplicate: 0, - deferred: 1, - } + EidCookieSyncOutcome::Written.to_string(), + "written", + "should label the one outcome the new-EC line reports" ); } diff --git a/crates/trusted-server-core/src/ec/identity/measurement.rs b/crates/trusted-server-core/src/ec/identity/measurement.rs new file mode 100644 index 0000000000..bc8ad8933a --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/measurement.rs @@ -0,0 +1,162 @@ +//! The one `identity request measurement` line per request (spec 14, step 1). +//! +//! It carries sizes, counts and labels only: no EC ID, no UID and no cookie +//! value can reach it (spec 9.3). + +use super::flush::FlushOutcome; + +/// What kind of identity route the request was. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum RouteKind { + /// The one browser-driven writer request. + #[display("writer")] + Writer, + /// Any other request: it reads and stages but never writes after send. + #[display("stager")] + Stager, + /// The request generated a new EC. + #[display("new_ec")] + NewEc, +} + +/// What the adapter knows about the request after phase 4. +#[derive(Debug, Clone, Copy)] +pub struct MeasurementInput<'a> { + /// Whether phase 1 read the identity row. + pub phase_one_read: bool, + /// KV reads made after phase 1. + pub kv_reads: u32, + /// KV writes made after phase 1. + pub kv_writes: u32, + /// How phase 4 ended, or `None` when it did not run. + pub flush: Option<&'a FlushOutcome>, + /// Whether a withdrawal tombstone needed its after-send run. + pub withdrawal_after_send: bool, + /// Whether that run left the tombstone unwritten. + pub withdrawal_after_send_failed: bool, +} + +/// The measurement of one request, formatted by [`core::fmt::Display`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IdentityMeasurement { + pub(crate) cookie_header_bytes: usize, + pub(crate) route: RouteKind, + pub(crate) kv_reads: u32, + pub(crate) kv_writes: u32, + pub(crate) flush: String, + pub(crate) removal_lost: u32, + pub(crate) withdrawal_after_send: bool, + pub(crate) withdrawal_after_send_failed: bool, +} + +impl IdentityMeasurement { + /// The inbound `Cookie` header size in bytes. + #[must_use] + pub fn cookie_header_bytes(&self) -> usize { + self.cookie_header_bytes + } + + /// The route kind label. + #[must_use] + pub fn route(&self) -> RouteKind { + self.route + } + + /// KV reads this request made. + #[must_use] + pub fn kv_reads(&self) -> u32 { + self.kv_reads + } + + /// KV writes this request made. + #[must_use] + pub fn kv_writes(&self) -> u32 { + self.kv_writes + } + + /// The phase 4 outcome label. + #[must_use] + pub fn flush(&self) -> &str { + &self.flush + } + + /// Removals every attempt failed to write. + #[must_use] + pub fn removal_lost(&self) -> u32 { + self.removal_lost + } + + /// Whether a withdrawal tombstone needed its after-send run. + #[must_use] + pub fn withdrawal_after_send(&self) -> bool { + self.withdrawal_after_send + } + + /// Whether the after-send withdrawal run left the tombstone unwritten. + #[must_use] + pub fn withdrawal_after_send_failed(&self) -> bool { + self.withdrawal_after_send_failed + } + + /// Logs the measurement at info level. + pub fn log(&self) { + log::info!("identity request measurement: {self}"); + } +} + +impl core::fmt::Display for IdentityMeasurement { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + formatter, + "cookie_header_bytes={} route={} kv_reads={} kv_writes={} flush={} removal_lost={} \ + withdrawal_after_send={} withdrawal_after_send_failed={}", + self.cookie_header_bytes, + self.route, + self.kv_reads, + self.kv_writes, + self.flush, + self.removal_lost, + u8::from(self.withdrawal_after_send), + u8::from(self.withdrawal_after_send_failed), + ) + } +} + +/// The label for a phase 4 outcome: lower snake case, with a drop reason +/// appended. +pub(crate) fn flush_label(flush: Option<&FlushOutcome>) -> String { + match flush { + None => "not_run".to_owned(), + Some(FlushOutcome::Skipped) => "skipped".to_owned(), + Some(FlushOutcome::NoChange) => "no_change".to_owned(), + Some(FlushOutcome::Written) => "written".to_owned(), + Some(FlushOutcome::Dropped(reason)) => { + format!("dropped_{}", reason.replace(' ', "_")) + } + Some(FlushOutcome::WriteFailed { .. }) => "write_failed".to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flush_label_covers_every_outcome() { + assert_eq!( + flush_label(None), + "not_run", + "should label a skipped phase 4" + ); + assert_eq!( + flush_label(Some(&FlushOutcome::Dropped("read failed"))), + "dropped_read_failed", + "should join a drop reason with underscores" + ); + assert_eq!( + flush_label(Some(&FlushOutcome::Written)), + "written", + "should label a landed write" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index 00076b73d1..a745c5085c 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -15,6 +15,7 @@ pub mod flush; pub(crate) mod generic_pull; pub mod ids; pub mod input; +pub mod measurement; pub(crate) mod mirror; pub(crate) mod mutation; pub(crate) mod policy; diff --git a/crates/trusted-server-core/src/ec/identity/request.rs b/crates/trusted-server-core/src/ec/identity/request.rs index b31e71c2d3..2cdff075f0 100644 --- a/crates/trusted-server-core/src/ec/identity/request.rs +++ b/crates/trusted-server-core/src/ec/identity/request.rs @@ -40,6 +40,7 @@ pub(crate) struct RequestIdentity { mutation: IdentityMutation, pending_removals: Option, removal_lost: u32, + cookie_header_bytes: usize, capture_unpersisted: BTreeMap<(IdentityProviderId, SourceDomain), u32>, } @@ -124,6 +125,17 @@ impl RequestIdentity { self.removal_lost } + /// The inbound `Cookie` header size in bytes. + #[must_use] + pub(crate) fn cookie_header_bytes(&self) -> usize { + self.cookie_header_bytes + } + + /// Records the inbound `Cookie` header size in bytes. + pub(crate) fn set_cookie_header_bytes(&mut self, bytes: usize) { + self.cookie_header_bytes = bytes; + } + /// Counts one removal that every attempt failed to write. pub(crate) fn record_removal_lost(&mut self) { self.removal_lost = self.removal_lost.saturating_add(1); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 80d80330c6..4fc200788c 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -20,7 +20,10 @@ use error_stack::{Report, ResultExt}; use crate::error::TrustedServerError; use super::generation::ec_hash; -use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; +use super::kv_backend::{ + EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, KvOperationCounts, + OperationCountingEcKv, +}; use super::kv_types::{KvEntry, KvMetadata}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; @@ -99,41 +102,13 @@ impl PartnerIdUpdate { } } -/// Terminal result of one browser EID-cookie persistence attempt. -// Temporary: the browser-ingest write path moved into the identity mutation; a later identity task deletes this. -#[allow(dead_code)] +/// Terminal result of the browser EID-cookie persistence the `EID sync +/// measurement` line still reports: a new EC's create. #[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] pub(crate) enum EidCookieSyncOutcome { - /// The stored values already matched without a write. - #[display("already_matched")] - AlreadyMatched, - /// The one conditional write succeeded. + /// The new EC's create stored the browser ingest. #[display("written")] Written, - /// The write added missing IDs while deferring different values of unknown freshness. - #[display("written_with_deferred_freshness")] - WrittenWithDeferredFreshness, - /// A conflicting writer persisted every desired value. - #[display("conflict_matched")] - ConflictMatched, - /// A conflict left at least one desired value absent or different. - #[display("deferred_conflict")] - DeferredConflict, - /// A different stored value had unknown freshness. - #[display("deferred_freshness")] - DeferredFreshness, - /// An eventually consistent read missed a row already proven to exist. - #[display("deferred_stale_read")] - DeferredStaleRead, - /// The identity graph row was missing. - #[display("missing")] - Missing, - /// Consent had been withdrawn in the identity graph. - #[display("consent_withdrawn")] - ConsentWithdrawn, - /// KV or serialization failed. - #[display("failed")] - Failed, } pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { @@ -203,6 +178,15 @@ impl KvIdentityGraph { } } + /// Returns a graph over the same store that also counts its operations + /// into `counts`. + #[must_use] + pub fn with_operation_counts(self, counts: &KvOperationCounts) -> Self { + Self { + store: Arc::new(OperationCountingEcKv::new(self.store, counts.clone())), + } + } + /// Returns the configured store name. #[must_use] pub fn store_name(&self) -> &str { diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index 46d589cd45..1e899577f3 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -11,6 +11,8 @@ //! expressed through [`EcKvWriteMode`] so compare-and-swap loops stay in //! core while the platform supplies the actual precondition mechanics. +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use error_stack::Report; @@ -174,6 +176,89 @@ pub trait EcKvStore: Send + Sync { fn delete(&self, key: &str) -> Result<(), Report>; } +/// Counts of KV operations one request makes, shared by every clone. +/// +/// Reads are row lookups and writes are inserts and deletes. Existence +/// checks and key listings are not counted: they read no row. A graph reports into it through +/// [`KvIdentityGraph::with_operation_counts`](super::kv::KvIdentityGraph::with_operation_counts). +#[derive(Debug, Clone, Default)] +pub struct KvOperationCounts { + reads: Arc, + writes: Arc, +} + +impl KvOperationCounts { + /// Reads counted so far. + #[must_use] + pub fn reads(&self) -> u32 { + self.reads.load(Ordering::Relaxed) + } + + /// Writes counted so far. + #[must_use] + pub fn writes(&self) -> u32 { + self.writes.load(Ordering::Relaxed) + } + + fn record_read(&self) { + self.reads.fetch_add(1, Ordering::Relaxed); + } + + fn record_write(&self) { + self.writes.fetch_add(1, Ordering::Relaxed); + } +} + +/// An [`EcKvStore`] that counts every operation it forwards. +pub(crate) struct OperationCountingEcKv { + inner: Arc, + counts: KvOperationCounts, +} + +impl OperationCountingEcKv { + /// Wraps `inner`, reporting into `counts`. + pub(crate) fn new(inner: Arc, counts: KvOperationCounts) -> Self { + Self { inner, counts } + } +} + +impl EcKvStore for OperationCountingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.counts.record_read(); + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.counts.record_write(); + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.counts.record_write(); + self.inner.delete(key) + } +} + /// An in-memory store for tests, reachable from other crates through the /// `test-utils` feature. #[cfg(any(test, feature = "test-utils"))] diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 71330910c0..338a1e4d9c 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -88,17 +88,22 @@ use crate::settings::Settings; use device::DeviceSignals; use self::finalize::BrowserSeed; +use self::identity::measurement::{IdentityMeasurement, MeasurementInput, RouteKind, flush_label}; use self::identity::request::{RequestIdentity, WriterDecision}; use self::identity::runtime::IdentityRuntime; use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; use self::kv_types::KvEntry; -/// Bounded request classifications that may persist browser EID cookies. +/// Bounded request classifications that may stage browser EID cookies. /// /// Adapters classify publisher navigations and `POST /auction` only after /// pre-route filters allow dispatch. The shared page-bids handler classifies an /// admitted SPA navigation, while EC finalization classifies new identities. -/// Challenged or blocked requests remain unclassified and cannot persist EIDs. +/// Challenged or blocked requests remain unclassified and cannot stage EIDs. +/// +/// Only [`Self::NewEc`] still reports an `EID sync measurement` line. A +/// returning visitor's request reports the `identity request measurement` +/// line instead, which carries no source label. #[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] pub enum EidSyncSource { /// Publisher top-level document navigation. @@ -353,7 +358,17 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, eid_sync_source: None, - identity: RequestIdentity::default(), + identity: { + let mut identity = RequestIdentity::default(); + identity.set_cookie_header_bytes( + req.headers() + .get_all(header::COOKIE) + .iter() + .map(|value| value.as_bytes().len()) + .sum(), + ); + identity + }, failed_withdrawals: Vec::new(), }) } @@ -509,6 +524,34 @@ impl EcContext { self.device_signals = Some(signals); } + /// Builds the `identity request measurement` for this request after + /// phase 4 (spec 14, step 1). + /// + /// The route kind is `new_ec` for a request that generated an EC, + /// `writer` for the one browser-driven writer, and `stager` otherwise. + #[must_use] + pub fn identity_measurement(&self, input: &MeasurementInput<'_>) -> IdentityMeasurement { + let route = if self.ec_generated { + RouteKind::NewEc + } else if self.identity.writer() { + RouteKind::Writer + } else { + RouteKind::Stager + }; + IdentityMeasurement { + cookie_header_bytes: self.identity.cookie_header_bytes(), + route, + kv_reads: input + .kv_reads + .saturating_add(u32::from(input.phase_one_read)), + kv_writes: input.kv_writes, + flush: flush_label(input.flush), + removal_lost: self.identity.removal_lost(), + withdrawal_after_send: input.withdrawal_after_send, + withdrawal_after_send_failed: input.withdrawal_after_send_failed, + } + } + /// The identity state this request decides and collects. #[must_use] pub(crate) fn identity(&self) -> &RequestIdentity { diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 721226f38b..28cf68967a 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -212,6 +212,22 @@ curl -X POST https://edge.example.com/auction \ -d '{"adUnits":[{"code":"banner","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' ``` +#### `ts_eids_s` mirror payload + +The endpoint never writes KV. The response may carry `ext.ts_eids_s`, the +payload for the client-only `__Host-ts-eids-s` mirror cookie that the shim +writes for page scripts. It has one of three forms: + +- `{ "value": "", "max_age": 86400 }` when the request read a live, + consenting row and the servable server IDs are not empty. `value` is the + unpadded base64url encoding of compact JSON `{"v":1,"eids":[...]}`, at most + 2,048 bytes. `max_age` is at most 86,400 seconds and never past the earliest + expiry of an ID in the set. +- `{ "clear": true }` when consent denies EID persistence, when the row is a + tombstone or missing, or when the servable set is empty. The shim expires + the cookie. +- Absent when the request read no KV. The shim leaves the cookie as it is. + #### Browser EIDs (`eids`) The optional `eids` array carries OpenRTB 2.6 EID objects from the browser: @@ -332,6 +348,16 @@ any `OPTIONS` returns `403`; missing `[creative_opportunities]` returns `404`. The `200`, `400`, and `403` terminal responses are `private, no-store`. The endpoint has no built-in authentication, CORS grant, or rate limiter. +**Identity:** the endpoint never writes KV. Its JSON body may carry a +`ts_eids_s` member, the payload for the client-only `__Host-ts-eids-s` mirror +cookie (see [`ext.ts_eids_s`](#ts-eids-s-mirror-payload) under `POST /auction` +for its shape): + +- `{ "value": "...", "max_age": 86400 }`: write the cookie with this value and + `Max-Age`. +- `{ "clear": true }`: expire the cookie. +- Absent: the request read no KV, so leave the cookie as it is. + **Configuration:** `[creative_opportunities]`, `[auction]`, slot page patterns, provider profiles, and consent settings. Bot and prefetch requests retain slot shape but skip live provider calls. @@ -369,6 +395,8 @@ below `[publisher].domain` receive reflected credentialed CORS for The endpoint is Fastly-only and requires the EC store and a statically configured `[[ec.partners]]` token. +**No writes:** the endpoint never writes KV, and it omits an ID that has expired. The response shape is unchanged: one UID per source, even for a record that holds several. + **Request:** - Uses `ts-ec` cookie and consent signals diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index fa436e2ac4..e141f1d133 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -636,28 +636,50 @@ Legacy consent-store records are not read or migrated into `ec.ec_store`. Their ### Identity Providers (`[ec.identity]`) -Policy for server-side identity providers. The table is optional; every key has a default. `ec.pull_sync_concurrency` was removed: a config that sets it fails validation. Use `max_concurrency` instead. - -| Field | Type | Default | Description | -| -------------------- | ------- | ------- | ----------------------------------------------------------------------- | -| `max_concurrency` | Integer | `4` | Maximum providers dispatched at once (at least 1) | -| `hold_budget_ms` | Integer | `100` | Total time a navigation may wait for hold-mode providers (at most 1000) | -| `hold_retry_sec` | Integer | `300` | Seconds before a provider that timed out in hold mode is held again | -| `max_response_bytes` | Integer | `65536` | Largest provider response body accepted | - -Each provider can have an `[ec.identity.providers.]` table: - -| Field | Type | Default | Description | -| ----------------------------- | ------- | ------------ | ------------------------------------------------------------------ | -| `mode` | String | `after_send` | `hold` or `after_send` | -| `timeout_ms` | Integer | none | Hold-mode timeout, at most `hold_budget_ms`; the budget when unset | -| `after_send_timeout_ms` | Integer | `1000` | Timeout for any dispatch after send, `1` to `10000` | -| `sources` | Array | none | Source domains the provider may write | -| `refresh_after_sec` | Integer | `86400` | Seconds before a stored ID is refreshed (greater than 0) | -| `max_calls_per_minute` | Integer | `1` | Provider calls allowed per minute | -| `breaker_failures_per_minute` | Integer | `50` | Failures per minute that open the circuit breaker | -| `cookie_priority` | Integer | `0` | Ordering of the provider's ID when the cookie is trimmed | -| `revoke_scope` | String | `provider` | `provider` or `global` | +Policy for server-side identity providers. The table is optional; every key has a default. + +| Key | Type | Default | Meaning | +| -------------------- | ------- | ------- | -------------------------------------------------------------------------------------- | +| `max_concurrency` | Integer | `4` | Most providers dispatched at once (at least 1). Replaces `ec.pull_sync_concurrency`. | +| `hold_budget_ms` | Integer | `100` | Total wall-clock time a navigation may wait for all hold-mode providers (at most 1000) | +| `hold_retry_sec` | Integer | `300` | Seconds to wait before calling a provider again after the hold budget cut its call off | +| `max_response_bytes` | Integer | `65536` | Largest provider response body accepted, before and after decoding | + +Each provider can have an `[ec.identity.providers.]` table. A provider with no table uses the defaults. + +| Key | Type | Default | Meaning | +| ----------------------------- | ------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `mode` | String | `after_send` | `hold` runs before the navigation's auction; `after_send` runs after the response is sent. Providers run only on a returning visitor's page load. | +| `timeout_ms` | Integer | `hold_budget_ms` | Hold-mode call timeout, at most `hold_budget_ms` | +| `after_send_timeout_ms` | Integer | `1000` | Timeout for any call made after send, `1` to `10000` | +| `sources` | Array | every source the provider declares | Narrows the source domains the provider may write. Required for a capture-only provider. | +| `refresh_after_sec` | Integer | `86400` | Seconds before a stored ID is refreshed, and the wait after a skipped call (greater than 0) | +| `max_calls_per_minute` | Integer | `1` | Per-visitor burst cap, enforced in 60-second windows | +| `breaker_failures_per_minute` | Integer | `50` | Transport failures, timeouts, 429s and 5xx responses in one 60-second window after which the provider is skipped for every visitor | +| `cookie_priority` | Integer | `0` | Lower values are left out of the `__Host-ts-eids-s` mirror first when it exceeds its size cap | +| `revoke_scope` | String | `provider` (capture only) | `provider` or `global`. Applies only to a provider that captures IDs, from Part 2 of the identity rollout. | + +**Pull partner provider IDs.** Each `[[ec.partners]]` entry with `pull_sync_enabled = true` registers a provider whose ID is `pull-` plus the source domain with dots replaced by `-`. For example, `ssp.example.com` is `pull-ssp-example-com`. A domain that contains `-`, or an ID that would be longer than 63 characters, also gets `-` and the first 8 hex digits of the SHA-256 of the full domain, so `a-b.example.com` and `a.b.example.com` get different IDs. Configure the partner's pacing in `[ec.identity.providers.pull-ssp-example-com]`. + +**Removed keys.** Validation rejects each of these with an error that names its replacement: + +| Removed key | Replacement | +| ------------------------------------------ | ---------------------------------------------------------------------------- | +| `ec.pull_sync_concurrency` | `[ec.identity].max_concurrency` | +| `[[ec.partners]].pull_sync_ttl_sec` | `refresh_after_sec` in `[ec.identity.providers.]` | +| `[[ec.partners]].pull_sync_rate_limit = N` | `max_calls_per_minute = ceil(N / 60)` in `[ec.identity.providers.]` | + +The old rate limit counted calls per hour; the new one counts them per minute and defaults to 1, so a partner that kept the old default of 10 per hour is now paced at an effective 60 per hour. + +#### Rolling back + +This release moves the KV identity row to schema version 2. A release before it cannot read a version 2 row, so rolling back past this release is unsupported once it has written one; going back needs a KV migration to version 1, which is not provided. + +A later release that adds vendor providers can be rolled back to this one with these steps: + +- Remove `owner` and every vendor `[ec.identity.providers.*]` table from the config first. This release rejects an `owner` or a provider table it does not know. +- Complete entry sets that a vendor wrote stay readable, but they are not served while the provider that wrote them is not configured. +- A write prunes an unconfigured provider's state down to its opt-out fields, so opt-outs survive. Its invalidated-value list and its provider state do not, so invalidated values can be seeded once more and stateful providers start over when rolled forward again. ### `[ec]` diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index bd917df473..11ec21b1d1 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -82,6 +82,15 @@ api_token = "partner_api_token" bidstream_enabled = true ``` +Pull partners (`pull_sync_enabled = true`) run from a visitor's second page +load, because they run after send on a returning visitor's page navigation. A +partner that returns an error backs off instead of being called on every +request, and its pace comes from its `[ec.identity.providers.]` +table, for example `pull-ssp-example-com` for `ssp.example.com`: set +`refresh_after_sec` and `max_calls_per_minute` there. The old +`pull_sync_ttl_sec` and `pull_sync_rate_limit` keys are rejected; see the +[Configuration Reference](/guide/configuration#identity-providers-ec-identity). + Provision the bearer token value under `partner_api_token`, then deploy or restart after changing partner configuration. The token is required for this demo because it exercises the inbound partner APIs. diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 57121a0b36..fa7b5b63c0 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -61,11 +61,11 @@ sequenceDiagram TS-->>B: Response + Set-Cookie: ts-ec=... else Return Visit (EC cookie present) Note over TS: Phase 2: Routing
EC exists — skip generation - Note over TS: Phase 3: Finalize
Ingest Prebid EID cookies + Note over TS: Phase 3: Finalize
Stage Prebid EID cookies
(written once in phase 4) TS-->>B: Response
(no cookie refresh) end - Note over TS,KV: Phase 4: Post-send (background)
Dispatch pull-sync to partners + Note over TS,KV: Phase 4: Post-send (background)
Writer request only: call due pull partners,
then one KV write ``` ### Response Finalization @@ -83,7 +83,7 @@ flowchart TD ExplicitWithdrawal -- "No" --> HeaderOnly ConsentCheck -- "Yes" --> WasPresent{EC was present
in request?} - WasPresent -- "Yes, not generated" --> Returning["Ingest Prebid EID cookies
No cookie or KV TTL refresh"] + WasPresent -- "Yes, not generated" --> Returning["Stage Prebid EID cookies for the writer's one flush
No cookie or KV TTL refresh"] WasPresent -- "No, just generated" --> NewEc["Ingest Prebid EID cookies
Set ts-ec cookie"] ``` @@ -270,15 +270,29 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner Browser EIDs are forwarded as sent; KV adds EIDs only for registered sources the browser did not send. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). -### Pull-Sync Completeness Marker +### One KV Write per Page Load + +Only one request per page view writes browser-driven identity changes to KV: a full page navigation from a returning visitor (a request that already carries a valid `ts-ec`). Every other route reads KV and stages its changes but never writes them: + +- `/auction`, `/_ts/page-bids` and `/_ts/api/v1/identify` no longer write KV. +- The writer request collects every change from the page load: browser EIDs from the `ts-eids` and `sharedId` cookies, pull partner results and the cluster size. It sends the response first, then writes everything once, after send, with a generation check. A failed write is not retried as a whole. +- A new visitor's first request still creates the row before routing, as before. +- Two kinds of write are exempt because a withdrawal or removal must not be lost: the consent-withdrawal tombstone (with one more attempt after send when it fails before send), and a retry that writes only the removals of a failed flush. + +Pull partners run in the same after-send step, from a visitor's second page load, because they need a returning visitor's row. A partner that fails backs off instead of being called on every request, and its pace is set by its `[ec.identity.providers.]` table (see the [Configuration Reference](/guide/configuration#identity-providers-ec-identity)). + +The `ts-ec-pull-complete` completeness cookie no longer exists. Trusted Server neither sets nor reads it. A copy that a browser still holds is ignored and expires on its own within its one-hour `Max-Age`. -When the identity graph contains a UID for every pull-enabled partner, Trusted Server sets a signed, host-only `ts-ec-pull-complete` cookie. The cookie contains no partner UID or EC ID. It authenticates a one-hour expiration and a fingerprint of the current pull-partner source-domain set, bound to the active EC ID with key material derived from `ec.passphrase`. +Each request also logs one `identity request measurement: ...` line after the response is sent. It carries the inbound `Cookie` header size in bytes, KV reads and writes labelled by route kind (`writer`, `stager` or `new_ec`), the flush outcome, the number of lost removals, and whether a withdrawal tombstone needed its after-send run and failed. It holds no EC ID, UID or cookie value. It replaces the `EID sync measurement: ...` line for returning visitors; that line is now logged only for a new EC. -A valid marker avoids a KV lookup only when pull-sync completeness is the sole reason to inspect the row. Auctions that need stored EIDs, browser EID-cookie ingestion, explicit withdrawal, and generation continue to use KV. The marker acts as recent proof that the row existed, so deletion of a previously complete row is not detected until the marker expires, at most one hour after issuance. Partner-set changes, passphrase rotation, malformed values, and expiration invalidate the marker and restore the normal lookup and orphan-recovery path. +### The `ts-eids-s` Mirror Cookie -Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal expires both `ts-ec` and any present `ts-ec-pull-complete` marker even when KV is unavailable. +`__Host-ts-eids-s` is a client-only mirror of the server-side IDs for page scripts. The shim writes it from the `ext.ts_eids_s` member of `/auction` responses and the `ts_eids_s` member of `/_ts/page-bids` responses (see the [API Reference](/guide/api-reference)). The server never sets, reads or trusts it, and it strips it from requests to the origin and to Prebid Server. -Issuing or expiring the marker adds `Set-Cookie` to the outgoing response. Cache-privacy handling makes an otherwise shareable response private when that happens. A valid marker is not refreshed on each request, so this cost is limited to responses that establish or clear marker state in exchange for avoiding later KV reads. +- Attributes: `Path=/; Secure; SameSite=Lax`, no `Domain`, and not `HttpOnly`, because the shim writes it with `document.cookie`. +- Lifetime: a `Max-Age` of at most one day, and never past the earliest expiry of an ID it holds. +- Size: at most 2,048 bytes. When the IDs do not fit, whole sources are left out, lowest `cookie_priority` first, never part of a source. +- The shim's `ts-eids` trim now keeps registered sources first, so a size-capped `ts-eids` loses unregistered sources before registered ones. ## Configuration @@ -327,7 +341,7 @@ receives any Trusted Server identity cookie (`ts-ec`, `ts-eids`, or - When consent is blocked but not explicitly withdrawn, Trusted Server strips EC response headers for that request but leaves any existing `ts-ec` cookie intact; cookie expiry and tombstones happen only on explicit withdrawal. - `/_ts/api/v1/identify` is read-oriented and returns identity enrichment for the authenticated partner. It computes `cluster_size` only when the EC entry does not already store one. - `/_ts/api/v1/batch-sync` validates every input, groups valid mappings by normalized EC ID, and applies the last valid UID for each group once. Group outcomes still account for every original input; infrastructure failures reject the failing and remaining groups. Mapping timestamps remain required for API compatibility but do not order writes. See the [API Reference](/guide/api-reference) for the complete contract. -- Pull sync fills missing partner UIDs only. Existing partner UIDs are not periodically refreshed because EC entries no longer store per-partner sync timestamps. +- Pull sync fills missing partner UIDs only, from a returning visitor's second page load. Existing partner UIDs are not periodically refreshed because EC entries no longer store per-partner sync timestamps. ## Next Steps diff --git a/trusted-server.example.toml b/trusted-server.example.toml index d84e455c27..c362b5e22b 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -83,10 +83,12 @@ ec_store = "ec_identity_store" # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds -# Server-side identity provider policy (defaults shown except max_concurrency). -[ec.identity] -# Max identity providers dispatched at once. -max_concurrency = 3 +# Server-side identity provider policy. Every key is optional; the values +# below are the defaults. +# [ec.identity] +# Max identity providers dispatched at once. Replaces the removed +# ec.pull_sync_concurrency. +# max_concurrency = 4 # Total time a navigation may wait for hold-mode providers (at most 1000). # hold_budget_ms = 100 # Seconds before a provider that timed out in hold mode is held again. @@ -94,8 +96,13 @@ max_concurrency = 3 # Largest provider response body accepted, in bytes. # max_response_bytes = 65536 -# Per-provider policy; the table name is the identity provider ID. -# [ec.identity.providers.example-provider] +# Per-provider policy; the table name is the identity provider ID. A pull +# partner (`pull_sync_enabled = true`) registers as `pull-` plus its source +# domain with dots replaced by `-`, so `ssp.example.com` is +# `pull-ssp-example-com`. These keys replace the removed per-partner +# `pull_sync_ttl_sec` (now `refresh_after_sec`) and `pull_sync_rate_limit = N` +# (now `max_calls_per_minute = ceil(N / 60)`). +# [ec.identity.providers.pull-ssp-example-com] # mode = "after_send" # or "hold" # timeout_ms = 100 # hold mode only; at most hold_budget_ms # after_send_timeout_ms = 1000 # 1..=10000 @@ -103,7 +110,8 @@ max_concurrency = 3 # max_calls_per_minute = 1 # breaker_failures_per_minute = 50 # cookie_priority = 0 -# revoke_scope = "provider" # or "global" +# `revoke_scope = "provider"` (or "global") applies only to a provider that +# captures IDs, from Part 2 of the identity rollout; do not set it here. # Example partner configuration. Provision referenced keys in # trusted_server_secrets before validating/pushing. From 42dac1bff1959f715a260a9911070b220a733845 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 00:59:47 +0530 Subject: [PATCH 44/89] Count the new-EC create write in the identity measurement Record each create attempt of the new-EC path on the request and add it to kv_writes, and skip the inferred phase 1 read when the snapshot came from that create. Keep the flush outcome as a value and format its label on display. Word the single-write docs as per EC-bearing request, and note in the changelog that the example config moves max_concurrency from 3 to 4 and that the pull rate is a ceiling. Signed-off-by: dhruv8sh --- CHANGELOG.md | 6 +- .../trusted-server-adapter-fastly/src/main.rs | 32 +++++++- .../src/ec/identity/measurement.rs | 79 ++++++++++++------- .../src/ec/identity/request.rs | 12 +++ crates/trusted-server-core/src/ec/mod.rs | 17 ++-- docs/guide/edge-cookies.md | 4 +- 6 files changed, 108 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c393e5cfdc..00af3f5699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Identity KV is written once per page load. Only a full page navigation from a returning visitor (the writer request) writes KV, once, after the response is sent: browser EIDs, pull partner results and the cluster size all go in that one write, with a generation check and no retry of the whole write. Every other route reads KV and stages but writes nothing. A consent-withdrawal tombstone and the removal-only retry of a failed write are the only exceptions. Mid-branch builds drop browser ingest until the whole change ships. +- Identity KV is written at most once per EC-bearing request. Only a full page navigation from a returning visitor (the writer request) writes KV, once, after the response is sent: browser EIDs, pull partner results and the cluster size all go in that one write, with a generation check and no retry of the whole write. Every other route reads KV and stages but writes nothing. A consent-withdrawal tombstone and the removal-only retry of a failed write are the only exceptions. Mid-branch builds drop browser ingest until the whole change ships. - `POST /auction`, `GET /_ts/page-bids` and `GET /_ts/api/v1/identify` no longer write KV, and `/_ts/api/v1/identify` omits expired IDs. The `/identify` response shape is unchanged: one UID per source. -- Pull partners now run from a visitor's second page load, from the writer request after send, and back off on errors. They are paced by `[ec.identity.providers.pull-]` (`refresh_after_sec`, `max_calls_per_minute`), so the old default of 10 calls per hour becomes an effective 60 per hour. Runtime startup now builds each pull partner's descriptor: a partner whose allowed host fails the descriptor host check stops startup. See [Identity Providers](docs/guide/configuration.md#identity-providers-ec-identity). -- **Breaking:** `ec.pull_sync_concurrency`, and `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]`, are removed, and validation now rejects them with an error that names the replacement. Use `[ec.identity].max_concurrency`, `refresh_after_sec`, and `max_calls_per_minute = ceil(N / 60)` for `pull_sync_rate_limit = N`, the last two in `[ec.identity.providers.]`. +- Pull partners now run from a visitor's second page load, from the writer request after send, and back off on errors. They are paced by `[ec.identity.providers.pull-]` (`refresh_after_sec`, `max_calls_per_minute`), so the old default of 10 calls per hour becomes an effective 60 per hour. That 60 per hour is a ceiling, not a rate: `refresh_after_sec` and error backoff gate each partner first. Runtime startup now builds each pull partner's descriptor: a partner whose allowed host fails the descriptor host check stops startup. See [Identity Providers](docs/guide/configuration.md#identity-providers-ec-identity). +- **Breaking:** `ec.pull_sync_concurrency`, and `pull_sync_ttl_sec` and `pull_sync_rate_limit` on `[[ec.partners]]`, are removed, and validation now rejects them with an error that names the replacement. Use `[ec.identity].max_concurrency` (the example config's value moves from 3 to the default 4), `refresh_after_sec`, and `max_calls_per_minute = ceil(N / 60)` for `pull_sync_rate_limit = N`, the last two in `[ec.identity.providers.]`. - `/auction` responses carry `ext.ts_eids_s` and `/_ts/page-bids` responses carry `ts_eids_s`: the payload (`value` with `max_age`, `clear`, or absent) for the client-only `__Host-ts-eids-s` mirror cookie, at most 2,048 bytes and one day. The shim's `ts-eids` size trim now keeps registered sources first. - The `ts-ec-pull-complete` cookie is removed: Trusted Server no longer sets or reads it, and a copy a browser holds expires within one hour. - **Breaking:** The KV identity row moves to schema version 2. A release before this one cannot read a version 2 row, so rolling back past this release is unsupported once it has written one. See [Rolling back](docs/guide/configuration.md#rolling-back). This release should merge after the EID-fidelity changes, whose behaviour its golden tests pin. diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 56e6dfc9b0..b1fa11baac 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1134,7 +1134,7 @@ mod tests { ); assert_eq!( measurement.flush(), - "written", + Some(FlushOutcome::Written), "should hold the phase 4 outcome" ); assert_eq!(measurement.removal_lost(), 0, "should lose no removal"); @@ -2225,6 +2225,36 @@ mod tests { with_header(navigation(cookie), name, value) } + #[test] + fn identity_measurement_counts_the_create_write_of_a_new_ec_navigation() { + let scenario = Scenario::new(); + let cookie = format!("sharedId={SHARED_ID}"); + + let (outcome, measurement) = + drive_measured_for_test(&scenario.state, navigation(Some(&cookie))); + + assert_eq!( + measurement.route(), + RouteKind::NewEc, + "should label a first visit as a new EC" + ); + assert_eq!( + measurement.kv_writes(), + 1, + "should count the create write and no other" + ); + assert_eq!( + measurement.kv_reads(), + 0, + "should count no read: the create reads nothing" + ); + assert_eq!( + outcome, + Some(FlushOutcome::Skipped), + "should not flush a new visitor" + ); + } + #[test] fn new_visitor_navigation_writes_once() { let scenario = Scenario::new(); diff --git a/crates/trusted-server-core/src/ec/identity/measurement.rs b/crates/trusted-server-core/src/ec/identity/measurement.rs index bc8ad8933a..7d9edf6190 100644 --- a/crates/trusted-server-core/src/ec/identity/measurement.rs +++ b/crates/trusted-server-core/src/ec/identity/measurement.rs @@ -3,6 +3,8 @@ //! It carries sizes, counts and labels only: no EC ID, no UID and no cookie //! value can reach it (spec 9.3). +use core::fmt::Write as _; + use super::flush::FlushOutcome; /// What kind of identity route the request was. @@ -43,7 +45,7 @@ pub struct IdentityMeasurement { pub(crate) route: RouteKind, pub(crate) kv_reads: u32, pub(crate) kv_writes: u32, - pub(crate) flush: String, + pub(crate) flush: Option, pub(crate) removal_lost: u32, pub(crate) withdrawal_after_send: bool, pub(crate) withdrawal_after_send_failed: bool, @@ -74,10 +76,10 @@ impl IdentityMeasurement { self.kv_writes } - /// The phase 4 outcome label. + /// How phase 4 ended, or `None` when it did not run. #[must_use] - pub fn flush(&self) -> &str { - &self.flush + pub fn flush(&self) -> Option { + self.flush } /// Removals every attempt failed to write. @@ -114,7 +116,7 @@ impl core::fmt::Display for IdentityMeasurement { self.route, self.kv_reads, self.kv_writes, - self.flush, + FlushLabel(self.flush), self.removal_lost, u8::from(self.withdrawal_after_send), u8::from(self.withdrawal_after_send_failed), @@ -122,41 +124,58 @@ impl core::fmt::Display for IdentityMeasurement { } } -/// The label for a phase 4 outcome: lower snake case, with a drop reason +/// A phase 4 outcome as a lower snake case label, with a drop reason /// appended. -pub(crate) fn flush_label(flush: Option<&FlushOutcome>) -> String { - match flush { - None => "not_run".to_owned(), - Some(FlushOutcome::Skipped) => "skipped".to_owned(), - Some(FlushOutcome::NoChange) => "no_change".to_owned(), - Some(FlushOutcome::Written) => "written".to_owned(), - Some(FlushOutcome::Dropped(reason)) => { - format!("dropped_{}", reason.replace(' ', "_")) +pub(crate) struct FlushLabel(pub(crate) Option); + +impl core::fmt::Display for FlushLabel { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0 { + None => formatter.write_str("not_run"), + Some(FlushOutcome::Skipped) => formatter.write_str("skipped"), + Some(FlushOutcome::NoChange) => formatter.write_str("no_change"), + Some(FlushOutcome::Written) => formatter.write_str("written"), + Some(FlushOutcome::Dropped(reason)) => { + formatter.write_str("dropped_")?; + reason.chars().try_for_each(|character| { + formatter.write_char(if character == ' ' { '_' } else { character }) + }) + } + Some(FlushOutcome::WriteFailed { .. }) => formatter.write_str("write_failed"), } - Some(FlushOutcome::WriteFailed { .. }) => "write_failed".to_owned(), } } #[cfg(test)] mod tests { use super::*; + use crate::ec::identity::removal::RemovalWriteOutcome; #[test] fn flush_label_covers_every_outcome() { - assert_eq!( - flush_label(None), - "not_run", - "should label a skipped phase 4" - ); - assert_eq!( - flush_label(Some(&FlushOutcome::Dropped("read failed"))), - "dropped_read_failed", - "should join a drop reason with underscores" - ); - assert_eq!( - flush_label(Some(&FlushOutcome::Written)), - "written", - "should label a landed write" - ); + let cases = [ + (None, "not_run"), + (Some(FlushOutcome::Skipped), "skipped"), + (Some(FlushOutcome::NoChange), "no_change"), + (Some(FlushOutcome::Written), "written"), + ( + Some(FlushOutcome::Dropped("read failed")), + "dropped_read_failed", + ), + ( + Some(FlushOutcome::WriteFailed { + removals: RemovalWriteOutcome::Failed, + }), + "write_failed", + ), + ]; + + for (outcome, expected) in cases { + assert_eq!( + FlushLabel(outcome).to_string(), + expected, + "should label {outcome:?}" + ); + } } } diff --git a/crates/trusted-server-core/src/ec/identity/request.rs b/crates/trusted-server-core/src/ec/identity/request.rs index 2cdff075f0..22a6e845a9 100644 --- a/crates/trusted-server-core/src/ec/identity/request.rs +++ b/crates/trusted-server-core/src/ec/identity/request.rs @@ -41,6 +41,7 @@ pub(crate) struct RequestIdentity { pending_removals: Option, removal_lost: u32, cookie_header_bytes: usize, + create_attempts: u32, capture_unpersisted: BTreeMap<(IdentityProviderId, SourceDomain), u32>, } @@ -125,6 +126,17 @@ impl RequestIdentity { self.removal_lost } + /// Create writes the new-EC path attempted before routing. + #[must_use] + pub(crate) fn create_attempts(&self) -> u32 { + self.create_attempts + } + + /// Counts one create write of the new-EC path. + pub(crate) fn record_create_attempt(&mut self) { + self.create_attempts = self.create_attempts.saturating_add(1); + } + /// The inbound `Cookie` header size in bytes. #[must_use] pub(crate) fn cookie_header_bytes(&self) -> usize { diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 338a1e4d9c..4ca631ead0 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -88,7 +88,7 @@ use crate::settings::Settings; use device::DeviceSignals; use self::finalize::BrowserSeed; -use self::identity::measurement::{IdentityMeasurement, MeasurementInput, RouteKind, flush_label}; +use self::identity::measurement::{IdentityMeasurement, MeasurementInput, RouteKind}; use self::identity::request::{RequestIdentity, WriterDecision}; use self::identity::runtime::IdentityRuntime; use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph, apply_partner_id_updates}; @@ -432,6 +432,7 @@ impl EcContext { apply_partner_id_updates(&mut entry, seed.updates()); if let Some(graph) = kv { + self.identity.record_create_attempt(); match graph.create_if_absent(&ec_id, &entry) { Ok(CreateIfAbsentOutcome::Written) => { self.kv_snapshot = EcKvSnapshot::Present { @@ -541,11 +542,15 @@ impl EcContext { IdentityMeasurement { cookie_header_bytes: self.identity.cookie_header_bytes(), route, - kv_reads: input - .kv_reads - .saturating_add(u32::from(input.phase_one_read)), - kv_writes: input.kv_writes, - flush: flush_label(input.flush), + // A snapshot that came from the new-EC create read nothing, and + // each create attempt is one write made before routing. + kv_reads: input.kv_reads.saturating_add(u32::from( + input.phase_one_read && self.identity.create_attempts() == 0, + )), + kv_writes: input + .kv_writes + .saturating_add(self.identity.create_attempts()), + flush: input.flush.copied(), removal_lost: self.identity.removal_lost(), withdrawal_after_send: input.withdrawal_after_send, withdrawal_after_send_failed: input.withdrawal_after_send_failed, diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index fa7b5b63c0..31e1085e8b 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -270,9 +270,9 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner Browser EIDs are forwarded as sent; KV adds EIDs only for registered sources the browser did not send. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). -### One KV Write per Page Load +### One KV Write per EC-Bearing Request -Only one request per page view writes browser-driven identity changes to KV: a full page navigation from a returning visitor (a request that already carries a valid `ts-ec`). Every other route reads KV and stages its changes but never writes them: +At most one request per page view writes browser-driven identity changes to KV: a full page navigation from a returning visitor (a request that already carries a valid `ts-ec`). Every other route reads KV and stages its changes but never writes them: - `/auction`, `/_ts/page-bids` and `/_ts/api/v1/identify` no longer write KV. - The writer request collects every change from the page load: browser EIDs from the `ts-eids` and `sharedId` cookies, pull partner results and the cluster size. It sends the response first, then writes everything once, after send, with a generation check. A failed write is not retried as a whole. From 639f9ad9993d8d7074f8acd1fde70a9afbe8b718 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 27 Oct 2026 01:08:30 +0530 Subject: [PATCH 45/89] Let integrations register identity providers Integrations can add identity providers to their registration, and the identity runtime and catalog now take the registered providers. Capture rules must lie under the registering integration's own routes. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 24 +- .../trusted-server-adapter-fastly/src/main.rs | 6 +- .../src/auction/endpoints.rs | 18 +- crates/trusted-server-core/src/config.rs | 22 +- crates/trusted-server-core/src/ec/admin.rs | 6 +- .../src/ec/identity/catalog.rs | 48 +++- .../src/ec/identity/examples.rs | 229 ++++++++++++++++++ .../src/ec/identity/flush.rs | 6 +- .../src/ec/identity/mod.rs | 4 + .../src/ec/identity/runtime.rs | 35 ++- .../src/integrations/registry.rs | 217 ++++++++++++++++- crates/trusted-server-core/src/publisher.rs | 7 +- 12 files changed, 582 insertions(+), 40 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/identity/examples.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index cc103cd5fe..0e8c02ee46 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -237,8 +237,24 @@ pub(crate) fn load_settings_from_config_store( /// Returns an error when a provider cannot be described or the catalog rules fail. pub(crate) fn build_identity_runtime( settings: &Settings, + integrations: &IntegrationRegistry, ) -> Result, Report> { - IdentityRuntime::from_settings(settings).map(Arc::new) + IdentityRuntime::from_settings(settings, integrations).map(Arc::new) +} + +/// Builds the identity runtime for settings loaded without application +/// state, from the integration registry those settings describe. +/// +/// # Errors +/// +/// Returns an error when the integration registry or the identity runtime +/// cannot be built. +pub(crate) fn build_identity_runtime_from_reloaded_settings( + settings: &Settings, +) -> Result, Report> { + let plan = Arc::new(compile_auction_plan(settings)?); + let registry = IntegrationRegistry::with_plan(settings, plan)?; + build_identity_runtime(settings, ®istry) } pub(crate) fn build_state_from_settings( @@ -250,7 +266,7 @@ pub(crate) fn build_state_from_settings( plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; let registry = IntegrationRegistry::with_plan(&settings, plan)?; - let identity = build_identity_runtime(&settings)?; + let identity = build_identity_runtime(&settings, ®istry)?; let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; @@ -1693,8 +1709,8 @@ mod tests { filters: Vec>, ) -> RouterService { let settings = test_settings(); - let identity = - build_identity_runtime(&settings).expect("should build the identity runtime"); + let identity = build_identity_runtime(&settings, &IntegrationRegistry::default()) + .expect("should build the identity runtime"); let plan = Arc::new( trusted_server_core::auction::compile_auction_plan(&settings) .expect("should compile auction plan"), diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index b1fa11baac..ba5b56d891 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -48,8 +48,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, build_identity_runtime, - load_settings_from_config_store, + AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, + build_identity_runtime_from_reloaded_settings, load_settings_from_config_store, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -540,7 +540,7 @@ fn edgezero_main(mut req: FastlyRequest, sandbox: &mut Sandbox, ordinal: u64, re // No application state was built, so the identity runtime // comes from the reloaded settings. Without one, finalize and // the withdrawal retry still run; only phase 4 is skipped. - let identity = match build_identity_runtime(&settings) { + let identity = match build_identity_runtime_from_reloaded_settings(&settings) { Ok(identity) => { ec_state .ec_context diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 0135af99d3..e86bf7b3d0 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1994,11 +1994,19 @@ mod tests { let mut settings = create_test_settings(); settings.ec.partners = vec![partner]; let with_provider = std::sync::Arc::new( - IdentityRuntime::from_settings(&settings).expect("should build the runtime"), + IdentityRuntime::from_settings( + &settings, + &crate::integrations::IntegrationRegistry::default(), + ) + .expect("should build the runtime"), ); settings.ec.partners = Vec::new(); let without_provider = std::sync::Arc::new( - IdentityRuntime::from_settings(&settings).expect("should build the runtime"), + IdentityRuntime::from_settings( + &settings, + &crate::integrations::IntegrationRegistry::default(), + ) + .expect("should build the runtime"), ); let ec_id = format!("{}.ABC123", "a".repeat(64)); @@ -2471,7 +2479,11 @@ mod tests { let mut settings = create_test_settings(); settings.ec.partners = vec![partner]; let runtime = std::sync::Arc::new( - IdentityRuntime::from_settings(&settings).expect("should build the runtime"), + IdentityRuntime::from_settings( + &settings, + &crate::integrations::IntegrationRegistry::default(), + ) + .expect("should build the runtime"), ); let registry = merge_registry(&["ssp.example.com"]); let ec_id = merge_ec_id(); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index d4c9a67ece..0cea15f394 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -13,6 +13,7 @@ use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; +use crate::ec::identity::ProviderDescriptor; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ @@ -269,7 +270,7 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result, Report> { + crate::ec::identity::generic_pull::generic_pull_descriptors(&settings.ec.partners) +} + fn identity_configuration_error(message: String) -> Report { Report::new(TrustedServerError::Configuration { message }) } diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index ed93077dd4..e47e0c1238 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -1135,7 +1135,11 @@ mod tests { .expect("should build registry"); let mut settings = create_test_settings(); settings.ec.partners = vec![partner]; - let runtime = IdentityRuntime::from_settings(&settings).expect("should build the runtime"); + let runtime = IdentityRuntime::from_settings( + &settings, + &crate::integrations::IntegrationRegistry::default(), + ) + .expect("should build the runtime"); let mut entry = KvEntry::minimal("other.example.com", "other", 1); entry.ids.insert( "ssp.example.com".to_owned(), diff --git a/crates/trusted-server-core/src/ec/identity/catalog.rs b/crates/trusted-server-core/src/ec/identity/catalog.rs index f14a849533..e4a12a15cc 100644 --- a/crates/trusted-server-core/src/ec/identity/catalog.rs +++ b/crates/trusted-server-core/src/ec/identity/catalog.rs @@ -103,16 +103,21 @@ impl ProviderCatalog { /// /// It reads `settings.ec.partners` directly and never builds a /// `PartnerRegistry`, so a deploy-time `api_token` that holds a secret - /// store key name is fine. In Part 1 it holds generic pull providers - /// only, with no source owners. It validates their descriptors with - /// [`validate_identity_catalog`] before building. + /// store key name is fine. It holds the generic pull providers, then + /// the `registered` providers integrations contributed, with no source + /// owners yet. It validates every descriptor with + /// [`validate_identity_catalog`] before building, so a duplicate ID is + /// reported by that one validator. /// /// # Errors /// /// Returns [`TrustedServerError::Configuration`] when a pull partner /// cannot be described or the catalog rules fail. - pub(crate) fn from_settings(settings: &Settings) -> Result> { - let providers = settings + pub(crate) fn from_settings( + settings: &Settings, + registered: &[Arc], + ) -> Result> { + let mut providers = settings .ec .partners .iter() @@ -128,6 +133,7 @@ impl ProviderCatalog { }) }) .collect::, _>>()?; + providers.extend(registered.iter().cloned()); let descriptors: Vec<_> = providers .iter() .map(|provider| provider.descriptor().clone()) @@ -244,6 +250,7 @@ impl ProviderCatalog { mod tests { use super::*; use crate::ec::identity::descriptor::Captures; + use crate::ec::identity::examples::{ExampleIdProvider, example_config}; use crate::ec::identity::generic_pull::generic_pull_descriptors; use crate::ec::identity::test_support::{ec_partner, fixed_capture_provider}; use crate::settings::{ProviderSettings, RevokeScope}; @@ -359,7 +366,7 @@ mod tests { crate::ec::registry::PartnerRegistry::from_config(&settings.ec.partners).is_err(), "should hold settings the registry rejects" ); - let catalog = ProviderCatalog::from_settings(&settings).expect("should build"); + let catalog = ProviderCatalog::from_settings(&settings, &[]).expect("should build"); let id = "pull-ssp-example-com".parse().expect("should parse the ID"); assert!(catalog.is_configured(&id), "should hold the pull provider"); @@ -371,4 +378,33 @@ mod tests { "should have no owners in Part 1" ); } + + #[test] + fn registered_provider_with_a_generic_pull_id_is_rejected_by_the_shared_validator() { + let mut settings = create_test_settings(); + settings.ec.partners = vec![partner()]; + + let duplicate = fixed_capture_provider("pull-ssp-example-com", &["other.example.com"]); + let error = ProviderCatalog::from_settings(&settings, &[duplicate]) + .err() + .expect("should reject a registered ID that a pull partner already uses"); + assert!( + format!("{error}") + .contains("identity provider `pull-ssp-example-com` is declared more than once"), + "should report the duplicate ID: {error}" + ); + + let registered = + ExampleIdProvider::new(&example_config()).expect("should build the example provider"); + let catalog = ProviderCatalog::from_settings(&settings, &[Arc::new(registered)]) + .expect("control: a distinct registered ID should build"); + assert!( + catalog.is_configured(&"exampleid".parse().expect("should parse")), + "should hold the registered provider" + ); + assert!( + catalog.is_configured(&"pull-ssp-example-com".parse().expect("should parse")), + "should keep the pull provider" + ); + } } diff --git a/crates/trusted-server-core/src/ec/identity/examples.rs b/crates/trusted-server-core/src/ec/identity/examples.rs new file mode 100644 index 0000000000..b1ce42a572 --- /dev/null +++ b/crates/trusted-server-core/src/ec/identity/examples.rs @@ -0,0 +1,229 @@ +//! Fictional providers from spec 4.6, for tests of registration, coordination +//! and capture. + +use std::time::Duration; + +use error_stack::Report; +use http::Method; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use url::Url; +use validator::Validate; + +use super::descriptor::{Captures, ConsentPolicy, Input, ProviderDescriptor, Resolves}; +use super::exchange::{ProviderRequest, ProviderResponse}; +use super::ids::UnixTime; +use super::input::{CaptureInput, ResolveInput}; +use super::provider::{IdentityProvider, IdentityProviderConfig}; +use super::resolution::{IdentityProviderError, Resolution, ResolvedEid, ResolvedUid}; +use crate::settings::IntegrationConfig; + +/// Integration ID of [`ExampleIdProvider`]. +pub(crate) const EXAMPLEID_INTEGRATION_ID: &str = "exampleid"; + +/// Integration ID of [`ExampleCaptureProvider`]. +pub(crate) const EXAMPLECAPTURE_INTEGRATION_ID: &str = "examplecapture"; + +/// Config of the fictional `exampleid` integration. +#[derive(Debug, Clone, Deserialize, Validate)] +pub(crate) struct ExampleIdConfig { + pub enabled: bool, + #[validate(url)] + pub endpoint: String, + pub partner_id: u32, + pub gvl_id: u16, +} + +impl IntegrationConfig for ExampleIdConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +impl IdentityProviderConfig for ExampleIdConfig { + fn build_descriptor(&self) -> Result> { + ProviderDescriptor::builder(EXAMPLEID_INTEGRATION_ID.parse()?) + .with_source("exampleid.example.com".parse()?) + .with_resolves( + Resolves::new() + .with_endpoint(&self.endpoint) + .with_needs([Input::ClientIp, Input::UserAgent]) + .with_wants([Input::PageUrl]) + .with_private_state(), + ) + .with_consent_policy(ConsentPolicy::strict().with_tcf_vendor(self.gvl_id)) + .build() + } +} + +/// A valid [`ExampleIdConfig`]. +pub(crate) fn example_config() -> ExampleIdConfig { + ExampleIdConfig { + enabled: true, + endpoint: "https://api.example.com/fetch".to_owned(), + partner_id: 7, + gvl_id: 1, + } +} + +/// A resolving provider that round-trips private state. +pub(crate) struct ExampleIdProvider { + descriptor: ProviderDescriptor, + partner_id: u32, +} + +/// The private state [`ExampleIdProvider`] keeps between calls. +#[derive(Serialize, Deserialize)] +pub(crate) struct ExampleIdState { + signature: String, +} + +#[derive(Serialize)] +struct FetchBody<'a> { + partner: u32, + ip: String, + ua: &'a str, + page: Option<&'a str>, + gdpr: bool, + consent: Option<&'a str>, + signature: Option, +} + +#[derive(Deserialize)] +struct FetchReply { + uid: Option, + signature: Option, + link_type: Option, + max_age_sec: Option, +} + +impl ExampleIdProvider { + /// Builds the provider from its validated config. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the config cannot + /// describe a valid provider. + pub(crate) fn new(config: &ExampleIdConfig) -> Result> { + Ok(Self { + descriptor: config.build_descriptor()?, + partner_id: config.partner_id, + }) + } +} + +impl IdentityProvider for ExampleIdProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn request( + &self, + input: &ResolveInput<'_>, + ) -> Result, Report> { + let consent = input.consent(); + let body = FetchBody { + partner: self.partner_id, + ip: input.client_ip()?.to_string(), + ua: input.user_agent()?, + page: input.page_url().ok().map(Url::as_str), + gdpr: consent.gdpr_applies(), + consent: consent.raw_tc_string(), + signature: input + .state::()? + .map(|state| state.signature), + }; + ProviderRequest::post_json(input.endpoint()?, &body).map(Some) + } + + fn response( + &self, + response: &ProviderResponse, + input: &ResolveInput<'_>, + ) -> Result> { + let reply: FetchReply = response.success_json()?; + let Some(uid) = reply.uid else { + return Ok(Resolution::no_match()); + }; + + let mut uid = ResolvedUid::new(uid).with_atype(1); + if let Some(link_type) = reply.link_type { + uid = uid.with_ext(json!({ "linkType": link_type })); + } + let mut eid = ResolvedEid::new(input.primary_source(), uid); + if let Some(max_age) = reply.max_age_sec { + eid = eid.with_refresh_in(Duration::from_secs(max_age)); + } + + let resolution = Resolution::found([eid]); + Ok(match reply.signature { + Some(signature) => resolution.replacing_state(&ExampleIdState { signature }), + None => resolution, + }) + } +} + +/// A capture-only provider that reads tokens from the API it proxies. +pub(crate) struct ExampleCaptureProvider { + descriptor: ProviderDescriptor, +} + +#[derive(Deserialize)] +struct TokenReply { + tokens: Vec, +} + +#[derive(Deserialize)] +struct Token { + source: String, + value: String, + expires: u64, +} + +impl ExampleCaptureProvider { + /// Builds the provider with the capture path of spec 4.6. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the descriptor is invalid. + pub(crate) fn new() -> Result> { + Self::with_capture_path("/integrations/examplecapture/api/token") + } + + /// Builds the provider capturing from `path` instead. + /// + /// # Errors + /// + /// Returns [`IdentityProviderError::Config`] when the descriptor is invalid. + pub(crate) fn with_capture_path(path: &str) -> Result> { + let descriptor = ProviderDescriptor::builder(EXAMPLECAPTURE_INTEGRATION_ID.parse()?) + .with_source("examplecapture.example.com".parse()?) + .with_captures(Captures::new().with_path(Method::POST, path)) + .build()?; + Ok(Self { descriptor }) + } +} + +impl IdentityProvider for ExampleCaptureProvider { + fn descriptor(&self) -> &ProviderDescriptor { + &self.descriptor + } + + fn capture( + &self, + input: &CaptureInput<'_>, + ) -> Result> { + let Ok(reply) = input.response().json::() else { + return Ok(Resolution::no_match()); + }; + Ok(Resolution::found(reply.tokens.into_iter().filter_map( + |token| { + let source = input.source_named(&token.source)?; + Some( + ResolvedEid::new(source, ResolvedUid::new(token.value)) + .with_expires_at(UnixTime::from_secs(token.expires)), + ) + }, + ))) + } +} diff --git a/crates/trusted-server-core/src/ec/identity/flush.rs b/crates/trusted-server-core/src/ec/identity/flush.rs index fa83f82fdf..b6200fa6d9 100644 --- a/crates/trusted-server-core/src/ec/identity/flush.rs +++ b/crates/trusted-server-core/src/ec/identity/flush.rs @@ -573,7 +573,11 @@ mod tests { } fn with_settings(row: Option<&KvEntry>, settings: Settings) -> Self { - let runtime = IdentityRuntime::from_settings(&settings).expect("should build runtime"); + let runtime = IdentityRuntime::from_settings( + &settings, + &crate::integrations::IntegrationRegistry::default(), + ) + .expect("should build runtime"); let client = Arc::new(FakeHttpClient::new()); let backend = Arc::new(FakeBackend::new()); let services = fake_services(&client, &backend); diff --git a/crates/trusted-server-core/src/ec/identity/mod.rs b/crates/trusted-server-core/src/ec/identity/mod.rs index a745c5085c..14fe2c80a0 100644 --- a/crates/trusted-server-core/src/ec/identity/mod.rs +++ b/crates/trusted-server-core/src/ec/identity/mod.rs @@ -10,6 +10,10 @@ pub(crate) mod catalog; pub(crate) mod coordinator; pub(crate) mod denials; pub mod descriptor; +// Temporary: later identity tasks import the items they use. +#[cfg(test)] +#[allow(dead_code)] +pub(crate) mod examples; pub mod exchange; pub mod flush; pub(crate) mod generic_pull; diff --git a/crates/trusted-server-core/src/ec/identity/runtime.rs b/crates/trusted-server-core/src/ec/identity/runtime.rs index 5ae2a52f25..f14b07b2d2 100644 --- a/crates/trusted-server-core/src/ec/identity/runtime.rs +++ b/crates/trusted-server-core/src/ec/identity/runtime.rs @@ -11,6 +11,7 @@ use super::input::RandomSource; use super::revision::IdentityKeys; use crate::ec::rate_limiter::RateLimiter; use crate::error::TrustedServerError; +use crate::integrations::IntegrationRegistry; use crate::platform::RuntimeServices; use crate::settings::{IdentitySettings, Settings}; @@ -39,14 +40,18 @@ impl core::fmt::Debug for IdentityRuntime { } impl IdentityRuntime { - /// Builds the runtime from settings, without resolving any secret. + /// Builds the runtime from settings and the providers `integrations` + /// registered, without resolving any secret. /// /// # Errors /// /// Returns [`TrustedServerError::Configuration`] when a provider cannot be /// described or the catalog rules fail. - pub fn from_settings(settings: &Settings) -> Result> { - let catalog = ProviderCatalog::from_settings(settings)? + pub fn from_settings( + settings: &Settings, + integrations: &IntegrationRegistry, + ) -> Result> { + let catalog = ProviderCatalog::from_settings(settings, &integrations.identity_providers())? .with_narrowing(narrowing(&settings.ec.identity)); Ok(Self { catalog, @@ -170,7 +175,8 @@ mod tests { #[test] fn runtime_debug_prints_no_key_material() { let settings = settings_with_listed_sources(None); - let runtime = IdentityRuntime::from_settings(&settings).expect("should build"); + let runtime = IdentityRuntime::from_settings(&settings, &IntegrationRegistry::default()) + .expect("should build"); let printed = format!("{runtime:?}"); @@ -192,15 +198,20 @@ mod tests { fn runtime_narrows_providers_by_their_sources_table() { let id = pid("pull-ssp-example-com"); - let unlisted = IdentityRuntime::from_settings(&settings_with_listed_sources(None)) - .expect("should build"); - let kept = IdentityRuntime::from_settings(&settings_with_listed_sources(Some(vec![ - "ssp.example.com".to_owned(), - ]))) + let unlisted = IdentityRuntime::from_settings( + &settings_with_listed_sources(None), + &IntegrationRegistry::default(), + ) + .expect("should build"); + let kept = IdentityRuntime::from_settings( + &settings_with_listed_sources(Some(vec!["ssp.example.com".to_owned()])), + &IntegrationRegistry::default(), + ) .expect("should build"); - let dropped = IdentityRuntime::from_settings(&settings_with_listed_sources(Some(vec![ - "other.example.com".to_owned(), - ]))) + let dropped = IdentityRuntime::from_settings( + &settings_with_listed_sources(Some(vec!["other.example.com".to_owned()])), + &IntegrationRegistry::default(), + ) .expect("should build"); assert_eq!( diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index c5ad86d426..3af01d073c 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -12,6 +12,7 @@ use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; use crate::ec::finalize::BrowserSeed; +use crate::ec::identity::{IdentityProvider, IdentityProviderError}; use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; @@ -630,6 +631,7 @@ pub struct IntegrationRegistration { pub html_stream_processors: Vec>, pub head_injectors: Vec>, pub request_filters: Vec>, + pub identity_providers: Vec>, } impl IntegrationRegistration { @@ -656,6 +658,7 @@ impl IntegrationRegistrationBuilder { html_stream_processors: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), + identity_providers: Vec::new(), }, } } @@ -702,6 +705,34 @@ impl IntegrationRegistrationBuilder { self } + /// Registers an identity provider this integration contributes. + #[must_use] + pub fn with_identity_provider(mut self, provider: Arc) -> Self { + self.registration.identity_providers.push(provider); + self + } + + /// Registers a provider built by a fallible constructor. + /// + /// A failure becomes [`TrustedServerError::Integration`], naming the + /// integration the builder already holds. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Integration`] when `provider` is an error. + pub fn try_with_identity_provider( + self, + provider: Result>, + ) -> Result> { + match provider { + Ok(provider) => Ok(self.with_identity_provider(Arc::new(provider))), + Err(error) => Err(Report::new(TrustedServerError::Integration { + integration: self.registration.integration_id.to_owned(), + message: error.current_context().to_string(), + })), + } + } + /// Mark this integration's JS module for deferred loading via /// `